refactor: vedem ocr get texts

hrp/pkg/uixt/README.md

@@ -26,11 +26,11 @@ You can get more installation introduction on [hybridgroup/gocv].
 
 ### OCR
 
-OCR API is a paid service, you need to pre-purchase and configure the account key.
+OCR API is a paid service, you need to pre-purchase and configure the environment variables.
 
-```bash
-$ make build tags=ocr
-```
+- VEDEM_OCR_URL
+- VEDEM_OCR_AK
+- VEDEM_OCR_SK
 
 ## Thanks
 

hrp/pkg/uixt/demo/main_test.go

@@ -30,14 +30,13 @@ func TestIOSDemo(t *testing.T) {
 
 	// 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」
 	for {
-		_, err1 := driverExt.GetTextXY("青少年模式")
-		point, err2 := driverExt.GetTextXY("我知道了")
-		if err1 != nil || err2 != nil {
+		points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"})
+		if err != nil {
 			time.Sleep(1 * time.Second)
 			continue
 		}
 
-		err := driverExt.TapAbsXY(point.X, point.Y, "")
+		err = driverExt.TapAbsXY(points[1].X, points[1].Y, "")
 		if err != nil {
 			t.Fatal(err)
 		}

hrp/pkg/uixt/ocr_off.go

@@ -1,17 +0,0 @@
-//go:build !ocr
-
-package uixt
-
-import (
-	"github.com/rs/zerolog/log"
-)
-
-func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
-	log.Fatal().Msg("OCR is not supported")
-	return
-}
-
-func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
-	log.Fatal().Msg("OCR is not supported")
-	return
-}

hrp/pkg/uixt/ocr_vedem.go

@@ -1,5 +1,3 @@
-//go:build ocr
-
 package uixt
 
 import (
@@ -159,25 +157,22 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
 	return rects[idx], nil
 }
 
-func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[string]image.Rectangle, err error) {
+func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []image.Rectangle, err error) {
 	ocrResults, err := s.getOCRResult(imageBuf)
 	if err != nil {
 		log.Error().Err(err).Msg("getOCRResult failed")
 		return
 	}
 
-	var ocrTexts []string
-	rects = map[string]image.Rectangle{}
-
 	for _, text := range texts {
+		var found bool
 		for _, ocrResult := range ocrResults {
-			ocrTexts = append(ocrTexts, ocrResult.Text)
-
 			// not contains text
 			if !strings.Contains(ocrResult.Text, text) {
 				continue
 			}
 
+			found = true
 			rect := image.Rectangle{
 				// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
 				Min: image.Point{
@@ -189,12 +184,11 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[
 					Y: int(ocrResult.Points[2].Y),
 				},
 			}
-			rects[text] = rect
+			rects = append(rects, rect)
 			break
 		}
-
-		if _, ok := rects[text]; !ok {
-			rects[text] = image.Rectangle{}
+		if !found {
+			rects = append(rects, image.Rectangle{})
 		}
 	}
 
@@ -220,12 +214,13 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width,
 		return
 	}
 
-	log.Info().Str("ocrText", ocrText).Msgf("FindText success")
+	log.Info().Str("ocrText", ocrText).
+		Interface("rect", rect).Msgf("FindTextByOCR success")
 	x, y, width, height = dExt.MappingToRectInUIKit(rect)
 	return
 }
 
-func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
+func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (points [][]float64, err error) {
 	var bufSource *bytes.Buffer
 	if bufSource, err = dExt.takeScreenShot(); err != nil {
 		err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -240,15 +235,11 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float6
 		return
 	}
 
-	ps = map[string][]float64{}
-	log.Info().Interface("ocrTexts", ocrTexts).Msgf("FindTexts success")
-	for text, rect := range rects {
-		if rect == (image.Rectangle{}) {
-			ps[text] = []float64{}
-			continue
-		}
+	log.Info().Interface("ocrTexts", ocrTexts).
+		Interface("rects", rects).Msgf("FindTextsByOCR success")
+	for _, rect := range rects {
 		x, y, width, height := dExt.MappingToRectInUIKit(rect)
-		ps[text] = []float64{x, y, width, height}
+		points = append(points, []float64{x, y, width, height})
 	}
 
 	return

hrp/pkg/uixt/tap.go

@@ -41,22 +41,18 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er
 	return point, nil
 }
 
-func (dExt *DriverExt) GetTextXYs(ocrText []string) (points map[string]PointF, err error) {
+func (dExt *DriverExt) GetTextXYs(ocrText []string) (points []PointF, err error) {
 	ps, err := dExt.FindTextsByOCR(ocrText)
 	if err != nil {
-		return map[string]PointF{}, err
+		return nil, err
 	}
 
-	points = map[string]PointF{}
-	for text, point := range ps {
-		if len(point) == 0 {
-			points[text] = PointF{}
-			continue
-		}
-		points[text] = PointF{
+	for _, point := range ps {
+		pointF := PointF{
 			X: point[0] + point[2]*0.5,
 			Y: point[1] + point[3]*0.5,
 		}
+		points = append(points, pointF)
 	}
 
 	return points, nil

scripts/build.sh

@@ -4,10 +4,10 @@
 
 # Usage:
 # $ make build
-# $ make build tags=ocr
+# $ make build tags=opencv
 # or
 # $ bash scripts/build.sh
-# $ bash scripts/build.sh ocr
+# $ bash scripts/build.sh opencv
 
 set -e
 set -x
@@ -16,7 +16,7 @@ set -x
 mkdir -p "output"
 bin_path="output/hrp"
 
-# optional build tags: opencv ocr
+# optional build tags: opencv
 tags=$1
 
 # build