From 6b2b662bb16df56ba311114d4613870842fd1c12 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 12 Oct 2022 20:25:02 +0800 Subject: [PATCH] refactor: vedem ocr get texts --- hrp/pkg/uixt/README.md | 8 +++--- hrp/pkg/uixt/demo/main_test.go | 7 ++--- hrp/pkg/uixt/ocr_off.go | 17 ------------ hrp/pkg/uixt/{ocr_on.go => ocr_vedem.go} | 35 +++++++++--------------- hrp/pkg/uixt/tap.go | 14 ++++------ scripts/build.sh | 6 ++-- 6 files changed, 28 insertions(+), 59 deletions(-) delete mode 100644 hrp/pkg/uixt/ocr_off.go rename hrp/pkg/uixt/{ocr_on.go => ocr_vedem.go} (88%) diff --git a/hrp/pkg/uixt/README.md b/hrp/pkg/uixt/README.md index 6fc5449a..c422d5d6 100644 --- a/hrp/pkg/uixt/README.md +++ b/hrp/pkg/uixt/README.md @@ -26,11 +26,11 @@ You can get more installation introduction on [hybridgroup/gocv]. ### OCR -OCR API is a paid service, you need to pre-purchase and configure the account key. +OCR API is a paid service, you need to pre-purchase and configure the environment variables. -```bash -$ make build tags=ocr -``` +- VEDEM_OCR_URL +- VEDEM_OCR_AK +- VEDEM_OCR_SK ## Thanks diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index e544eee6..044f3f92 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -30,14 +30,13 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - _, err1 := driverExt.GetTextXY("青少年模式") - point, err2 := driverExt.GetTextXY("我知道了") - if err1 != nil || err2 != nil { + points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) + if err != nil { time.Sleep(1 * time.Second) continue } - err := driverExt.TapAbsXY(point.X, point.Y, "") + err = driverExt.TapAbsXY(points[1].X, points[1].Y, "") if err != nil { t.Fatal(err) } diff --git a/hrp/pkg/uixt/ocr_off.go b/hrp/pkg/uixt/ocr_off.go deleted file mode 100644 index 03b2e505..00000000 --- a/hrp/pkg/uixt/ocr_off.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build !ocr - -package uixt - -import ( - "github.com/rs/zerolog/log" -) - -func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) { - log.Fatal().Msg("OCR is not supported") - return -} - -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) { - log.Fatal().Msg("OCR is not supported") - return -} diff --git a/hrp/pkg/uixt/ocr_on.go b/hrp/pkg/uixt/ocr_vedem.go similarity index 88% rename from hrp/pkg/uixt/ocr_on.go rename to hrp/pkg/uixt/ocr_vedem.go index 7a5faf91..5ca2ef37 100644 --- a/hrp/pkg/uixt/ocr_on.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -1,5 +1,3 @@ -//go:build ocr - package uixt import ( @@ -159,25 +157,22 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) ( return rects[idx], nil } -func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[string]image.Rectangle, err error) { +func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []image.Rectangle, err error) { ocrResults, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return } - var ocrTexts []string - rects = map[string]image.Rectangle{} - for _, text := range texts { + var found bool for _, ocrResult := range ocrResults { - ocrTexts = append(ocrTexts, ocrResult.Text) - // not contains text if !strings.Contains(ocrResult.Text, text) { continue } + found = true rect := image.Rectangle{ // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 Min: image.Point{ @@ -189,12 +184,11 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[ Y: int(ocrResult.Points[2].Y), }, } - rects[text] = rect + rects = append(rects, rect) break } - - if _, ok := rects[text]; !ok { - rects[text] = image.Rectangle{} + if !found { + rects = append(rects, image.Rectangle{}) } } @@ -220,12 +214,13 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, return } - log.Info().Str("ocrText", ocrText).Msgf("FindText success") + log.Info().Str("ocrText", ocrText). + Interface("rect", rect).Msgf("FindTextByOCR success") x, y, width, height = dExt.MappingToRectInUIKit(rect) return } -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) { +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (points [][]float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) @@ -240,15 +235,11 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float6 return } - ps = map[string][]float64{} - log.Info().Interface("ocrTexts", ocrTexts).Msgf("FindTexts success") - for text, rect := range rects { - if rect == (image.Rectangle{}) { - ps[text] = []float64{} - continue - } + log.Info().Interface("ocrTexts", ocrTexts). + Interface("rects", rects).Msgf("FindTextsByOCR success") + for _, rect := range rects { x, y, width, height := dExt.MappingToRectInUIKit(rect) - ps[text] = []float64{x, y, width, height} + points = append(points, []float64{x, y, width, height}) } return diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 10e1f79c..7957f3dd 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -41,22 +41,18 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er return point, nil } -func (dExt *DriverExt) GetTextXYs(ocrText []string) (points map[string]PointF, err error) { +func (dExt *DriverExt) GetTextXYs(ocrText []string) (points []PointF, err error) { ps, err := dExt.FindTextsByOCR(ocrText) if err != nil { - return map[string]PointF{}, err + return nil, err } - points = map[string]PointF{} - for text, point := range ps { - if len(point) == 0 { - points[text] = PointF{} - continue - } - points[text] = PointF{ + for _, point := range ps { + pointF := PointF{ X: point[0] + point[2]*0.5, Y: point[1] + point[3]*0.5, } + points = append(points, pointF) } return points, nil diff --git a/scripts/build.sh b/scripts/build.sh index 67c514ce..e867f9d3 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,10 +4,10 @@ # Usage: # $ make build -# $ make build tags=ocr +# $ make build tags=opencv # or # $ bash scripts/build.sh -# $ bash scripts/build.sh ocr +# $ bash scripts/build.sh opencv set -e set -x @@ -16,7 +16,7 @@ set -x mkdir -p "output" bin_path="output/hrp" -# optional build tags: opencv ocr +# optional build tags: opencv tags=$1 # build