From 1ed4fcd1e0d19cfa6382d0b49c505a46fdfe94e2 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Sun, 16 Oct 2022 13:11:13 +0800 Subject: [PATCH] fix: modify function call parameters --- hrp/pkg/uixt/demo/main_test.go | 2 +- hrp/pkg/uixt/ext.go | 78 +++++++++++++++++++++++---- hrp/pkg/uixt/ocr_vedem.go | 99 ++++++++++++++-------------------- hrp/pkg/uixt/tap.go | 12 ++--- hrp/step.go | 2 +- 5 files changed, 116 insertions(+), 77 deletions(-) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 9456a550..6dac96d2 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -32,7 +32,7 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}, nil) + points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 4696c656..d61aad8b 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -67,7 +67,7 @@ type MobileAction struct { Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app - RecognitionArea []float64 `json:"recognition_area,omitempty" yaml:"recognition_area,omitempty"` // used by ocr to get text position in the recognition area + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found @@ -104,10 +104,10 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { } } -// WithRecognitionArea inputs area of [(x1,y1), (x2,y2)] -func WithRecognitionArea(x1, y1, x2, y2 float64) ActionOption { +// WithScope inputs area of [(x1,y1), (x2,y2)] +func WithScope(x1, y1, x2, y2 float64) ActionOption { return func(o *MobileAction) { - o.RecognitionArea = []float64{x1, y1, x2, y2} + o.Scope = []float64{x1, y1, x2, y2} } } @@ -310,7 +310,7 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) { func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search, nil, index...) + return dExt.FindTextByOCR(search, WithCustomOption("index", index)) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, index...) @@ -347,7 +347,7 @@ func (dExt *DriverExt) IsLabelExist(label string) bool { } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, _, _, _, err := dExt.FindTextByOCR(text, nil) + _, _, _, _, err := dExt.FindTextByOCR(text) return err == nil } @@ -379,10 +379,25 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { AppLaunchUnattached, action.Params) case ACTION_SwipeToTapApp: if appName, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(appName, action.RecognitionArea, action.Index) + point, err = d.GetTextXY(appName, options...) return err } foundAppAction := func(d *DriverExt) error { @@ -411,10 +426,25 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { ACTION_SwipeToTapApp, action.Params) case ACTION_SwipeToTapText: if text, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findText := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(text, action.RecognitionArea, action.Index) + point, err = d.GetTextXY(text, options...) return err } foundTextAction := func(d *DriverExt) error { @@ -444,10 +474,24 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Params = textList } if texts, ok := action.Params.([]string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findText := func(d *DriverExt) error { var err error - points, err := d.GetTextXYs(texts, action.RecognitionArea) + points, err := d.GetTextXYs(texts, options...) if err != nil { return err } @@ -519,7 +563,21 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.RecognitionArea, action.Index) + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, options...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 11388ba6..cc79dcb5 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -110,9 +110,29 @@ func getLogID(header http.Header) string { return logID[0] } -func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []int, index ...int) (rect image.Rectangle, err error) { - if len(index) == 0 { - index = []int{0} // index not specified +func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := map[string]interface{}{} + for _, option := range options { + option(data) + } + + if _, ok := data["index"]; !ok { + data["index"] = []int{0} // index not specified + } + + index, ok := data["index"].([]int) + if !ok || len(index) == 0 { + index = []int{0} + } + + _, ok = data["scope"] + if !ok { + data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified + } + + scope, ok := data["scope"].([]int) + if !ok || len(scope) != 4 { + scope = []int{0, 0, math.MaxInt64, math.MaxInt64} } ocrResults, err := s.getOCRResult(imageBuf) @@ -121,22 +141,6 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in return } - if len(recAbsArea) != 4 { - recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} - } - - var minX, minY, maxX, maxY int - if recAbsArea[0] < recAbsArea[2] { - minX, maxX = recAbsArea[0], recAbsArea[2] - } else { - minX, maxX = recAbsArea[2], recAbsArea[0] - } - if recAbsArea[1] < recAbsArea[3] { - minY, maxY = recAbsArea[1], recAbsArea[3] - } else { - minY, maxY = recAbsArea[3], recAbsArea[1] - } - var rects []image.Rectangle var ocrTexts []string for _, ocrResult := range ocrResults { @@ -151,7 +155,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in Y: int(ocrResult.Points[2].Y), }, } - if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { ocrTexts = append(ocrTexts, ocrResult.Text) // not contains text @@ -196,27 +200,26 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in return rects[idx], nil } -func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, recAbsArea []int) (rects []image.Rectangle, err error) { +func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ...DataOption) (rects []image.Rectangle, err error) { ocrResults, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return } - if len(recAbsArea) != 4 { - recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} + data := map[string]interface{}{} + for _, option := range options { + option(data) } - var minX, minY, maxX, maxY int - if recAbsArea[0] < recAbsArea[2] { - minX, maxX = recAbsArea[0], recAbsArea[2] - } else { - minX, maxX = recAbsArea[2], recAbsArea[0] + _, ok := data["scope"] + if !ok { + data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified } - if recAbsArea[1] < recAbsArea[3] { - minY, maxY = recAbsArea[1], recAbsArea[3] - } else { - minY, maxY = recAbsArea[3], recAbsArea[1] + + scope, ok := data["scope"].([]int) + if !ok || len(scope) != 4 { + scope = []int{0, 0, math.MaxInt64, math.MaxInt64} } var success bool @@ -237,7 +240,7 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, recAbsArea }, } - if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { ocrTexts = append(ocrTexts, ocrResult.Text) // not contains text @@ -268,26 +271,15 @@ type OCRService interface { FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) } -func (dExt *DriverExt) FindTextByOCR(ocrText string, recognitionArea []float64, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } - if len(recognitionArea) != 4 { - recognitionArea = []float64{0, 0, 1, 1} - } - - absArea := []int{ - int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), - } - service := &veDEMOCRService{} - rect, err := service.FindText(ocrText, bufSource.Bytes(), absArea, index...) + rect, err := service.FindText(ocrText, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) err = fmt.Errorf("FindText failed: %v", err) @@ -300,26 +292,15 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, recognitionArea []float64, return } -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, recognitionArea []float64) (points [][]float64, err error) { +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } - if len(recognitionArea) != 4 { - recognitionArea = []float64{0, 0, 1, 1} - } - - absArea := []int{ - int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), - } - service := &veDEMOCRService{} - rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), absArea) + rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindTexts failed: %s", err.Error()) err = fmt.Errorf("FindTexts failed: %v", err) diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 611a5174..d46c34aa 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -28,8 +28,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { return dExt.TapAbsXY(x, y, identifier) } -func (dExt *DriverExt) GetTextXY(ocrText string, recognitionArea []float64, index ...int) (point PointF, err error) { - x, y, width, height, err := dExt.FindTextByOCR(ocrText, recognitionArea, index...) +func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) { + x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...) if err != nil { return PointF{}, err } @@ -41,8 +41,8 @@ func (dExt *DriverExt) GetTextXY(ocrText string, recognitionArea []float64, inde return point, nil } -func (dExt *DriverExt) GetTextXYs(ocrText []string, recognitionArea []float64) (points []PointF, err error) { - ps, err := dExt.FindTextsByOCR(ocrText, recognitionArea) +func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) { + ps, err := dExt.FindTextsByOCR(ocrText, options...) if err != nil { return nil, err } @@ -71,8 +71,8 @@ func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, return point, nil } -func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, recognitionArea []float64, index ...int) error { - point, err := dExt.GetTextXY(ocrText, recognitionArea, index...) +func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, options ...DataOption) error { + point, err := dExt.GetTextXY(ocrText, options...) if err != nil { if ignoreNotFoundError { return nil diff --git a/hrp/step.go b/hrp/step.go index 8873f947..1f3cb5ee 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -31,7 +31,7 @@ var ( WithDescription = uixt.WithDescription WithDirection = uixt.WithDirection WithCustomDirection = uixt.WithCustomDirection - WithRecognitionArea = uixt.WithRecognitionArea + WithScope = uixt.WithScope ) var (