diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go index cb8c57a3..7c9bb098 100644 --- a/hrp/internal/uixt/ext.go +++ b/hrp/internal/uixt/ext.go @@ -297,9 +297,9 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height flo return dExt.FindImageRectInUIKit(search) } -func (dExt *DriverExt) FindImageRectInUIKit(search string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(search); err != nil { + if bufSearch, err = getBufFromDisk(imagePath); err != nil { return 0, 0, 0, 0, err } if bufSource, err = dExt.takeScreenShot(); err != nil { diff --git a/hrp/internal/uixt/ext_ocr.go b/hrp/internal/uixt/ext_ocr.go index 78581b3e..dc267111 100644 --- a/hrp/internal/uixt/ext_ocr.go +++ b/hrp/internal/uixt/ext_ocr.go @@ -118,7 +118,7 @@ type OCRService interface { FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) } -func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("screenshot error: %v", err) @@ -126,7 +126,7 @@ func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64 } service := &veDEMOCRService{} - rect, err := service.FindText(search, bufSource.Bytes()) + rect, err := service.FindText(ocrText, bufSource.Bytes()) if err != nil { err = fmt.Errorf("find text failed: %v", err) return diff --git a/hrp/internal/uixt/tap.go b/hrp/internal/uixt/tap.go index d2c11290..5b7d1d42 100644 --- a/hrp/internal/uixt/tap.go +++ b/hrp/internal/uixt/tap.go @@ -17,6 +17,24 @@ func (dExt *DriverExt) TapXY(x, y float64) error { return dExt.WebDriver.TapFloat(x, y) } +func (dExt *DriverExt) TapByOCR(ocrText string) error { + x, y, width, height, err := dExt.FindTextByOCR(ocrText) + if err != nil { + return err + } + + return dExt.WebDriver.TapFloat(x+width*0.5, y+height*0.5) +} + +func (dExt *DriverExt) TapByCV(imagePath string) error { + x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath) + if err != nil { + return err + } + + return dExt.WebDriver.TapFloat(x+width*0.5, y+height*0.5) +} + func (dExt *DriverExt) Tap(param string) error { return dExt.TapOffset(param, 0.5, 0.5) } diff --git a/hrp/step.go b/hrp/step.go index 713b3417..d0b99aa4 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -34,6 +34,8 @@ const ( // UI handling uiHome MobileMethod = "home" uiTapXY MobileMethod = "tap_xy" + uiTapByOCR MobileMethod = "tap_ocr" + uiTapByCV MobileMethod = "tap_cv" uiTap MobileMethod = "tap" uiDoubleTapXY MobileMethod = "double_tap_xy" uiDoubleTap MobileMethod = "double_tap" diff --git a/hrp/step_ios_ui.go b/hrp/step_ios_ui.go index 823968cd..370234dd 100644 --- a/hrp/step_ios_ui.go +++ b/hrp/step_ios_ui.go @@ -95,6 +95,24 @@ func (s *StepIOS) Tap(params string) *StepIOS { return &StepIOS{step: s.step} } +// Tap taps on the target element by OCR recognition +func (s *StepIOS) TapByOCR(ocrText string) *StepIOS { + s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{ + Method: uiTapByOCR, + Params: ocrText, + }) + return &StepIOS{step: s.step} +} + +// Tap taps on the target element by CV recognition +func (s *StepIOS) TapByCV(imagePath string) *StepIOS { + s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{ + Method: uiTapByCV, + Params: imagePath, + }) + return &StepIOS{step: s.step} +} + // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates func (s *StepIOS) DoubleTapXY(x, y float64) *StepIOS { s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{ @@ -484,6 +502,16 @@ func (ud *uiDriver) doAction(action MobileAction) error { return ud.Tap(param) } return fmt.Errorf("invalid %s params: %v", uiTap, action.Params) + case uiTapByOCR: + if ocrText, ok := action.Params.(string); ok { + return ud.TapByOCR(ocrText) + } + return fmt.Errorf("invalid %s params: %v", uiTapByOCR, action.Params) + case uiTapByCV: + if imagePath, ok := action.Params.(string); ok { + return ud.TapByCV(imagePath) + } + return fmt.Errorf("invalid %s params: %v", uiTapByCV, action.Params) case uiDoubleTapXY: if location, ok := action.Params.([]float64); ok { // relative x,y of window size: [0.5, 0.5]