feat: tap by ocr and cv

hrp/internal/uixt/ext.go

@@ -297,9 +297,9 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height flo
 	return dExt.FindImageRectInUIKit(search)
 }
 
-func (dExt *DriverExt) FindImageRectInUIKit(search string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
 	var bufSource, bufSearch *bytes.Buffer
-	if bufSearch, err = getBufFromDisk(search); err != nil {
+	if bufSearch, err = getBufFromDisk(imagePath); err != nil {
 		return 0, 0, 0, 0, err
 	}
 	if bufSource, err = dExt.takeScreenShot(); err != nil {

hrp/internal/uixt/ext_ocr.go

@@ -118,7 +118,7 @@ type OCRService interface {
 	FindText(text string, imageBuf []byte) (rect image.Rectangle, err error)
 }
 
-func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
 	var bufSource *bytes.Buffer
 	if bufSource, err = dExt.takeScreenShot(); err != nil {
 		err = fmt.Errorf("screenshot error: %v", err)
@@ -126,7 +126,7 @@ func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64
 	}
 
 	service := &veDEMOCRService{}
-	rect, err := service.FindText(search, bufSource.Bytes())
+	rect, err := service.FindText(ocrText, bufSource.Bytes())
 	if err != nil {
 		err = fmt.Errorf("find text failed: %v", err)
 		return

hrp/internal/uixt/tap.go

@@ -17,6 +17,24 @@ func (dExt *DriverExt) TapXY(x, y float64) error {
 	return dExt.WebDriver.TapFloat(x, y)
 }
 
+func (dExt *DriverExt) TapByOCR(ocrText string) error {
+	x, y, width, height, err := dExt.FindTextByOCR(ocrText)
+	if err != nil {
+		return err
+	}
+
+	return dExt.WebDriver.TapFloat(x+width*0.5, y+height*0.5)
+}
+
+func (dExt *DriverExt) TapByCV(imagePath string) error {
+	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath)
+	if err != nil {
+		return err
+	}
+
+	return dExt.WebDriver.TapFloat(x+width*0.5, y+height*0.5)
+}
+
 func (dExt *DriverExt) Tap(param string) error {
 	return dExt.TapOffset(param, 0.5, 0.5)
 }

hrp/step.go

@@ -34,6 +34,8 @@ const (
 	// UI handling
 	uiHome        MobileMethod = "home"
 	uiTapXY       MobileMethod = "tap_xy"
+	uiTapByOCR    MobileMethod = "tap_ocr"
+	uiTapByCV     MobileMethod = "tap_cv"
 	uiTap         MobileMethod = "tap"
 	uiDoubleTapXY MobileMethod = "double_tap_xy"
 	uiDoubleTap   MobileMethod = "double_tap"

hrp/step_ios_ui.go

@@ -95,6 +95,24 @@ func (s *StepIOS) Tap(params string) *StepIOS {
 	return &StepIOS{step: s.step}
 }
 
+// Tap taps on the target element by OCR recognition
+func (s *StepIOS) TapByOCR(ocrText string) *StepIOS {
+	s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{
+		Method: uiTapByOCR,
+		Params: ocrText,
+	})
+	return &StepIOS{step: s.step}
+}
+
+// Tap taps on the target element by CV recognition
+func (s *StepIOS) TapByCV(imagePath string) *StepIOS {
+	s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{
+		Method: uiTapByCV,
+		Params: imagePath,
+	})
+	return &StepIOS{step: s.step}
+}
+
 // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates
 func (s *StepIOS) DoubleTapXY(x, y float64) *StepIOS {
 	s.step.IOS.Actions = append(s.step.IOS.Actions, MobileAction{
@@ -484,6 +502,16 @@ func (ud *uiDriver) doAction(action MobileAction) error {
 			return ud.Tap(param)
 		}
 		return fmt.Errorf("invalid %s params: %v", uiTap, action.Params)
+	case uiTapByOCR:
+		if ocrText, ok := action.Params.(string); ok {
+			return ud.TapByOCR(ocrText)
+		}
+		return fmt.Errorf("invalid %s params: %v", uiTapByOCR, action.Params)
+	case uiTapByCV:
+		if imagePath, ok := action.Params.(string); ok {
+			return ud.TapByCV(imagePath)
+		}
+		return fmt.Errorf("invalid %s params: %v", uiTapByCV, action.Params)
 	case uiDoubleTapXY:
 		if location, ok := action.Params.([]float64); ok {
 			// relative x,y of window size: [0.5, 0.5]