feat: add TapAbsXY

hrp/internal/uixt/ext.go

@@ -44,6 +44,7 @@ const (
 	// UI handling
 	ACTION_Home        MobileMethod = "home"
 	ACTION_TapXY       MobileMethod = "tap_xy"
+	ACTION_TapAbsXY    MobileMethod = "tap_abs_xy"
 	ACTION_TapByOCR    MobileMethod = "tap_ocr"
 	ACTION_TapByCV     MobileMethod = "tap_cv"
 	ACTION_Tap         MobileMethod = "tap"
@@ -317,12 +318,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			var point PointF
 			findApp := func(d *DriverExt) error {
 				var err error
-				point, err = d.GetTextCoordinate(appName, action.Index)
+				point, err = d.GetTextXY(appName, action.Index)
 				return err
 			}
 			foundAppAction := func(d *DriverExt) error {
 				// click app to launch
-				return d.tapFloat(point.X, point.Y-20, action.Identifier)
+				return d.TapAbsXY(point.X, point.Y-20, action.Identifier)
 			}
 
 			// go to home screen
@@ -349,12 +350,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			var point PointF
 			findText := func(d *DriverExt) error {
 				var err error
-				point, err = d.GetTextCoordinate(text, action.Index)
+				point, err = d.GetTextXY(text, action.Index)
 				return err
 			}
 			foundTextAction := func(d *DriverExt) error {
 				// tap text
-				return d.tapFloat(point.X, point.Y, action.Identifier)
+				return d.TapAbsXY(point.X, point.Y, action.Identifier)
 			}
 
 			// default to retry 10 times
@@ -389,6 +390,15 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			return dExt.TapXY(location[0], location[1], action.Identifier)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
+	case ACTION_TapAbsXY:
+		if location, ok := action.Params.([]float64); ok {
+			// absolute coordinates x,y of window size: [100, 300]
+			if len(location) != 2 {
+				return fmt.Errorf("invalid tap location params: %v", location)
+			}
+			return dExt.TapAbsXY(location[0], location[1], action.Identifier)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
 	case ACTION_Tap:
 		if param, ok := action.Params.(string); ok {
 			return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)

hrp/internal/uixt/swipe_test.go

@@ -11,12 +11,12 @@ func TestSwipeUntil(t *testing.T) {
 	var point PointF
 	findApp := func(d *DriverExt) error {
 		var err error
-		point, err = d.GetTextCoordinate("抖音")
+		point, err = d.GetTextXY("抖音")
 		return err
 	}
 	foundAppAction := func(d *DriverExt) error {
 		// click app, launch douyin
-		return d.tapFloat(point.X, point.Y, "")
+		return d.TapAbsXY(point.X, point.Y, "")
 	}
 
 	driverExt.Driver.Homescreen()
@@ -32,12 +32,12 @@ func TestSwipeUntil(t *testing.T) {
 
 	findLive := func(d *DriverExt) error {
 		var err error
-		point, err = d.GetTextCoordinate("点击进入直播间")
+		point, err = d.GetTextXY("点击进入直播间")
 		return err
 	}
 	foundLiveAction := func(d *DriverExt) error {
 		// enter live room
-		return d.tapFloat(point.X, point.Y, "")
+		return d.TapAbsXY(point.X, point.Y, "")
 	}
 
 	// swipe until live room found

hrp/internal/uixt/tap.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 )
 
-func (dExt *DriverExt) tapFloat(x, y float64, identifier string) error {
+func (dExt *DriverExt) TapAbsXY(x, y float64, identifier string) error {
+	// tap on absolute coordinate [x, y]
 	if len(identifier) > 0 {
 		option := WithCustomOption("log", map[string]interface{}{
 			"enable": true,
@@ -16,7 +17,7 @@ func (dExt *DriverExt) tapFloat(x, y float64, identifier string) error {
 }
 
 func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
-	// tap on coordinate: [x, y] should be relative
+	// tap on [x, y] percent of window size
 	if x > 1 || y > 1 {
 		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
 	}
@@ -24,10 +25,10 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
 	x = x * float64(dExt.windowSize.Width)
 	y = y * float64(dExt.windowSize.Height)
 
-	return dExt.tapFloat(x, y, identifier)
+	return dExt.TapAbsXY(x, y, identifier)
 }
 
-func (dExt *DriverExt) GetTextCoordinate(ocrText string, index ...int) (point PointF, err error) {
+func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, err error) {
 	x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
 	if err != nil {
 		return PointF{}, err
@@ -40,8 +41,21 @@ func (dExt *DriverExt) GetTextCoordinate(ocrText string, index ...int) (point Po
 	return point, nil
 }
 
+func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) {
+	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...)
+	if err != nil {
+		return PointF{}, err
+	}
+
+	point = PointF{
+		X: x + width*0.5,
+		Y: y + height*0.5,
+	}
+	return point, nil
+}
+
 func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
-	point, err := dExt.GetTextCoordinate(ocrText, index...)
+	point, err := dExt.GetTextXY(ocrText, index...)
 	if err != nil {
 		if ignoreNotFoundError {
 			return nil
@@ -49,11 +63,11 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun
 		return err
 	}
 
-	return dExt.tapFloat(point.X, point.Y, identifier)
+	return dExt.TapAbsXY(point.X, point.Y, identifier)
 }
 
 func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
-	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath)
+	point, err := dExt.GetImageXY(imagePath, index...)
 	if err != nil {
 		if ignoreNotFoundError {
 			return nil
@@ -61,7 +75,7 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou
 		return err
 	}
 
-	return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
+	return dExt.TapAbsXY(point.X, point.Y, identifier)
 }
 
 func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
@@ -83,7 +97,7 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identif
 		return err
 	}
 
-	return dExt.tapFloat(x+width*xOffset, y+height*yOffset, identifier)
+	return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, identifier)
 }
 
 func (dExt *DriverExt) DoubleTapXY(x, y float64) error {

hrp/internal/uixt/tap_test.go

@@ -25,6 +25,14 @@ func TestDriverExt_TapXY(t *testing.T) {
 	checkErr(t, err)
 }
 
+func TestDriverExt_TapAbsXY(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	err = driverExt.TapAbsXY(100, 300, "")
+	checkErr(t, err)
+}
+
 func TestDriverExt_TapWithOCR(t *testing.T) {
 	driverExt, err := InitWDAClient(nil)
 	checkErr(t, err)

hrp/step_ios_ui.go

@@ -85,6 +85,19 @@ func (s *StepIOS) TapXY(x, y float64, options ...uixt.ActionOption) *StepIOS {
 	return &StepIOS{step: s.step}
 }
 
+// TapAbsXY taps the point {X,Y}, X & Y is absolute coordinates
+func (s *StepIOS) TapAbsXY(x, y float64, options ...uixt.ActionOption) *StepIOS {
+	action := uixt.MobileAction{
+		Method: uixt.ACTION_TapAbsXY,
+		Params: []float64{x, y},
+	}
+	for _, option := range options {
+		option(&action)
+	}
+	s.step.IOS.Actions = append(s.step.IOS.Actions, action)
+	return &StepIOS{step: s.step}
+}
+
 // Tap taps on the target element
 func (s *StepIOS) Tap(params string, options ...uixt.ActionOption) *StepIOS {
 	action := uixt.MobileAction{