From 6608d1bdec8465c37ef871176c2a0999efaf8000 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 27 Sep 2022 17:32:56 +0800 Subject: [PATCH] feat: add TapAbsXY --- hrp/internal/uixt/ext.go | 18 ++++++++++++++---- hrp/internal/uixt/swipe_test.go | 8 ++++---- hrp/internal/uixt/tap.go | 32 +++++++++++++++++++++++--------- hrp/internal/uixt/tap_test.go | 8 ++++++++ hrp/step_ios_ui.go | 13 +++++++++++++ 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go index edcabd51..d0bd92df 100644 --- a/hrp/internal/uixt/ext.go +++ b/hrp/internal/uixt/ext.go @@ -44,6 +44,7 @@ const ( // UI handling ACTION_Home MobileMethod = "home" ACTION_TapXY MobileMethod = "tap_xy" + ACTION_TapAbsXY MobileMethod = "tap_abs_xy" ACTION_TapByOCR MobileMethod = "tap_ocr" ACTION_TapByCV MobileMethod = "tap_cv" ACTION_Tap MobileMethod = "tap" @@ -317,12 +318,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextCoordinate(appName, action.Index) + point, err = d.GetTextXY(appName, action.Index) return err } foundAppAction := func(d *DriverExt) error { // click app to launch - return d.tapFloat(point.X, point.Y-20, action.Identifier) + return d.TapAbsXY(point.X, point.Y-20, action.Identifier) } // go to home screen @@ -349,12 +350,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findText := func(d *DriverExt) error { var err error - point, err = d.GetTextCoordinate(text, action.Index) + point, err = d.GetTextXY(text, action.Index) return err } foundTextAction := func(d *DriverExt) error { // tap text - return d.tapFloat(point.X, point.Y, action.Identifier) + return d.TapAbsXY(point.X, point.Y, action.Identifier) } // default to retry 10 times @@ -389,6 +390,15 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return dExt.TapXY(location[0], location[1], action.Identifier) } return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) + case ACTION_TapAbsXY: + if location, ok := action.Params.([]float64); ok { + // absolute coordinates x,y of window size: [100, 300] + if len(location) != 2 { + return fmt.Errorf("invalid tap location params: %v", location) + } + return dExt.TapAbsXY(location[0], location[1], action.Identifier) + } + return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) case ACTION_Tap: if param, ok := action.Params.(string); ok { return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index) diff --git a/hrp/internal/uixt/swipe_test.go b/hrp/internal/uixt/swipe_test.go index e52610c0..5314a663 100644 --- a/hrp/internal/uixt/swipe_test.go +++ b/hrp/internal/uixt/swipe_test.go @@ -11,12 +11,12 @@ func TestSwipeUntil(t *testing.T) { var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextCoordinate("抖音") + point, err = d.GetTextXY("抖音") return err } foundAppAction := func(d *DriverExt) error { // click app, launch douyin - return d.tapFloat(point.X, point.Y, "") + return d.TapAbsXY(point.X, point.Y, "") } driverExt.Driver.Homescreen() @@ -32,12 +32,12 @@ func TestSwipeUntil(t *testing.T) { findLive := func(d *DriverExt) error { var err error - point, err = d.GetTextCoordinate("点击进入直播间") + point, err = d.GetTextXY("点击进入直播间") return err } foundLiveAction := func(d *DriverExt) error { // enter live room - return d.tapFloat(point.X, point.Y, "") + return d.TapAbsXY(point.X, point.Y, "") } // swipe until live room found diff --git a/hrp/internal/uixt/tap.go b/hrp/internal/uixt/tap.go index a4105418..ada1ae73 100644 --- a/hrp/internal/uixt/tap.go +++ b/hrp/internal/uixt/tap.go @@ -4,7 +4,8 @@ import ( "fmt" ) -func (dExt *DriverExt) tapFloat(x, y float64, identifier string) error { +func (dExt *DriverExt) TapAbsXY(x, y float64, identifier string) error { + // tap on absolute coordinate [x, y] if len(identifier) > 0 { option := WithCustomOption("log", map[string]interface{}{ "enable": true, @@ -16,7 +17,7 @@ func (dExt *DriverExt) tapFloat(x, y float64, identifier string) error { } func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { - // tap on coordinate: [x, y] should be relative + // tap on [x, y] percent of window size if x > 1 || y > 1 { return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) } @@ -24,10 +25,10 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { x = x * float64(dExt.windowSize.Width) y = y * float64(dExt.windowSize.Height) - return dExt.tapFloat(x, y, identifier) + return dExt.TapAbsXY(x, y, identifier) } -func (dExt *DriverExt) GetTextCoordinate(ocrText string, index ...int) (point PointF, err error) { +func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, err error) { x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...) if err != nil { return PointF{}, err @@ -40,8 +41,21 @@ func (dExt *DriverExt) GetTextCoordinate(ocrText string, index ...int) (point Po return point, nil } +func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) { + x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...) + if err != nil { + return PointF{}, err + } + + point = PointF{ + X: x + width*0.5, + Y: y + height*0.5, + } + return point, nil +} + func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error { - point, err := dExt.GetTextCoordinate(ocrText, index...) + point, err := dExt.GetTextXY(ocrText, index...) if err != nil { if ignoreNotFoundError { return nil @@ -49,11 +63,11 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun return err } - return dExt.tapFloat(point.X, point.Y, identifier) + return dExt.TapAbsXY(point.X, point.Y, identifier) } func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error { - x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath) + point, err := dExt.GetImageXY(imagePath, index...) if err != nil { if ignoreNotFoundError { return nil @@ -61,7 +75,7 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou return err } - return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier) + return dExt.TapAbsXY(point.X, point.Y, identifier) } func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error { @@ -83,7 +97,7 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identif return err } - return dExt.tapFloat(x+width*xOffset, y+height*yOffset, identifier) + return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, identifier) } func (dExt *DriverExt) DoubleTapXY(x, y float64) error { diff --git a/hrp/internal/uixt/tap_test.go b/hrp/internal/uixt/tap_test.go index bde3c861..faaaa14e 100644 --- a/hrp/internal/uixt/tap_test.go +++ b/hrp/internal/uixt/tap_test.go @@ -25,6 +25,14 @@ func TestDriverExt_TapXY(t *testing.T) { checkErr(t, err) } +func TestDriverExt_TapAbsXY(t *testing.T) { + driverExt, err := InitWDAClient(nil) + checkErr(t, err) + + err = driverExt.TapAbsXY(100, 300, "") + checkErr(t, err) +} + func TestDriverExt_TapWithOCR(t *testing.T) { driverExt, err := InitWDAClient(nil) checkErr(t, err) diff --git a/hrp/step_ios_ui.go b/hrp/step_ios_ui.go index 588b3728..69c63d05 100644 --- a/hrp/step_ios_ui.go +++ b/hrp/step_ios_ui.go @@ -85,6 +85,19 @@ func (s *StepIOS) TapXY(x, y float64, options ...uixt.ActionOption) *StepIOS { return &StepIOS{step: s.step} } +// TapAbsXY taps the point {X,Y}, X & Y is absolute coordinates +func (s *StepIOS) TapAbsXY(x, y float64, options ...uixt.ActionOption) *StepIOS { + action := uixt.MobileAction{ + Method: uixt.ACTION_TapAbsXY, + Params: []float64{x, y}, + } + for _, option := range options { + option(&action) + } + s.step.IOS.Actions = append(s.step.IOS.Actions, action) + return &StepIOS{step: s.step} +} + // Tap taps on the target element func (s *StepIOS) Tap(params string, options ...uixt.ActionOption) *StepIOS { action := uixt.MobileAction{