From 45bdfa36f2c15327a568ec25ccd30b0a6b11ad98 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 26 Apr 2023 22:47:00 +0800 Subject: [PATCH] fix: handle screen scale for iOS --- hrp/pkg/uixt/android_test.go | 14 -------------- hrp/pkg/uixt/client.go | 1 + hrp/pkg/uixt/ext.go | 21 ++++++--------------- hrp/pkg/uixt/ios_device.go | 10 ++++++++++ hrp/pkg/uixt/ios_driver.go | 35 ++++++++++++++++++++--------------- hrp/pkg/uixt/ios_test.go | 11 +++++++++++ hrp/pkg/uixt/opencv.go | 2 -- 7 files changed, 48 insertions(+), 46 deletions(-) diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index b2a89037..be5740ce 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -153,20 +153,6 @@ func TestDriver_GetAppiumSettings(t *testing.T) { // t.Log(appiumSettings) } -func TestDriver_DeviceScaleRatio(t *testing.T) { - driver, err := NewUIADriver(nil, uiaServerURL) - if err != nil { - t.Fatal(err) - } - - scaleRatio, err := driver.Scale() - if err != nil { - t.Fatal(err) - } - - t.Log(scaleRatio) -} - func TestDriver_DeviceInfo(t *testing.T) { driver, err := NewUIADriver(nil, uiaServerURL) if err != nil { diff --git a/hrp/pkg/uixt/client.go b/hrp/pkg/uixt/client.go index 95a8b277..b8166da8 100644 --- a/hrp/pkg/uixt/client.go +++ b/hrp/pkg/uixt/client.go @@ -20,6 +20,7 @@ type Driver struct { urlPrefix *url.URL sessionId string client *http.Client + scale float64 // cache the last launched package name lastLaunchedPackageName string } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index ce1aeb24..ffe61f64 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -51,7 +51,6 @@ type DriverExt struct { windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - scale float64 OCRService IOCRService // used to get text from image screenShots []string // cache screenshot paths @@ -71,10 +70,6 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, errors.Wrap(err, "failed to get windows size") } - if dExt.scale, err = dExt.Driver.Scale(); err != nil { - return nil, err - } - if dExt.OCRService, err = newVEDEMOCRService(); err != nil { return nil, err } @@ -187,12 +182,6 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) ( return dExt.FindImageRectInUIKit(search, options...) } -func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { - x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale - width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale - return -} - func (dExt *DriverExt) IsOCRExist(text string) bool { _, err := dExt.FindScreenTextByOCR(text) return err == nil @@ -213,11 +202,13 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool { return true } +// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner +// the value of (x, y) is between 0 and 1, which means the percentage of the screen func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { - return int(x1 * float64(dExt.windowSize.Width) * dExt.scale), - int(y1 * float64(dExt.windowSize.Height) * dExt.scale), - int(x2 * float64(dExt.windowSize.Width) * dExt.scale), - int(y2 * float64(dExt.windowSize.Height) * dExt.scale) + return int(x1 * float64(dExt.windowSize.Width)), + int(y1 * float64(dExt.windowSize.Height)), + int(x2 * float64(dExt.windowSize.Width)), + int(y2 * float64(dExt.windowSize.Height)) } func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { diff --git a/hrp/pkg/uixt/ios_device.go b/hrp/pkg/uixt/ios_device.go index 93828a8d..cac6bfce 100644 --- a/hrp/pkg/uixt/ios_device.go +++ b/hrp/pkg/uixt/ios_device.go @@ -631,6 +631,11 @@ func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver WebDriver } wd.mjpegClient = convertToHTTPClient(wd.mjpegHTTPConn) + // init WDA scale + if wd.scale, err = wd.Scale(); err != nil { + return nil, err + } + return wd, nil } @@ -659,6 +664,11 @@ func (dev *IOSDevice) NewUSBDriver(capabilities Capabilities) (driver WebDriver, return nil, errors.Wrap(code.IOSDeviceUSBDriverError, err.Error()) } + // init WDA scale + if wd.scale, err = wd.Scale(); err != nil { + return nil, err + } + return wd, nil } diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 830d78cc..64cb8f24 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -164,11 +164,16 @@ func (wd *wdaDriver) Screen() (screen Screen, err error) { func (wd *wdaDriver) Scale() (float64, error) { screen, err := wd.Screen() if err != nil { - return 0, err + return 0, errors.Wrap(code.MobileUIDriverError, + fmt.Sprintf("get screen info failed: %v", err)) } return screen.Scale, nil } +func (wd *wdaDriver) toScale(x float64) float64 { + return x / wd.scale +} + func (wd *wdaDriver) ActiveAppInfo() (info AppInfo, err error) { // [[FBRoute GET:@"/wda/activeAppInfo"] respondWithTarget:self action:@selector(handleActiveAppInfo:)] // [[FBRoute GET:@"/wda/activeAppInfo"].withoutSession @@ -375,8 +380,8 @@ func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } // new data options in post data for extra WDA configurations newData := NewData(data, options...) @@ -392,8 +397,8 @@ func (wd *wdaDriver) DoubleTap(x, y int) error { func (wd *wdaDriver) DoubleTapFloat(x, y float64) (err error) { // [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/doubleTap") return @@ -406,8 +411,8 @@ func (wd *wdaDriver) TouchAndHold(x, y int, second ...float64) error { func (wd *wdaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err error) { // [[FBRoute POST:@"/wda/touchAndHold"] respondWithTarget:self action:@selector(handleTouchAndHoldCoordinate:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } if len(second) == 0 || second[0] <= 0 { second = []float64{1.0} @@ -424,10 +429,10 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) err func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) { // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] data := map[string]interface{}{ - "fromX": fromX, - "fromY": fromY, - "toX": toX, - "toY": toY, + "fromX": wd.toScale(fromX), + "fromY": wd.toScale(fromY), + "toX": wd.toScale(toX), + "toY": wd.toScale(toY), } // new data options in post data for extra WDA configurations @@ -491,10 +496,10 @@ func (wd *wdaDriver) PressBack(options ...DataOption) (err error) { } data := map[string]interface{}{ - "fromX": float64(windowSize.Width) * 0, - "fromY": float64(windowSize.Height) * 0.5, - "toX": float64(windowSize.Width) * 0.6, - "toY": float64(windowSize.Height) * 0.5, + "fromX": wd.toScale(float64(windowSize.Width) * 0), + "fromY": wd.toScale(float64(windowSize.Height) * 0.5), + "toX": wd.toScale(float64(windowSize.Width) * 0.6), + "toY": wd.toScale(float64(windowSize.Height) * 0.5), } // new data options in post data for extra WDA configurations diff --git a/hrp/pkg/uixt/ios_test.go b/hrp/pkg/uixt/ios_test.go index c894f5aa..3865953a 100644 --- a/hrp/pkg/uixt/ios_test.go +++ b/hrp/pkg/uixt/ios_test.go @@ -68,6 +68,17 @@ func TestNewUSBDriver(t *testing.T) { // t.Log(driver.IsWdaHealthy()) } +func TestDriver_DeviceScaleRatio(t *testing.T) { + setup(t) + + scaleRatio, err := driver.Scale() + if err != nil { + t.Fatal(err) + } + + t.Log(scaleRatio) +} + func Test_remoteWD_DeleteSession(t *testing.T) { setup(t) diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index 635e18e0..e1b3cbdb 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -71,7 +71,6 @@ func (dExt *DriverExt) Debug(dm DebugMode) { func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { newExt = new(DriverExt) newExt.Driver = dExt.Driver - newExt.scale = dExt.scale newExt.matchMode = dExt.matchMode newExt.threshold = threshold return @@ -80,7 +79,6 @@ func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { newExt = new(DriverExt) newExt.Driver = dExt.Driver - newExt.scale = dExt.scale newExt.matchMode = matchMode newExt.threshold = dExt.threshold return