fix: handle screen scale for iOS

hrp/pkg/uixt/android_test.go

@@ -153,20 +153,6 @@ func TestDriver_GetAppiumSettings(t *testing.T) {
 	// t.Log(appiumSettings)
 }
 
-func TestDriver_DeviceScaleRatio(t *testing.T) {
-	driver, err := NewUIADriver(nil, uiaServerURL)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	scaleRatio, err := driver.Scale()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Log(scaleRatio)
-}
-
 func TestDriver_DeviceInfo(t *testing.T) {
 	driver, err := NewUIADriver(nil, uiaServerURL)
 	if err != nil {

hrp/pkg/uixt/client.go

@@ -20,6 +20,7 @@ type Driver struct {
 	urlPrefix *url.URL
 	sessionId string
 	client    *http.Client
+	scale     float64
 	// cache the last launched package name
 	lastLaunchedPackageName string
 }

hrp/pkg/uixt/ext.go

@@ -51,7 +51,6 @@ type DriverExt struct {
 	windowSize      Size
 	frame           *bytes.Buffer
 	doneMjpegStream chan bool
-	scale           float64
 	OCRService      IOCRService // used to get text from image
 	screenShots     []string    // cache screenshot paths
 
@@ -71,10 +70,6 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error)
 		return nil, errors.Wrap(err, "failed to get windows size")
 	}
 
-	if dExt.scale, err = dExt.Driver.Scale(); err != nil {
-		return nil, err
-	}
-
 	if dExt.OCRService, err = newVEDEMOCRService(); err != nil {
 		return nil, err
 	}
@@ -187,12 +182,6 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (
 	return dExt.FindImageRectInUIKit(search, options...)
 }
 
-func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
-	x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale
-	width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale
-	return
-}
-
 func (dExt *DriverExt) IsOCRExist(text string) bool {
 	_, err := dExt.FindScreenTextByOCR(text)
 	return err == nil
@@ -213,11 +202,13 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool {
 	return true
 }
 
+// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner
+// the value of (x, y) is between 0 and 1, which means the percentage of the screen
 func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) {
-	return int(x1 * float64(dExt.windowSize.Width) * dExt.scale),
-		int(y1 * float64(dExt.windowSize.Height) * dExt.scale),
-		int(x2 * float64(dExt.windowSize.Width) * dExt.scale),
-		int(y2 * float64(dExt.windowSize.Height) * dExt.scale)
+	return int(x1 * float64(dExt.windowSize.Width)),
+		int(y1 * float64(dExt.windowSize.Height)),
+		int(x2 * float64(dExt.windowSize.Width)),
+		int(y2 * float64(dExt.windowSize.Height))
 }
 
 func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool {

hrp/pkg/uixt/ios_device.go

@@ -631,6 +631,11 @@ func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver WebDriver
 	}
 	wd.mjpegClient = convertToHTTPClient(wd.mjpegHTTPConn)
 
+	// init WDA scale
+	if wd.scale, err = wd.Scale(); err != nil {
+		return nil, err
+	}
+
 	return wd, nil
 }
 
@@ -659,6 +664,11 @@ func (dev *IOSDevice) NewUSBDriver(capabilities Capabilities) (driver WebDriver,
 		return nil, errors.Wrap(code.IOSDeviceUSBDriverError, err.Error())
 	}
 
+	// init WDA scale
+	if wd.scale, err = wd.Scale(); err != nil {
+		return nil, err
+	}
+
 	return wd, nil
 }
 

hrp/pkg/uixt/ios_driver.go

@@ -164,11 +164,16 @@ func (wd *wdaDriver) Screen() (screen Screen, err error) {
 func (wd *wdaDriver) Scale() (float64, error) {
 	screen, err := wd.Screen()
 	if err != nil {
-		return 0, err
+		return 0, errors.Wrap(code.MobileUIDriverError,
+			fmt.Sprintf("get screen info failed: %v", err))
 	}
 	return screen.Scale, nil
 }
 
+func (wd *wdaDriver) toScale(x float64) float64 {
+	return x / wd.scale
+}
+
 func (wd *wdaDriver) ActiveAppInfo() (info AppInfo, err error) {
 	// [[FBRoute GET:@"/wda/activeAppInfo"] respondWithTarget:self action:@selector(handleActiveAppInfo:)]
 	// [[FBRoute GET:@"/wda/activeAppInfo"].withoutSession
@@ -375,8 +380,8 @@ func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error {
 func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) {
 	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
 	data := map[string]interface{}{
-		"x": x,
-		"y": y,
+		"x": wd.toScale(x),
+		"y": wd.toScale(y),
 	}
 	// new data options in post data for extra WDA configurations
 	newData := NewData(data, options...)
@@ -392,8 +397,8 @@ func (wd *wdaDriver) DoubleTap(x, y int) error {
 func (wd *wdaDriver) DoubleTapFloat(x, y float64) (err error) {
 	// [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)]
 	data := map[string]interface{}{
-		"x": x,
-		"y": y,
+		"x": wd.toScale(x),
+		"y": wd.toScale(y),
 	}
 	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/doubleTap")
 	return
@@ -406,8 +411,8 @@ func (wd *wdaDriver) TouchAndHold(x, y int, second ...float64) error {
 func (wd *wdaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err error) {
 	// [[FBRoute POST:@"/wda/touchAndHold"] respondWithTarget:self action:@selector(handleTouchAndHoldCoordinate:)]
 	data := map[string]interface{}{
-		"x": x,
-		"y": y,
+		"x": wd.toScale(x),
+		"y": wd.toScale(y),
 	}
 	if len(second) == 0 || second[0] <= 0 {
 		second = []float64{1.0}
@@ -424,10 +429,10 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) err
 func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) {
 	// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
 	data := map[string]interface{}{
-		"fromX": fromX,
-		"fromY": fromY,
-		"toX":   toX,
-		"toY":   toY,
+		"fromX": wd.toScale(fromX),
+		"fromY": wd.toScale(fromY),
+		"toX":   wd.toScale(toX),
+		"toY":   wd.toScale(toY),
 	}
 
 	// new data options in post data for extra WDA configurations
@@ -491,10 +496,10 @@ func (wd *wdaDriver) PressBack(options ...DataOption) (err error) {
 	}
 
 	data := map[string]interface{}{
-		"fromX": float64(windowSize.Width) * 0,
-		"fromY": float64(windowSize.Height) * 0.5,
-		"toX":   float64(windowSize.Width) * 0.6,
-		"toY":   float64(windowSize.Height) * 0.5,
+		"fromX": wd.toScale(float64(windowSize.Width) * 0),
+		"fromY": wd.toScale(float64(windowSize.Height) * 0.5),
+		"toX":   wd.toScale(float64(windowSize.Width) * 0.6),
+		"toY":   wd.toScale(float64(windowSize.Height) * 0.5),
 	}
 
 	// new data options in post data for extra WDA configurations

hrp/pkg/uixt/ios_test.go

@@ -68,6 +68,17 @@ func TestNewUSBDriver(t *testing.T) {
 	// t.Log(driver.IsWdaHealthy())
 }
 
+func TestDriver_DeviceScaleRatio(t *testing.T) {
+	setup(t)
+
+	scaleRatio, err := driver.Scale()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(scaleRatio)
+}
+
 func Test_remoteWD_DeleteSession(t *testing.T) {
 	setup(t)
 

hrp/pkg/uixt/opencv.go

@@ -71,7 +71,6 @@ func (dExt *DriverExt) Debug(dm DebugMode) {
 func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
 	newExt = new(DriverExt)
 	newExt.Driver = dExt.Driver
-	newExt.scale = dExt.scale
 	newExt.matchMode = dExt.matchMode
 	newExt.threshold = threshold
 	return
@@ -80,7 +79,6 @@ func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt)
 func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
 	newExt = new(DriverExt)
 	newExt.Driver = dExt.Driver
-	newExt.scale = dExt.scale
 	newExt.matchMode = matchMode
 	newExt.threshold = dExt.threshold
 	return