From 2f310fea8a311d5c1bb7e829a1c0bfefbf61c1ae Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 20 Sep 2023 18:04:04 +0800 Subject: [PATCH] fix: tap/swipe points with random offset --- hrp/pkg/uixt/action.go | 92 +++++++---------------------- hrp/pkg/uixt/android_adb_driver.go | 18 ++++-- hrp/pkg/uixt/android_uia2_driver.go | 56 ++++++++++++++---- hrp/pkg/uixt/ios_driver.go | 91 ++++++++++++++++++++-------- hrp/pkg/uixt/video_crawler.go | 2 +- 5 files changed, 146 insertions(+), 113 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index d17dd2e9..242b59ea 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -249,7 +249,7 @@ func (o *ActionOptions) screenshotActions() []string { return actions } -func (o *ActionOptions) getRandomOffset() int { +func (o *ActionOptions) getRandomOffset() float64 { if len(o.OffsetRandomRange) != 2 { // invalid offset random range, should be [min, max] return 0 @@ -257,91 +257,33 @@ func (o *ActionOptions) getRandomOffset() int { minOffset := o.OffsetRandomRange[0] maxOffset := o.OffsetRandomRange[1] - return builtin.GetRandomNumber(minOffset, maxOffset) + return float64(builtin.GetRandomNumber(minOffset, maxOffset)) + rand.Float64() } -func NewActionOptions(options ...ActionOption) *ActionOptions { - actionOptions := &ActionOptions{} - for _, option := range options { - option(actionOptions) - } - return actionOptions -} - -func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} { - actionOptions := NewActionOptions(options...) - - if actionOptions.Identifier != "" { +func (o *ActionOptions) updateData(data map[string]interface{}) { + if o.Identifier != "" { data["log"] = map[string]interface{}{ "enable": true, - "data": actionOptions.Identifier, + "data": o.Identifier, } } - // handle point offset - if len(actionOptions.Offset) == 2 { - if x, ok := data["x"]; ok { - xf, _ := builtin.Interface2Float64(x) - data["x"] = xf + float64(actionOptions.Offset[0]+actionOptions.getRandomOffset()) - } - if y, ok := data["y"]; ok { - yf, _ := builtin.Interface2Float64(y) - data["y"] = yf + float64(actionOptions.Offset[1]+actionOptions.getRandomOffset()) - } - } else if len(actionOptions.Offset) == 4 { - // Android uia2: [startX, startY, endX, endY] - if startX, ok := data["startX"]; ok { - vf, _ := builtin.Interface2Float64(startX) - data["startX"] = vf + float64(actionOptions.Offset[0]+actionOptions.getRandomOffset()) - } - if startY, ok := data["startY"]; ok { - vf, _ := builtin.Interface2Float64(startY) - data["startY"] = vf + float64(actionOptions.Offset[1]+actionOptions.getRandomOffset()) - } - if endX, ok := data["endX"]; ok { - vf, _ := builtin.Interface2Float64(endX) - data["endX"] = vf + float64(actionOptions.Offset[2]+actionOptions.getRandomOffset()) - } - if endY, ok := data["endY"]; ok { - vf, _ := builtin.Interface2Float64(endY) - data["endY"] = vf + float64(actionOptions.Offset[3]+actionOptions.getRandomOffset()) - } - - // iOS WDA: [fromX, fromY, toX, toY] - if fromX, ok := data["fromX"]; ok { - vf, _ := builtin.Interface2Float64(fromX) - data["fromX"] = vf + float64(actionOptions.Offset[0]+actionOptions.getRandomOffset()) - } - if fromY, ok := data["fromY"]; ok { - vf, _ := builtin.Interface2Float64(fromY) - data["fromY"] = vf + float64(actionOptions.Offset[1]+actionOptions.getRandomOffset()) - } - if toX, ok := data["toX"]; ok { - vf, _ := builtin.Interface2Float64(toX) - data["toX"] = vf + float64(actionOptions.Offset[2]+actionOptions.getRandomOffset()) - } - if toY, ok := data["toY"]; ok { - vf, _ := builtin.Interface2Float64(toY) - data["toY"] = vf + float64(actionOptions.Offset[3]+actionOptions.getRandomOffset()) - } - } - - if actionOptions.Steps > 0 { - data["steps"] = actionOptions.Steps + if o.Steps > 0 { + data["steps"] = o.Steps } if _, ok := data["steps"]; !ok { data["steps"] = 12 // default steps } - if actionOptions.PressDuration > 0 { - data["duration"] = actionOptions.PressDuration + if o.PressDuration > 0 { + data["duration"] = o.PressDuration } if _, ok := data["duration"]; !ok { data["duration"] = 0 // default duration } - if actionOptions.Frequency > 0 { - data["frequency"] = actionOptions.Frequency + if o.Frequency > 0 { + data["frequency"] = o.Frequency } if _, ok := data["frequency"]; !ok { data["frequency"] = 60 // default frequency @@ -352,13 +294,19 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) } // custom options - if actionOptions.Custom != nil { - for k, v := range actionOptions.Custom { + if o.Custom != nil { + for k, v := range o.Custom { data[k] = v } } +} - return data +func NewActionOptions(options ...ActionOption) *ActionOptions { + actionOptions := &ActionOptions{} + for _, option := range options { + option(actionOptions) + } + return actionOptions } type ActionOption func(o *ActionOptions) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index c360a9f4..b2d84652 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -225,9 +225,11 @@ func (ad *adbDriver) TapFloat(x, y float64, options ...ActionOption) (err error) actionOptions := NewActionOptions(options...) if len(actionOptions.Offset) == 2 { - x += float64(actionOptions.Offset[0] + actionOptions.getRandomOffset()) - y += float64(actionOptions.Offset[1] + actionOptions.getRandomOffset()) + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -275,11 +277,15 @@ func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio actionOptions := NewActionOptions(options...) if len(actionOptions.Offset) == 4 { - fromX += float64(actionOptions.Offset[0] + actionOptions.getRandomOffset()) - fromY += float64(actionOptions.Offset[1] + actionOptions.getRandomOffset()) - toX += float64(actionOptions.Offset[2] + actionOptions.getRandomOffset()) - toY += float64(actionOptions.Offset[3] + actionOptions.getRandomOffset()) + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() // adb shell input swipe fromX fromY toX toY _, err := ad.adbClient.RunShellCommand( diff --git a/hrp/pkg/uixt/android_uia2_driver.go b/hrp/pkg/uixt/android_uia2_driver.go index 3a09e36d..674b70c3 100644 --- a/hrp/pkg/uixt/android_uia2_driver.go +++ b/hrp/pkg/uixt/android_uia2_driver.go @@ -259,14 +259,23 @@ func (ud *uiaDriver) Tap(x, y int, options ...ActionOption) error { func (ud *uiaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) + actionOptions := NewActionOptions(options...) + + if len(actionOptions.Offset) == 2 { + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) + } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() + data := map[string]interface{}{ "x": x, "y": y, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "appium/tap") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap") return } @@ -299,6 +308,18 @@ func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e } func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -306,11 +327,11 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...Action "endY": toY, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) // register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag")) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "touch/drag") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/drag") return } @@ -325,6 +346,18 @@ func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { // register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform")) + actionOptions := NewActionOptions(options...) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -332,10 +365,10 @@ func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio "endY": toY, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) - _, err := ud.httpPOST(newData, "/session", ud.sessionId, "touch/perform") + _, err := ud.httpPOST(data, "/session", ud.sessionId, "touch/perform") return err } @@ -385,13 +418,14 @@ func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) { // register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys")) // https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85 + actionOptions := NewActionOptions(options...) data := map[string]interface{}{ "text": text, } // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + actionOptions.updateData(data) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "keys") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "keys") return } diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index aa2242e6..52390256 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -461,14 +461,25 @@ func (wd *wdaDriver) Tap(x, y int, options ...ActionOption) error { func (wd *wdaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] - data := map[string]interface{}{ - "x": wd.toScale(x), - "y": wd.toScale(y), - } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + actionOptions := NewActionOptions(options...) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/tap/0") + x = wd.toScale(x) + y = wd.toScale(y) + if len(actionOptions.Offset) == 2 { + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) + } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() + + data := map[string]interface{}{ + "x": x, + "y": y, + } + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) + + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0") return } @@ -510,17 +521,34 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] + actionOptions := NewActionOptions(options...) + + fromX = wd.toScale(fromX) + fromY = wd.toScale(fromY) + toX = wd.toScale(toX) + toY = wd.toScale(toY) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ - "fromX": wd.toScale(fromX), - "fromY": wd.toScale(fromY), - "toX": wd.toScale(toX), - "toY": wd.toScale(toY), + "fromX": fromX, + "fromY": fromY, + "toX": toX, + "toY": toY, } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } @@ -557,12 +585,13 @@ func (wd *wdaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe func (wd *wdaDriver) SendKeys(text string, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] + actionOptions := NewActionOptions(options...) data := map[string]interface{}{"value": strings.Split(text, "")} // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/keys") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys") return } @@ -572,22 +601,38 @@ func (wd *wdaDriver) Input(text string, options ...ActionOption) (err error) { // PressBack simulates a short press on the BACK button. func (wd *wdaDriver) PressBack(options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) + windowSize, err := wd.WindowSize() if err != nil { return } + fromX := wd.toScale(float64(windowSize.Width) * 0) + fromY := wd.toScale(float64(windowSize.Height) * 0.5) + toX := wd.toScale(float64(windowSize.Width) * 0.6) + toY := wd.toScale(float64(windowSize.Height) * 0.5) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() data := map[string]interface{}{ - "fromX": wd.toScale(float64(windowSize.Width) * 0), - "fromY": wd.toScale(float64(windowSize.Height) * 0.5), - "toX": wd.toScale(float64(windowSize.Width) * 0.6), - "toY": wd.toScale(float64(windowSize.Height) * 0.5), + "fromX": fromX, + "fromY": fromY, + "toX": toX, + "toY": toY, } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 0cec910c..4d8620ad 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -161,7 +161,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // swipe to next feed video log.Info().Msg("swipe to next feed video") swipeStartTime := time.Now() - if err = dExt.SwipeRelative(0.9, 0.8, 0.9, 0.1); err != nil { + if err = dExt.SwipeRelative(0.9, 0.8, 0.9, 0.1, WithOffsetRandomRange(-10, 10)); err != nil { log.Error().Err(err).Msg("feed swipe up failed") return err }