From ce93df1f23538a3195e53572c75a53dbc84e9dc6 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 5 May 2025 16:54:59 +0800 Subject: [PATCH] change: mark ui operation before action --- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 37 ++++++++++++++---------------- uixt/android_driver_uia2.go | 45 +++++++++++++++++-------------------- uixt/browser_driver.go | 27 +++++++++++----------- uixt/harmony_driver_hdc.go | 32 ++++++++++++-------------- uixt/ios_driver_wda.go | 31 +++++++++++++------------ 6 files changed, 81 insertions(+), 93 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 05588622..f318f348 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2505051635 +v5.0.0-beta-2505051654 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index a9c191a7..cf8ff702 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -310,11 +310,6 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) - // adb shell input tap x y - xStr := fmt.Sprintf("%.1f", x) - yStr := fmt.Sprintf("%.1f", y) - _, err := ad.runShellCommand("input", "tap", xStr, yStr) - // mark UI operation if actionOptions.MarkOperationEnabled { if markErr := MarkUIOperation(ad, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { @@ -322,6 +317,10 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } } + // adb shell input tap x y + xStr := fmt.Sprintf("%.1f", x) + yStr := fmt.Sprintf("%.1f", y) + _, err := ad.runShellCommand("input", "tap", xStr, yStr) if err != nil { return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) } @@ -386,6 +385,13 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO } fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(ad, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark drag operation") + } + } + duration := 200.0 if actionOptions.Duration > 0 { duration = actionOptions.Duration * 1000 @@ -401,14 +407,6 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO fmt.Sprintf("%.1f", toX), fmt.Sprintf("%.1f", toY), fmt.Sprintf("%d", int(duration)), ) - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(ad, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark drag operation") - } - } - if err != nil { return errors.Wrap(err, "adb drag failed") } @@ -426,13 +424,6 @@ func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) - // adb shell input swipe fromX fromY toX toY - _, err = ad.runShellCommand( - "input", "swipe", - fmt.Sprintf("%.1f", fromX), fmt.Sprintf("%.1f", fromY), - fmt.Sprintf("%.1f", toX), fmt.Sprintf("%.1f", toY), - ) - // mark UI operation if actionOptions.MarkOperationEnabled { if markErr := MarkUIOperation(ad, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil { @@ -440,6 +431,12 @@ func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action } } + // adb shell input swipe fromX fromY toX toY + _, err = ad.runShellCommand( + "input", "swipe", + fmt.Sprintf("%.1f", fromX), fmt.Sprintf("%.1f", fromY), + fmt.Sprintf("%.1f", toX), fmt.Sprintf("%.1f", toY), + ) if err != nil { return errors.Wrap(err, "adb swipe failed") } diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 29558063..47b435f3 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -301,6 +301,13 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(ud, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark tap operation") + } + } + duration := 100.0 if actionOptions.PressDuration > 0 { duration = actionOptions.PressDuration * 1000 // convert to ms @@ -324,14 +331,6 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error urlStr := fmt.Sprintf("/session/%s/actions/tap", ud.Session.ID) _, err := ud.Session.POST(data, urlStr) - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(ud, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark tap operation") - } - } - return err } @@ -371,6 +370,13 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(ud, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark drag operation") + } + } + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -382,14 +388,6 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action // register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag")) urlStr := fmt.Sprintf("/session/%s/touch/drag", ud.Session.ID) _, err = ud.Session.POST(data, urlStr) - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(ud, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark drag operation") - } - } - return err } @@ -410,6 +408,13 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio } fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(ud, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark swipe operation") + } + } + duration := 200.0 if actionOptions.PressDuration > 0 { duration = actionOptions.PressDuration * 1000 // ms @@ -433,14 +438,6 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio urlStr := fmt.Sprintf("/session/%s/actions/swipe", ud.Session.ID) _, err = ud.Session.POST(data, urlStr) - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(ud, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark swipe operation") - } - } - return err } diff --git a/uixt/browser_driver.go b/uixt/browser_driver.go index de8ca867..869ef9e4 100644 --- a/uixt/browser_driver.go +++ b/uixt/browser_driver.go @@ -113,12 +113,6 @@ func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option. actionOptions := option.NewActionOptions(options...) - if actionOptions.Duration > 0 { - data["duration"] = actionOptions.Duration - } - - _, err = wd.HttpPOST(data, wd.sessionId, "ui/drag") - // mark UI operation if actionOptions.MarkOperationEnabled { if markErr := MarkUIOperation(wd, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { @@ -126,6 +120,11 @@ func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option. } } + if actionOptions.Duration > 0 { + data["duration"] = actionOptions.Duration + } + + _, err = wd.HttpPOST(data, wd.sessionId, "ui/drag") return } @@ -527,6 +526,14 @@ func (wd *BrowserDriver) Tap(x, y float64, options ...option.ActionOption) error func (wd *BrowserDriver) TapFloat(x, y float64, options ...option.ActionOption) error { actionOptions := option.NewActionOptions(options...) + + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(wd, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark tap operation") + } + } + duration := 0.1 if actionOptions.Duration > 0 { duration = actionOptions.Duration @@ -537,14 +544,6 @@ func (wd *BrowserDriver) TapFloat(x, y float64, options ...option.ActionOption) "duration": duration, } _, err := wd.HttpPOST(data, wd.sessionId, "ui/tap") - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(wd, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark tap operation") - } - } - return err } diff --git a/uixt/harmony_driver_hdc.go b/uixt/harmony_driver_hdc.go index 56222d27..4c609e7e 100644 --- a/uixt/harmony_driver_hdc.go +++ b/uixt/harmony_driver_hdc.go @@ -157,13 +157,6 @@ func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) - if actionOptions.Identifier != "" { - startTime := int(time.Now().UnixMilli()) - hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100}) - } - err := hd.uiDriver.InjectGesture( - ghdc.NewGesture().Start(ghdc.Point{X: int(x), Y: int(y)}).Pause(100)) - // mark UI operation if actionOptions.MarkOperationEnabled { if markErr := MarkUIOperation(hd, ACTION_TapAbsXY, []float64{x, y}); markErr != nil { @@ -171,7 +164,12 @@ func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } } - return err + if actionOptions.Identifier != "" { + startTime := int(time.Now().UnixMilli()) + hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100}) + } + return hd.uiDriver.InjectGesture( + ghdc.NewGesture().Start(ghdc.Point{X: int(x), Y: int(y)}).Pause(100)) } func (hd *HDCDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { @@ -198,6 +196,13 @@ func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action } fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(hd, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark swipe operation") + } + } + duration := 200 if actionOptions.PressDuration > 0 { duration = int(actionOptions.PressDuration * 1000) @@ -206,18 +211,9 @@ func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action startTime := int(time.Now().UnixMilli()) hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100}) } - err = hd.uiDriver.InjectGesture( + return hd.uiDriver.InjectGesture( ghdc.NewGesture().Start(ghdc.Point{X: int(fromX), Y: int(fromY)}). MoveTo(ghdc.Point{X: int(toX), Y: int(toY)}, duration)) - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(hd, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark swipe operation") - } - } - - return err } func (hd *HDCDriver) SetIme(ime string) error { diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 1d75f642..8cd0ae2e 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -595,14 +595,6 @@ func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) - data := map[string]interface{}{ - "x": wd.toScale(x), - "y": wd.toScale(y), - } - option.MergeOptions(data, opts...) - - urlStr := fmt.Sprintf("/session/%s/wda/tap/0", wd.Session.ID) - _, err := wd.Session.POST(data, urlStr) // mark UI operation if actionOptions.MarkOperationEnabled { @@ -611,6 +603,14 @@ func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } } + data := map[string]interface{}{ + "x": wd.toScale(x), + "y": wd.toScale(y), + } + option.MergeOptions(data, opts...) + + urlStr := fmt.Sprintf("/session/%s/wda/tap/0", wd.Session.ID) + _, err := wd.Session.POST(data, urlStr) return err } @@ -663,6 +663,13 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + // mark UI operation + if actionOptions.MarkOperationEnabled { + if markErr := MarkUIOperation(wd, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { + log.Warn().Err(markErr).Msg("Failed to mark drag operation") + } + } + data := map[string]interface{}{ "fromX": math.Round(fromX*10) / 10, "fromY": math.Round(fromY*10) / 10, @@ -674,14 +681,6 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO urlStr := fmt.Sprintf("/session/%s/wda/dragfromtoforduration", wd.Session.ID) _, err = wd.Session.POST(data, urlStr) // _, err = wd.Session.POST(data, "/session", wd.Session.ID, "/wda/drag") - - // mark UI operation - if actionOptions.MarkOperationEnabled { - if markErr := MarkUIOperation(wd, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil { - log.Warn().Err(markErr).Msg("Failed to mark drag operation") - } - } - return err }