From 3e7e9b0ef912336b6b18b9f2528421cb63c90726 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 14:36:09 +0800 Subject: [PATCH 1/9] change: tap/swipe with offset --- internal/version/VERSION | 2 +- uixt/driver_ext_swipe.go | 2 +- uixt/driver_ext_test.go | 13 +++++++++++++ uixt/option/action.go | 17 +++++++++-------- uixt/option/screen.go | 5 +++-- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index d9a653d2..fe8b7add 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503141055 +v5.0.0-beta-2503171436 diff --git a/uixt/driver_ext_swipe.go b/uixt/driver_ext_swipe.go index 9cb43574..78d1a532 100644 --- a/uixt/driver_ext_swipe.go +++ b/uixt/driver_ext_swipe.go @@ -156,7 +156,7 @@ func (dExt *XTDriver) SwipeToTapApp(appName string, opts ...option.ActionOption) actionOptions := option.NewActionOptions(opts...) // tap app icon above the text - if len(actionOptions.Offset) == 0 { + if len(actionOptions.TapOffset) == 0 { opts = append(opts, option.WithTapOffset(0, -25)) } // set default swipe interval to 1 second diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index 9afdcd15..a50465ba 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -168,3 +168,16 @@ func TestDriverExt_ClosePopupsHandler(t *testing.T) { err := driver.ClosePopupsHandler() assert.Nil(t, err) } + +func TestDriverExt_Action_Risk(t *testing.T) { + driver := setupADBDriverExt(t) + + // tap point with offset + err := driver.TapXY(0.5, 0.5, option.WithTapOffset(-10, 10)) + assert.Nil(t, err) + + // swipe direction with offset + err = driver.Swipe(0.5, 0.5, 0.5, 0.9, + option.WithSwipeOffset(-50, 50, -50, 50)) + assert.Nil(t, err) +} diff --git a/uixt/option/action.go b/uixt/option/action.go index 594a6cd4..f2c5b0a0 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -95,13 +95,14 @@ func (o *ActionOptions) Options() []ActionOption { options = append(options, WithScope( o.Scope[0], o.Scope[1], o.Scope[2], o.Scope[3])) } - if len(o.Offset) == 2 { + if len(o.TapOffset) == 2 { // for tap [x,y] offset - options = append(options, WithTapOffset(o.Offset[0], o.Offset[1])) - } else if len(o.Offset) == 4 { + options = append(options, WithTapOffset(o.TapOffset[0], o.TapOffset[1])) + } + if len(o.SwipeOffset) == 4 { // for swipe [fromX, fromY, toX, toY] offset options = append(options, WithSwipeOffset( - o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3])) + o.SwipeOffset[0], o.SwipeOffset[1], o.SwipeOffset[2], o.SwipeOffset[3])) } if len(o.OffsetRandomRange) == 2 { options = append(options, WithOffsetRandomRange( @@ -132,9 +133,9 @@ func (o *ActionOptions) Options() []ActionOption { } func (o *ActionOptions) ApplyOffset(absX, absY float64) (float64, float64) { - if len(o.Offset) == 2 { - absX += float64(o.Offset[0]) - absY += float64(o.Offset[1]) + if len(o.TapOffset) == 2 { + absX += float64(o.TapOffset[0]) + absY += float64(o.TapOffset[1]) } absX += o.GenerateRandomOffset() absY += o.GenerateRandomOffset() @@ -276,7 +277,7 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { // swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY] func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption { return func(o *ActionOptions) { - o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY} + o.SwipeOffset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY} } } diff --git a/uixt/option/screen.go b/uixt/option/screen.go index 160dd4c9..01659f9c 100644 --- a/uixt/option/screen.go +++ b/uixt/option/screen.go @@ -216,7 +216,8 @@ type ScreenFilterOptions struct { AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + TapOffset []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"` // tap with point offset + SwipeOffset []int `json:"swipe_offset,omitempty" yaml:"swipe_offset,omitempty"` // swipe with direction offset OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element MatchOne bool `json:"match_one,omitempty" yaml:"match_one,omitempty"` @@ -242,7 +243,7 @@ func WithAbsScope(x1, y1, x2, y2 int) ActionOption { // tap [x, y] with offset [offsetX, offsetY] func WithTapOffset(offsetX, offsetY int) ActionOption { return func(o *ActionOptions) { - o.Offset = []int{offsetX, offsetY} + o.TapOffset = []int{offsetX, offsetY} } } From 7f69052be6289f2ebe23a8116bbc1f7b6f7514bb Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 15:09:04 +0800 Subject: [PATCH 2/9] change: FindScreenText return text rect --- internal/version/VERSION | 2 +- uixt/ai/cv.go | 7 ++----- uixt/driver_ext_screenshot.go | 9 ++++----- uixt/driver_ext_tap.go | 3 ++- uixt/driver_ext_test.go | 9 ++++++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index fe8b7add..cbeff748 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171436 +v5.0.0-beta-2503171509 diff --git a/uixt/ai/cv.go b/uixt/ai/cv.go index fa314855..0a6ff51d 100644 --- a/uixt/ai/cv.go +++ b/uixt/ai/cv.go @@ -86,13 +86,10 @@ func (t OCRText) Size() types.Size { } func (t OCRText) Center() PointF { - return getRectangleCenterPoint(t.Rect) -} - -func getRectangleCenterPoint(rect image.Rectangle) (point PointF) { + rect := t.Rect x, y := float64(rect.Min.X), float64(rect.Min.Y) width, height := float64(rect.Dx()), float64(rect.Dy()) - point = PointF{ + point := PointF{ X: x + width*0.5, Y: y + height*0.5, } diff --git a/uixt/driver_ext_screenshot.go b/uixt/driver_ext_screenshot.go index 2ace7882..854baeb6 100644 --- a/uixt/driver_ext_screenshot.go +++ b/uixt/driver_ext_screenshot.go @@ -145,7 +145,7 @@ func (dExt *XTDriver) GetScreenTexts(opts ...option.ActionOption) (ocrTexts ai.O return screenResult.Texts, nil } -func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (point ai.PointF, err error) { +func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (textRect ai.OCRText, err error) { options := option.NewActionOptions(opts...) if options.ScreenShotFileName == "" { opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("find_screen_text_%s", text))) @@ -155,16 +155,15 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) ( return } - result, err := ocrTexts.FindText(text, opts...) + textRect, err = ocrTexts.FindText(text, opts...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return } - point = result.Center() log.Info().Str("text", text). - Interface("point", point).Msgf("FindScreenText success") - return + Interface("textRect", textRect).Msgf("FindScreenText success") + return textRect, nil } func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) { diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index b4e8f151..8881fc38 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -12,13 +12,14 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { opts = append(opts, option.WithScreenShotFileName(fmt.Sprintf("tap_by_ocr_%s", text))) } - point, err := dExt.FindScreenText(text, opts...) + textRect, err := dExt.FindScreenText(text, opts...) if err != nil { if actionOptions.IgnoreNotFoundError { return nil } return err } + point := textRect.Center() return dExt.TapAbsXY(point.X, point.Y, opts...) } diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index a50465ba..dec94fb2 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -47,8 +47,8 @@ func TestDriverExt(t *testing.T) { driverExt.TapByOCR("推荐") texts, _ := driverExt.GetScreenTexts() t.Log(texts) - point, _ := driverExt.FindScreenText("hello") - t.Log(point) + textRect, _ := driverExt.FindScreenText("hello") + t.Log(textRect) err := driverExt.TapByCV( option.WithScreenShotUITypes("deepseek_send"), @@ -98,13 +98,14 @@ func TestDriverExt_FindScreenText(t *testing.T) { func TestDriverExt_Seek(t *testing.T) { driver := setupDriverExt(t) - point, err := driver.FindScreenText("首页") + textRect, err := driver.FindScreenText("首页") assert.Nil(t, err) size, err := driver.WindowSize() assert.Nil(t, err) width := size.Width + point := textRect.Center() y := point.Y - 40 for i := 0; i < 5; i++ { err := driver.Swipe(0.5, 0.8, 0.5, 0.2) @@ -180,4 +181,6 @@ func TestDriverExt_Action_Risk(t *testing.T) { err = driver.Swipe(0.5, 0.5, 0.5, 0.9, option.WithSwipeOffset(-50, 50, -50, 50)) assert.Nil(t, err) + + err = driver.TapByOCR("首页", option.WithTapOffset(-10, 10)) } From b34a2218fe3f4caab3c567330e8e52ddd3e889eb Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 15:36:35 +0800 Subject: [PATCH 3/9] feat: tap random point in ocr text rect --- internal/version/VERSION | 2 +- uixt/ai/cv.go | 12 ++++++++++++ uixt/driver_ext_tap.go | 9 ++++++++- uixt/driver_ext_test.go | 3 ++- uixt/option/action.go | 4 ++++ uixt/option/screen.go | 9 +++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index cbeff748..8da30f12 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171509 +v5.0.0-beta-2503171536 diff --git a/uixt/ai/cv.go b/uixt/ai/cv.go index 0a6ff51d..81b6c196 100644 --- a/uixt/ai/cv.go +++ b/uixt/ai/cv.go @@ -5,6 +5,7 @@ import ( "fmt" "image" "math" + "math/rand/v2" "regexp" "github.com/pkg/errors" @@ -96,6 +97,17 @@ func (t OCRText) Center() PointF { return point } +func (t OCRText) RandomPoint() PointF { + rect := t.Rect + x, y := float64(rect.Min.X), float64(rect.Min.Y) + width, height := float64(rect.Dx()), float64(rect.Dy()) + point := PointF{ + X: x + width*rand.Float64(), + Y: y + height*rand.Float64(), + } + return point +} + type OCRTexts []OCRText func (t OCRTexts) texts() (texts []string) { diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index 8881fc38..1388cd40 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -3,6 +3,7 @@ package uixt import ( "fmt" + "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" ) @@ -19,7 +20,13 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { } return err } - point := textRect.Center() + + var point ai.PointF + if actionOptions.TapRandom { + point = textRect.RandomPoint() + } else { + point = textRect.Center() + } return dExt.TapAbsXY(point.X, point.Y, opts...) } diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index dec94fb2..1ef1c3b4 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -182,5 +182,6 @@ func TestDriverExt_Action_Risk(t *testing.T) { option.WithSwipeOffset(-50, 50, -50, 50)) assert.Nil(t, err) - err = driver.TapByOCR("首页", option.WithTapOffset(-10, 10)) + // tap random point in ocr text rect + err = driver.TapByOCR("首页", option.WithTapRandom(true)) } diff --git a/uixt/option/action.go b/uixt/option/action.go index f2c5b0a0..26d84a29 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -99,6 +99,10 @@ func (o *ActionOptions) Options() []ActionOption { // for tap [x,y] offset options = append(options, WithTapOffset(o.TapOffset[0], o.TapOffset[1])) } + if o.TapRandom { + // tap random point in OCR/CV rectangle + options = append(options, WithTapRandom(true)) + } if len(o.SwipeOffset) == 4 { // for swipe [fromX, fromY, toX, toY] offset options = append(options, WithSwipeOffset( diff --git a/uixt/option/screen.go b/uixt/option/screen.go index 01659f9c..f2243a17 100644 --- a/uixt/option/screen.go +++ b/uixt/option/screen.go @@ -217,6 +217,7 @@ type ScreenFilterOptions struct { Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text TapOffset []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"` // tap with point offset + TapRandom bool `json:"tap_random,omitempty" yaml:"tap_random,omitempty"` // tap random point in text/image rectangle SwipeOffset []int `json:"swipe_offset,omitempty" yaml:"swipe_offset,omitempty"` // swipe with direction offset OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element @@ -247,6 +248,14 @@ func WithTapOffset(offsetX, offsetY int) ActionOption { } } +// WithTapRandom is used with TapByOCR and TapByCV +// when set true, tap random point in text/image rectangle +func WithTapRandom(tapRandom bool) ActionOption { + return func(o *ActionOptions) { + o.TapRandom = tapRandom + } +} + func WithRegex(regex bool) ActionOption { return func(o *ActionOptions) { o.Regex = regex From 9fb53590ca6fef4de608e9df8cda13ef5f84531e Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 17:44:04 +0800 Subject: [PATCH 4/9] refactor: rename ApplyTapOffset --- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 6 +++--- uixt/android_driver_uia2.go | 6 +++--- uixt/driver_ext_tap.go | 2 +- uixt/driver_ext_test.go | 8 ++++++-- uixt/harmony_driver_hdc.go | 2 +- uixt/ios_driver_wda.go | 6 +++--- uixt/option/action.go | 12 ++++++------ uixt/option/screen.go | 10 +++++----- 9 files changed, 29 insertions(+), 25 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 8da30f12..341422c6 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171536 +v5.0.0-beta-2503171744 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index e812852d..dc03f183 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -300,7 +300,7 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error { func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -320,7 +320,7 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error return err } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -341,7 +341,7 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := 1000.0 if actionOptions.Duration > 0 { duration = actionOptions.Duration * 1000 diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 938afc13..61d295f1 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -259,7 +259,7 @@ func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error return err } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) data := map[string]interface{}{ "actions": []interface{}{ @@ -295,7 +295,7 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error { func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := 100.0 if actionOptions.PressDuration > 0 { @@ -325,7 +325,7 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) duration := actionOptions.Duration if duration == 0 { duration = 1.0 diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index 1388cd40..7283fd18 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -22,7 +22,7 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { } var point ai.PointF - if actionOptions.TapRandom { + if actionOptions.TapRandomRect { point = textRect.RandomPoint() } else { point = textRect.Center() diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index 1ef1c3b4..e90f3de8 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -173,15 +173,19 @@ func TestDriverExt_ClosePopupsHandler(t *testing.T) { func TestDriverExt_Action_Risk(t *testing.T) { driver := setupADBDriverExt(t) - // tap point with offset + // tap point with constant offset err := driver.TapXY(0.5, 0.5, option.WithTapOffset(-10, 10)) assert.Nil(t, err) + // tap point with random offset + err = driver.TapXY(0.5, 0.5, option.WithOffsetRandomRange(-10, 10)) + assert.Nil(t, err) + // swipe direction with offset err = driver.Swipe(0.5, 0.5, 0.5, 0.9, option.WithSwipeOffset(-50, 50, -50, 50)) assert.Nil(t, err) // tap random point in ocr text rect - err = driver.TapByOCR("首页", option.WithTapRandom(true)) + err = driver.TapByOCR("首页", option.WithTapRandomRect(true)) } diff --git a/uixt/harmony_driver_hdc.go b/uixt/harmony_driver_hdc.go index fb248791..28dbf541 100644 --- a/uixt/harmony_driver_hdc.go +++ b/uixt/harmony_driver_hdc.go @@ -151,7 +151,7 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error { func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) if actionOptions.Identifier != "" { startTime := int(time.Now().UnixMilli()) diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 8a97a19b..5c663944 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -587,7 +587,7 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error { func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) data := map[string]interface{}{ "x": wd.toScale(x), "y": wd.toScale(y), @@ -608,7 +608,7 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error } actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) x = wd.toScale(x) y = wd.toScale(y) data := map[string]interface{}{ @@ -623,7 +623,7 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error // FIXME: hold not work func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { actionOptions := option.NewActionOptions(opts...) - x, y = actionOptions.ApplyOffset(x, y) + x, y = actionOptions.ApplyTapOffset(x, y) if actionOptions.Duration == 0 { opts = append(opts, option.WithPressDuration(1)) } diff --git a/uixt/option/action.go b/uixt/option/action.go index 26d84a29..c4b6c4c7 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -99,9 +99,9 @@ func (o *ActionOptions) Options() []ActionOption { // for tap [x,y] offset options = append(options, WithTapOffset(o.TapOffset[0], o.TapOffset[1])) } - if o.TapRandom { + if o.TapRandomRect { // tap random point in OCR/CV rectangle - options = append(options, WithTapRandom(true)) + options = append(options, WithTapRandomRect(true)) } if len(o.SwipeOffset) == 4 { // for swipe [fromX, fromY, toX, toY] offset @@ -136,17 +136,17 @@ func (o *ActionOptions) Options() []ActionOption { return options } -func (o *ActionOptions) ApplyOffset(absX, absY float64) (float64, float64) { +func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) { if len(o.TapOffset) == 2 { absX += float64(o.TapOffset[0]) absY += float64(o.TapOffset[1]) } - absX += o.GenerateRandomOffset() - absY += o.GenerateRandomOffset() + absX += o.generateRandomOffset() + absY += o.generateRandomOffset() return absX, absY } -func (o *ActionOptions) GenerateRandomOffset() float64 { +func (o *ActionOptions) generateRandomOffset() float64 { if len(o.OffsetRandomRange) != 2 { // invalid offset random range, should be [min, max] return 0 diff --git a/uixt/option/screen.go b/uixt/option/screen.go index f2243a17..feb3772f 100644 --- a/uixt/option/screen.go +++ b/uixt/option/screen.go @@ -216,8 +216,8 @@ type ScreenFilterOptions struct { AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text - TapOffset []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"` // tap with point offset - TapRandom bool `json:"tap_random,omitempty" yaml:"tap_random,omitempty"` // tap random point in text/image rectangle + TapOffset []int `json:"tap_offset,omitempty" yaml:"tap_offset,omitempty"` // tap with absolute point offset + TapRandomRect bool `json:"tap_random_rect,omitempty" yaml:"tap_random_rect,omitempty"` // tap random point in text/image rectangle SwipeOffset []int `json:"swipe_offset,omitempty" yaml:"swipe_offset,omitempty"` // swipe with direction offset OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element @@ -248,11 +248,11 @@ func WithTapOffset(offsetX, offsetY int) ActionOption { } } -// WithTapRandom is used with TapByOCR and TapByCV +// WithTapRandomRect is used with TapByOCR and TapByCV // when set true, tap random point in text/image rectangle -func WithTapRandom(tapRandom bool) ActionOption { +func WithTapRandomRect(tapRandom bool) ActionOption { return func(o *ActionOptions) { - o.TapRandom = tapRandom + o.TapRandomRect = tapRandom } } From 182de16751a762b28cd72bfddcaaf6afda0d8633 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 17:57:10 +0800 Subject: [PATCH 5/9] feat: ApplySwipeOffset --- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 3 +++ uixt/android_driver_uia2.go | 4 ++++ uixt/driver_ext_test.go | 7 ++++++- uixt/harmony_driver_hdc.go | 1 + uixt/ios_driver_wda.go | 3 +++ uixt/option/action.go | 15 +++++++++++++++ 7 files changed, 33 insertions(+), 2 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 341422c6..a2e325c3 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171744 +v5.0.0-beta-2503171757 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index dc03f183..3bf01d46 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -365,6 +365,7 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200.0 if actionOptions.Duration > 0 { @@ -393,6 +394,8 @@ func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action if err != nil { return err } + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) // adb shell input swipe fromX fromY toX toY _, err = ad.runShellCommand( diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 61d295f1..edb37f66 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -353,6 +353,9 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action if err != nil { return err } + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -380,6 +383,7 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200.0 if actionOptions.PressDuration > 0 { diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index e90f3de8..7b4f5a37 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -181,11 +181,16 @@ func TestDriverExt_Action_Risk(t *testing.T) { err = driver.TapXY(0.5, 0.5, option.WithOffsetRandomRange(-10, 10)) assert.Nil(t, err) - // swipe direction with offset + // swipe direction with constant offset err = driver.Swipe(0.5, 0.5, 0.5, 0.9, option.WithSwipeOffset(-50, 50, -50, 50)) assert.Nil(t, err) + // swipe direction with random offset + err = driver.Swipe(0.5, 0.5, 0.5, 0.9, + option.WithOffsetRandomRange(-50, 50)) + assert.Nil(t, err) + // tap random point in ocr text rect err = driver.TapByOCR("首页", option.WithTapRandomRect(true)) } diff --git a/uixt/harmony_driver_hdc.go b/uixt/harmony_driver_hdc.go index 28dbf541..6ffa155d 100644 --- a/uixt/harmony_driver_hdc.go +++ b/uixt/harmony_driver_hdc.go @@ -181,6 +181,7 @@ func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action if err != nil { return err } + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) duration := 200 if actionOptions.PressDuration > 0 { diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 5c663944..c08aa2ce 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -641,6 +641,9 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO fromY = wd.toScale(fromY) toX = wd.toScale(toX) toY = wd.toScale(toY) + actionOptions := option.NewActionOptions(opts...) + fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) + data := map[string]interface{}{ "fromX": math.Round(fromX*10) / 10, "fromY": math.Round(fromY*10) / 10, diff --git a/uixt/option/action.go b/uixt/option/action.go index c4b6c4c7..72f04c18 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -146,6 +146,21 @@ func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) { return absX, absY } +func (o *ActionOptions) ApplySwipeOffset(absFromX, absFromY, absToX, absToY float64) ( + float64, float64, float64, float64) { + if len(o.SwipeOffset) == 4 { + absFromX += float64(o.SwipeOffset[0]) + absFromY += float64(o.SwipeOffset[1]) + absToX += float64(o.SwipeOffset[2]) + absToY += float64(o.SwipeOffset[3]) + } + absFromX += o.generateRandomOffset() + absFromY += o.generateRandomOffset() + absToX += o.generateRandomOffset() + absToY += o.generateRandomOffset() + return absFromX, absFromY, absToX, absToY +} + func (o *ActionOptions) generateRandomOffset() float64 { if len(o.OffsetRandomRange) != 2 { // invalid offset random range, should be [min, max] From 5e2f33b362942acb38106005e4957b1480d8004e Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 18:06:32 +0800 Subject: [PATCH 6/9] feat: TapByCV WithTapRandomRect --- internal/version/VERSION | 2 +- uixt/ai/cv.go | 9 +++++++++ uixt/driver_ext_screenshot.go | 7 +++---- uixt/driver_ext_tap.go | 13 ++++++++++--- uixt/driver_ext_test.go | 6 ++++++ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index a2e325c3..5200bf63 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171757 +v5.0.0-beta-2503171806 diff --git a/uixt/ai/cv.go b/uixt/ai/cv.go index 81b6c196..2fe70839 100644 --- a/uixt/ai/cv.go +++ b/uixt/ai/cv.go @@ -249,6 +249,15 @@ func (box Box) Center() PointF { } } +func (box Box) RandomPoint() PointF { + width, height := float64(box.Width), float64(box.Height) + point := PointF{ + X: box.Point.X + width*rand.Float64(), + Y: box.Point.Y + height*rand.Float64(), + } + return point +} + type UIResults []UIResult func (u UIResults) FilterScope(scope option.AbsScope) (results UIResults) { diff --git a/uixt/driver_ext_screenshot.go b/uixt/driver_ext_screenshot.go index 854baeb6..674b4a5f 100644 --- a/uixt/driver_ext_screenshot.go +++ b/uixt/driver_ext_screenshot.go @@ -166,7 +166,7 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) ( return textRect, nil } -func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) { +func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (uiResult ai.UIResult, err error) { options := option.NewActionOptions(opts...) if options.ScreenShotFileName == "" { opts = append(opts, option.WithScreenShotFileName( @@ -182,11 +182,10 @@ func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF if err != nil { return } - uiResult, err := uiResults.GetUIResult(opts...) - point = uiResult.Center() + uiResult, err = uiResults.GetUIResult(opts...) log.Info().Interface("text", options.ScreenShotWithUITypes). - Interface("point", point).Msg("FindUIResult success") + Interface("uiResult", uiResult).Msg("FindUIResult success") return } diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index 7283fd18..dd65f50a 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -32,15 +32,22 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { } func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error { - options := option.NewActionOptions(opts...) + actionOptions := option.NewActionOptions(opts...) - point, err := dExt.FindUIResult(opts...) + uiResult, err := dExt.FindUIResult(opts...) if err != nil { - if options.IgnoreNotFoundError { + if actionOptions.IgnoreNotFoundError { return nil } return err } + var point ai.PointF + if actionOptions.TapRandomRect { + point = uiResult.RandomPoint() + } else { + point = uiResult.Center() + } + return dExt.TapAbsXY(point.X, point.Y, opts...) } diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index 7b4f5a37..a5a98eaa 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -193,4 +193,10 @@ func TestDriverExt_Action_Risk(t *testing.T) { // tap random point in ocr text rect err = driver.TapByOCR("首页", option.WithTapRandomRect(true)) + assert.Nil(t, err) + + err = driver.TapByCV( + option.WithScreenShotUITypes("deepseek_send"), + option.WithTapRandomRect(true)) + assert.Nil(t, err) } From 7d8efaa5409d9fe5b67fbdb5f291e647f46a5ac7 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 18:13:10 +0800 Subject: [PATCH 7/9] test: drag direction with random offset --- internal/version/VERSION | 2 +- uixt/driver_ext_test.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 5200bf63..cf0f07e8 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171806 +v5.0.0-beta-2503171813 diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go index a5a98eaa..243d1493 100644 --- a/uixt/driver_ext_test.go +++ b/uixt/driver_ext_test.go @@ -170,7 +170,7 @@ func TestDriverExt_ClosePopupsHandler(t *testing.T) { assert.Nil(t, err) } -func TestDriverExt_Action_Risk(t *testing.T) { +func TestDriverExt_Action_Offset(t *testing.T) { driver := setupADBDriverExt(t) // tap point with constant offset @@ -191,6 +191,11 @@ func TestDriverExt_Action_Risk(t *testing.T) { option.WithOffsetRandomRange(-50, 50)) assert.Nil(t, err) + // drag direction with random offset + err = driver.Drag(0.5, 0.5, 0.5, 0.9, + option.WithOffsetRandomRange(-50, 50)) + assert.Nil(t, err) + // tap random point in ocr text rect err = driver.TapByOCR("首页", option.WithTapRandomRect(true)) assert.Nil(t, err) From fb3f8a6fca1150cd59a68754296dc3b4f1a02881 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 20:08:51 +0800 Subject: [PATCH 8/9] change: add logs --- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 21 +++++++++++++++++++++ uixt/android_driver_uia2.go | 14 ++++++++++++++ uixt/driver_ext_tap.go | 5 +++++ uixt/harmony_driver_hdc.go | 7 +++++++ uixt/ios_driver_wda.go | 19 ++++++++++++++++++- 6 files changed, 66 insertions(+), 2 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index cf0f07e8..e696d89d 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503171813 +v5.0.0-beta-2503172016 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 3bf01d46..ed5c7ab6 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -173,6 +173,7 @@ func (ad *ADBDriver) WindowSize() (size types.Size, err error) { // Back simulates a short press on the BACK button. func (ad *ADBDriver) Back() (err error) { + log.Info().Msg("ADBDriver.Back") // adb shell input keyevent 4 _, err = ad.runShellCommand("input", "keyevent", fmt.Sprintf("%d", KCBack)) if err != nil { @@ -200,10 +201,12 @@ func (ad *ADBDriver) Orientation() (orientation types.Orientation, err error) { } func (ad *ADBDriver) Home() (err error) { + log.Info().Msg("ADBDriver.Home") return ad.PressKeyCode(KCHome, KMEmpty) } func (ad *ADBDriver) Unlock() (err error) { + log.Info().Msg("ADBDriver.Unlock") // Notice: brighten should be executed before unlock // brighten android device screen if err := ad.PressKeyCode(KCWakeup, KMEmpty); err != nil { @@ -219,6 +222,7 @@ func (ad *ADBDriver) Unlock() (err error) { } func (ad *ADBDriver) Backspace(count int, opts ...option.ActionOption) (err error) { + log.Info().Int("count", count).Msg("ADBDriver.Backspace") if count == 0 { return nil } @@ -263,6 +267,7 @@ func (ad *ADBDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta) (err error } func (ad *ADBDriver) AppLaunch(packageName string) (err error) { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppLaunch") // 不指定 Activity 名称启动(启动主 Activity) // adb shell monkey -p -c android.intent.category.LAUNCHER 1 sOutput, err := ad.runShellCommand( @@ -280,6 +285,7 @@ func (ad *ADBDriver) AppLaunch(packageName string) (err error) { } func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err error) { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppTerminate") // 强制停止应用,停止 相关的进程 // adb shell am force-stop _, err = ad.runShellCommand("am", "force-stop", packageName) @@ -291,6 +297,7 @@ func (ad *ADBDriver) AppTerminate(packageName string) (successful bool, err erro } func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapXY") absX, absY, err := convertToAbsolutePoint(ad, x, y) if err != nil { return err @@ -299,6 +306,7 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY") actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) @@ -314,6 +322,7 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap") var err error x, y, err = convertToAbsolutePoint(ad, x, y) if err != nil { @@ -340,6 +349,7 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error } func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) duration := 1000.0 @@ -360,6 +370,8 @@ func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er } func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag") actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) if err != nil { @@ -389,6 +401,8 @@ func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO } func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe") var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) if err != nil { @@ -419,6 +433,7 @@ func (ad *ADBDriver) ForceTouchFloat(x, y, pressure float64, second ...float64) } func (ad *ADBDriver) Input(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("ADBDriver.Input") err := ad.SendUnicodeKeys(text, opts...) if err == nil { return nil @@ -436,6 +451,7 @@ func (ad *ADBDriver) input(text string, _ ...option.ActionOption) error { } func (ad *ADBDriver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("ADBDriver.SendUnicodeKeys") // If the Unicode IME is not installed, fall back to the old interface. // There might be differences in the tracking schemes across different phones, and it is pending further verification. // In release version: without the Unicode IME installed, the test cannot execute. @@ -523,6 +539,7 @@ func (ad *ADBDriver) SendKeysByAdbKeyBoard(text string) (err error) { } func (ad *ADBDriver) AppClear(packageName string) error { + log.Info().Str("packageName", packageName).Msg("ADBDriver.AppClear") if _, err := ad.runShellCommand("pm", "clear", packageName); err != nil { log.Error().Str("packageName", packageName).Err(err).Msg("failed to clear package cache") return err @@ -552,6 +569,7 @@ func (ad *ADBDriver) ScreenShot(opts ...option.ActionOption) (raw *bytes.Buffer, } func (ad *ADBDriver) TapByHierarchy(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("ADBDriver.TapByHierarchy") sourceTree, err := ad.sourceTree() if err != nil { return err @@ -784,6 +802,7 @@ func (ad *ADBDriver) GetIme() (ime string, err error) { } func (ad *ADBDriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) { + log.Info().Msg("ADBDriver.ScreenRecord") options := option.NewActionOptions(opts...) var filePath string @@ -924,6 +943,7 @@ func (ad *ADBDriver) OpenUrl(url string) (err error) { } func (ad *ADBDriver) PushImage(localPath string) error { + log.Info().Str("localPath", localPath).Msg("ADBDriver.PushImage") remotePath := path.Join("/sdcard/DCIM/Camera/", path.Base(localPath)) if err := ad.Device.PushFile(localPath, remotePath); err != nil { return err @@ -936,6 +956,7 @@ func (ad *ADBDriver) PushImage(localPath string) error { } func (ad *ADBDriver) ClearImages() error { + log.Info().Msg("ADBDriver.ClearImages") _, _ = ad.Device.RunShellCommand("rm", "-rf", "/sdcard/DCIM/Camera/*") return nil } diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index edb37f66..88416934 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -215,6 +215,7 @@ func (ud *UIA2Driver) WindowSize() (size types.Size, err error) { // Back simulates a short press on the BACK button. func (ud *UIA2Driver) Back() (err error) { + log.Info().Msg("UIA2Driver.Back") // register(postHandler, new PressBack("/wd/hub/session/:sessionId/back")) urlStr := fmt.Sprintf("/session/%s/back", ud.Session.ID) _, err = ud.Session.POST(nil, urlStr) @@ -253,6 +254,7 @@ func (ud *UIA2Driver) Orientation() (orientation types.Orientation, err error) { } func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap") var err error x, y, err = convertToAbsolutePoint(ud, x, y) if err != nil { @@ -284,6 +286,7 @@ func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error } func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapXY") // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) absX, absY, err := convertToAbsolutePoint(ud, x, y) if err != nil { @@ -293,6 +296,7 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY") // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) @@ -324,6 +328,7 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error } func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) duration := actionOptions.Duration @@ -348,6 +353,8 @@ func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (e // Each step execution is throttled to 5 milliseconds per step, so for a 100 // steps, the swipe will take around 0.5 seconds to complete. func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag") var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY) if err != nil { @@ -377,6 +384,8 @@ func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.Action // `steps` is the number of move steps sent to the system func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { // register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform")) + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe") var err error actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY) @@ -412,6 +421,8 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio } func (ud *UIA2Driver) SetPasteboard(contentType types.PasteboardType, content string) (err error) { + log.Info().Str("contentType", string(contentType)). + Str("content", content).Msg("UIA2Driver.SetPasteboard") lbl := content const defaultLabelLen = 10 @@ -458,6 +469,7 @@ func (ud *UIA2Driver) GetPasteboard(contentType types.PasteboardType) (raw *byte // SendKeys Android input does not support setting frequency. func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.Input") // register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys")) // https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85 err = ud.SendUnicodeKeys(text, opts...) @@ -475,6 +487,7 @@ func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error } func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.SendUnicodeKeys") // If the Unicode IME is not installed, fall back to the old interface. // There might be differences in the tracking schemes across different phones, and it is pending further verification. // In release version: without the Unicode IME installed, the test cannot execute. @@ -505,6 +518,7 @@ func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) } func (ud *UIA2Driver) SendActionKey(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("UIA2Driver.SendActionKey") var actions []interface{} for i, c := range text { actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)}, diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go index dd65f50a..05dea676 100644 --- a/uixt/driver_ext_tap.go +++ b/uixt/driver_ext_tap.go @@ -5,6 +5,7 @@ import ( "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" + "github.com/rs/zerolog/log" ) func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { @@ -27,6 +28,8 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error { } else { point = textRect.Center() } + log.Info().Str("text", text).Interface("rawTextRect", textRect). + Interface("tapPoint", point).Msg("TapByOCR success") return dExt.TapAbsXY(point.X, point.Y, opts...) } @@ -48,6 +51,8 @@ func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error { } else { point = uiResult.Center() } + log.Info().Interface("rawUIResult", uiResult). + Interface("tapPoint", point).Msg("TapByCV success") return dExt.TapAbsXY(point.X, point.Y, opts...) } diff --git a/uixt/harmony_driver_hdc.go b/uixt/harmony_driver_hdc.go index 6ffa155d..1541245f 100644 --- a/uixt/harmony_driver_hdc.go +++ b/uixt/harmony_driver_hdc.go @@ -97,6 +97,7 @@ const ( ) func (hd *HDCDriver) Unlock() (err error) { + log.Info().Msg("HDCDriver.Unlock") // Todo 检查是否锁屏 hdc shell hidumper -s RenderService -a screen screenInfo, err := hd.Device.RunShellCommand("hidumper", "-s", "RenderService", "-a", "screen") if err != nil { @@ -124,6 +125,7 @@ func (hd *HDCDriver) AppLaunch(packageName string) error { } func (hd *HDCDriver) AppTerminate(packageName string) (bool, error) { + log.Info().Str("packageName", packageName).Msg("HDCDriver.AppTerminate") _, err := hd.Device.RunShellCommand("aa", "force-stop", packageName) if err != nil { log.Error().Err(err).Msg("failed to terminal app") @@ -142,6 +144,7 @@ func (hd *HDCDriver) Orientation() (orientation types.Orientation, err error) { } func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapXY") absX, absY, err := convertToAbsolutePoint(hd, x, y) if err != nil { return err @@ -150,6 +153,7 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY") actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) @@ -175,6 +179,8 @@ func (hd *HDCDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO // Swipe works like Drag, but `pressForDuration` value is 0 func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe") var err error actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(hd, fromX, fromY, toX, toY) @@ -209,6 +215,7 @@ func (hd *HDCDriver) AppClear(packageName string) error { } func (hd *HDCDriver) Back() error { + log.Info().Msg("HDCDriver.Back") return hd.uiDriver.PressBack() } diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index c08aa2ce..0c56aa66 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -108,7 +108,7 @@ func (wd *WDADriver) initMjpegClient() error { } mjpegHTTPConn, err := net.Dial( "tcp", - fmt.Sprintf("%s:%d", host, localMjpegPort), + net.JoinHostPort(host, fmt.Sprintf("%d", localMjpegPort)), ) if err != nil { return errors.Wrap(code.DeviceHTTPDriverError, err.Error()) @@ -492,6 +492,7 @@ func (wd *WDADriver) AlertSendKeys(text string) (err error) { } func (wd *WDADriver) AppLaunch(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunch") // [[FBRoute POST:@"/wda/apps/launch"] respondWithTarget:self action:@selector(handleSessionAppLaunch:)] data := make(map[string]interface{}) data["bundleId"] = bundleId @@ -508,6 +509,7 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) { } func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunchUnattached") // [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)] data := map[string]interface{}{"bundleId": bundleId} _, err = wd.Session.POST(data, "/wda/apps/launchUnattached") @@ -519,6 +521,7 @@ func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) { } func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppTerminate") // [[FBRoute POST:@"/wda/apps/terminate"] respondWithTarget:self action:@selector(handleSessionAppTerminate:)] data := map[string]interface{}{"bundleId": bundleId} var rawResp DriverRawResponse @@ -533,6 +536,7 @@ func (wd *WDADriver) AppTerminate(bundleId string) (successful bool, err error) } func (wd *WDADriver) AppActivate(bundleId string) (err error) { + log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppActivate") // [[FBRoute POST:@"/wda/apps/activate"] respondWithTarget:self action:@selector(handleSessionAppActivate:)] data := map[string]interface{}{"bundleId": bundleId} urlStr := fmt.Sprintf("/session/%s/wda/apps/activate", wd.Session.ID) @@ -541,6 +545,7 @@ func (wd *WDADriver) AppActivate(bundleId string) (err error) { } func (wd *WDADriver) AppDeactivate(second float64) (err error) { + log.Info().Float64("second", second).Msg("WDADriver.AppDeactivate") // [[FBRoute POST:@"/wda/deactivateApp"] respondWithTarget:self action:@selector(handleDeactivateAppCommand:)] if second < 3 { second = 3.0 @@ -576,6 +581,7 @@ func (wd *WDADriver) ForegroundInfo() (appInfo types.AppInfo, err error) { } func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapXY") // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] absX, absY, err := convertToAbsolutePoint(wd, x, y) if err != nil { @@ -585,6 +591,7 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error { } func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY") // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) @@ -600,6 +607,7 @@ func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { } func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap") // [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)] var err error x, y, err = convertToAbsolutePoint(wd, x, y) @@ -622,6 +630,7 @@ func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error // FIXME: hold not work func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (err error) { + log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TouchAndHold") actionOptions := option.NewActionOptions(opts...) x, y = actionOptions.ApplyTapOffset(x, y) if actionOptions.Duration == 0 { @@ -631,6 +640,8 @@ func (wd *WDADriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er } func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + log.Info().Float64("fromX", fromX).Float64("fromY", fromY). + Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag") // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] var err error fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(wd, fromX, fromY, toX, toY) @@ -692,6 +703,7 @@ func (wd *WDADriver) SetIme(ime string) error { } func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) { + log.Info().Str("text", text).Msg("WDADriver.Input") // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] data := map[string]interface{}{"value": strings.Split(text, "")} option.MergeOptions(data, opts...) @@ -701,6 +713,7 @@ func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) } func (wd *WDADriver) Backspace(count int, opts ...option.ActionOption) (err error) { + log.Info().Int("count", count).Msg("WDADriver.Backspace") if count == 0 { return nil } @@ -716,6 +729,7 @@ func (wd *WDADriver) AppClear(packageName string) error { // Back simulates a short press on the BACK button. func (wd *WDADriver) Back() (err error) { + log.Info().Msg("WDADriver.Back") return wd.Swipe(0, 0.5, 0.6, 0.5) } @@ -879,6 +893,7 @@ func (wd *WDADriver) triggerWDALog(data map[string]interface{}) (rawResp []byte, } func (wd *WDADriver) ScreenRecord(opts ...option.ActionOption) (videoPath string, err error) { + log.Info().Msg("WDADriver.ScreenRecord") timestamp := time.Now().Format("20060102_150405") + fmt.Sprintf("_%03d", time.Now().UnixNano()/1e6%1000) fileName := filepath.Join(config.GetConfig().ScreenShotsPath, fmt.Sprintf("%s.mp4", timestamp)) @@ -954,6 +969,7 @@ func (wd *WDADriver) StartCaptureLog(identifier ...string) error { } func (wd *WDADriver) PushImage(localPath string) error { + log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage") localFile, err := os.Open(localPath) if err != nil { return err @@ -974,6 +990,7 @@ func (wd *WDADriver) PushImage(localPath string) error { } func (wd *WDADriver) ClearImages() error { + log.Info().Msg("WDADriver.ClearImages") data := map[string]interface{}{} _, err := wd.Session.POST(data, "/gtf/albums/clear") From d63e5e0c1acb819b7b1493b563ad28f97d9ab704 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 17 Mar 2025 20:40:03 +0800 Subject: [PATCH 9/9] feat: add GitCommit/GitBranch/BuildTime in version info --- Makefile | 8 +++++++- cmd/root.go | 2 +- internal/version/VERSION | 2 +- internal/version/init.go | 13 ------------- internal/version/version.go | 31 +++++++++++++++++++++++++++++++ scripts/build.sh | 23 ----------------------- 6 files changed, 40 insertions(+), 39 deletions(-) delete mode 100644 internal/version/init.go create mode 100644 internal/version/version.go delete mode 100644 scripts/build.sh diff --git a/Makefile b/Makefile index f2f15434..1f1de82b 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,13 @@ bump: ## bump hrp version, e.g. make bump version=4.0.0 .PHONY: build build: ## build hrp cli tool @echo "[info] build hrp cli tool" - @. scripts/build.sh $(tags) + go build -ldflags "\ + -s -w \ + -X 'github.com/httprunner/httprunner/v5/internal/version.GitCommit=$(shell git rev-parse HEAD)' \ + -X 'github.com/httprunner/httprunner/v5/internal/version.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)' \ + -X 'github.com/httprunner/httprunner/v5/internal/version.BuildTime=$(shell date "+%y%m%d%H%M")'" \ + -o output/hrp ./cmd/cli + ./output/hrp -v .PHONY: install-hooks install-hooks: ## install git hooks diff --git a/cmd/root.go b/cmd/root.go index aed93bd3..77bc8ae1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -51,7 +51,7 @@ Copyright © 2017-present debugtalk. Apache-2.0 License.`, PersistentPreRun: func(cmd *cobra.Command, args []string) { hrp.InitLogger(logLevel, logJSON) }, - Version: version.VERSION, + Version: version.GetVersionInfo(), TraverseChildren: true, // parses flags on all parents before executing child command SilenceUsage: true, // silence usage when an error occurs } diff --git a/internal/version/VERSION b/internal/version/VERSION index e696d89d..b038c688 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2503172016 +v5.0.0-beta-2503172040 diff --git a/internal/version/init.go b/internal/version/init.go deleted file mode 100644 index 2ae38ef8..00000000 --- a/internal/version/init.go +++ /dev/null @@ -1,13 +0,0 @@ -package version - -import ( - _ "embed" - "strings" -) - -//go:embed VERSION -var VERSION string - -func init() { - VERSION = strings.TrimSpace(VERSION) -} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 00000000..f936a5f4 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,31 @@ +package version + +import ( + "fmt" + "strings" + + _ "embed" +) + +//go:embed VERSION +var VERSION string + +func init() { + VERSION = strings.TrimSpace(VERSION) +} + +// 版本信息,在编译时通过 -ldflags 注入 +var ( + GitCommit = "unknown" + GitBranch = "unknown" + BuildTime = "unknown" +) + +func GetVersionInfo() string { + commitID := GitCommit + if len(commitID) > 8 { + commitID = commitID[:8] + } + return fmt.Sprintf("%s (branch=%s, commit=%s, build=%s)", + VERSION, GitBranch, commitID, BuildTime) +} diff --git a/scripts/build.sh b/scripts/build.sh deleted file mode 100644 index ccb2a30e..00000000 --- a/scripts/build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# build hrp cli binary for testing -# release will be triggered on github actions, see .github/workflows/release.yml - -# Usage: -# $ make build -# or -# $ bash scripts/build.sh - -set -e -set -x - -# prepare path -mkdir -p "output" -bin_path="output/hrp" - -# build -go build -ldflags '-s -w' -o "$bin_path" cmd/cli/main.go - -# check output and version -ls -lh "$bin_path" -chmod +x "$bin_path" -./"$bin_path" -v