From 3613ffb2c096a7c14ba890dd20d256c2b86ddfab Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Tue, 26 Sep 2023 18:09:47 +0800 Subject: [PATCH 1/7] fix: compatible with swipe params --- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/swipe.go | 2 +- hrp/testcase.go | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 471e7432..9c6a0187 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.6 \ No newline at end of file +v4.3.6.20230926 \ No newline at end of file diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 0e34e23b..5ee1c408 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -114,7 +114,7 @@ func (dExt *DriverExt) prepareSwipeAction(options ...ActionOption) func(d *Drive log.Error().Err(err).Msgf("swipe %s failed", d) return err } - } else if d, ok := swipeDirection.([]float64); ok { + } else if d, ok := swipeDirection.([]float64); ok && len(d) == 4 { // custom direction: [fromX, fromY, toX, toY] if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], options...); err != nil { log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", diff --git a/hrp/testcase.go b/hrp/testcase.go index 3640f6f5..9b301465 100644 --- a/hrp/testcase.go +++ b/hrp/testcase.go @@ -380,6 +380,9 @@ func convertCompatMobileStep(mobileStep *MobileStep) { if ma.Method == uixt.ACTION_SwipeToTapText && actionOptions.MaxRetryTimes == 0 { ma.ActionOptions.MaxRetryTimes = 10 } + if ma.Method == uixt.ACTION_Swipe { + ma.ActionOptions.Direction = ma.Params + } mobileStep.Actions[i] = ma } } From 77161849e57284da436d9da7aab69ae60dae1b1d Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Sat, 7 Oct 2023 18:49:57 +0800 Subject: [PATCH 2/7] fix: use ADBKeyBoard if uiautomator failed --- hrp/pkg/uixt/android_adb_driver.go | 34 +++++++++++++++++++++++++++++ hrp/pkg/uixt/android_uia2_driver.go | 12 ++++++++-- hrp/pkg/uixt/ios_driver.go | 4 ++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index b2d84652..64b4d94e 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -14,6 +14,8 @@ import ( "github.com/httprunner/httprunner/v4/hrp/pkg/gadb" ) +const AdbKeyBoardPackageName = "com.android.adbkeyboard/.AdbIME" + type adbDriver struct { Driver @@ -327,6 +329,38 @@ func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error) return nil } +func (ad *adbDriver) IsAdbKeyBoardInstalled() bool { + output, err := ad.adbClient.RunShellCommand("ime", "list", "-a") + if err != nil { + return false + } + return strings.Contains(output, AdbKeyBoardPackageName) +} + +func (ad *adbDriver) SendKeysByAdbKeyBoard(text string) (err error) { + defer func() { + // Reset to default, don't care which keyboard was chosen before switch: + _, err = ad.adbClient.RunShellCommand("ime", "reset") + }() + + // Enable ADBKeyBoard from adb + if _, err = ad.adbClient.RunShellCommand("ime", "enable", AdbKeyBoardPackageName); err != nil { + return + } + // Switch to ADBKeyBoard from adb + if _, err = ad.adbClient.RunShellCommand("ime", "set", AdbKeyBoardPackageName); err != nil { + return + } + time.Sleep(time.Second) + // input Quoted text + text = strings.ReplaceAll(text, " ", "\\ ") + if _, err = ad.adbClient.RunShellCommand("am", "broadcast", "-a", "ADB_INPUT_TEXT", "--es", "msg", text); err != nil { + return + } + time.Sleep(time.Second) + return +} + func (ad *adbDriver) Input(text string, options ...ActionOption) (err error) { return ad.SendKeys(text, options...) } diff --git a/hrp/pkg/uixt/android_uia2_driver.go b/hrp/pkg/uixt/android_uia2_driver.go index 674b70c3..5f38556d 100644 --- a/hrp/pkg/uixt/android_uia2_driver.go +++ b/hrp/pkg/uixt/android_uia2_driver.go @@ -103,8 +103,8 @@ func (ud *uiaDriver) httpRequest(method string, rawURL string, rawBody []byte, d // wait for UIA2 server to resume automatically time.Sleep(3 * time.Second) oldSessionID := ud.sessionId - if err = ud.resetDriver(); err != nil { - log.Err(err).Msgf("failed to reset uia2 driver, retry count: %v", retryCount) + if err2 := ud.resetDriver(); err2 != nil { + log.Err(err2).Msgf("failed to reset uia2 driver, retry count: %v", retryCount) continue } log.Debug().Str("new session", ud.sessionId).Str("old session", oldSessionID).Msgf("successful to reset uia2 driver, retry count: %v", retryCount) @@ -426,6 +426,14 @@ func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) actionOptions.updateData(data) _, err = ud.httpPOST(data, "/session", ud.sessionId, "keys") + if err != nil { + // use com.android.adbkeyboard if existed + if ud.IsAdbKeyBoardInstalled() { + err = ud.SendKeysByAdbKeyBoard(text) + } else { + _, err = ud.adbClient.RunShellCommand("input", "text", text) + } + } return } diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 52390256..ea2d305f 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -55,8 +55,8 @@ func (wd *wdaDriver) httpRequest(method string, rawURL string, rawBody []byte, d // TODO: polling WDA to check if resumed automatically time.Sleep(5 * time.Second) oldSessionID := wd.sessionId - if err = wd.resetSession(); err != nil { - log.Err(err).Msgf("failed to reset wda driver, retry count: %v", retryCount) + if err2 := wd.resetSession(); err2 != nil { + log.Err(err2).Msgf("failed to reset wda driver, retry count: %v", retryCount) continue } log.Debug().Str("new session", wd.sessionId).Str("old session", oldSessionID).Msgf("successful to reset wda driver, retry count: %v", retryCount) From acbb3902ccbe0f1697fade66882db8ef5f67c55d Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Sat, 7 Oct 2023 18:53:13 +0800 Subject: [PATCH 3/7] change: use default ocrCluster and support to sepecify timeout --- hrp/pkg/uixt/service_vedem.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index 7e32a9e3..79d137fc 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -218,7 +218,9 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp bodyWriter.WriteField("uiTypes", uiType) } - bodyWriter.WriteField("ocrCluster", "highPrecision") + if actionOptions.Timeout > 0 { + bodyWriter.WriteField("timeout", fmt.Sprintf("%v", actionOptions.Timeout)) + } formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") if err != nil { From 41f705c0d94f10545e37d3cb6f6a1ee8f00417aa Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Sat, 7 Oct 2023 19:45:41 +0800 Subject: [PATCH 4/7] fix: device serial/udid should be specified --- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/android_device.go | 7 +++++-- hrp/pkg/uixt/ios_device.go | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 9c6a0187..d87fdf6c 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.6.20230926 \ No newline at end of file +v4.3.6.2310081724 \ No newline at end of file diff --git a/hrp/pkg/uixt/android_device.go b/hrp/pkg/uixt/android_device.go index 42b4757e..84edf057 100644 --- a/hrp/pkg/uixt/android_device.go +++ b/hrp/pkg/uixt/android_device.go @@ -88,10 +88,13 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er for _, option := range options { option(device) } - deviceList, err := GetAndroidDevices(device.SerialNumber) if err != nil { - return nil, err + return nil, errors.Wrap(code.AndroidDeviceConnectionError, err.Error()) + } + + if device.SerialNumber == "" && len(deviceList) > 1 { + return nil, errors.Wrap(code.AndroidDeviceConnectionError, "more than one device connected, please specify the serial") } dev := deviceList[0] diff --git a/hrp/pkg/uixt/ios_device.go b/hrp/pkg/uixt/ios_device.go index 843a34b3..b5283667 100644 --- a/hrp/pkg/uixt/ios_device.go +++ b/hrp/pkg/uixt/ios_device.go @@ -237,6 +237,10 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) { return nil, errors.Wrap(code.IOSDeviceConnectionError, err.Error()) } + if device.UDID == "" && len(deviceList) > 1 { + return nil, errors.Wrap(code.IOSDeviceConnectionError, "more than one device connected, please specify the udid") + } + dev := deviceList[0] udid := dev.Properties().SerialNumber From 79c91637db871524b5f45448b6e6e6982130a191 Mon Sep 17 00:00:00 2001 From: buyuxiang Date: Wed, 11 Oct 2023 14:59:49 +0800 Subject: [PATCH 5/7] close popups before swipeToTapApp --- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/service_vedem.go | 2 +- hrp/pkg/uixt/swipe.go | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index d87fdf6c..e764e622 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.6.2310081724 \ No newline at end of file +v4.3.6.2310111458 \ No newline at end of file diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index 79d137fc..74baef99 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -409,7 +409,7 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S screenResult.UploadedURL = imageResult.URL screenResult.Icons = imageResult.UIResult - if actionOptions.ScreenShotWithClosePopups { + if actionOptions.ScreenShotWithClosePopups && imageResult.CPResult != nil { screenResult.Popup = &PopupInfo{ Type: imageResult.CPResult.Type, Text: imageResult.CPResult.Text, diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 5ee1c408..4c97d212 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -178,6 +178,11 @@ func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) er return errors.Wrap(err, "go to home screen failed") } + // automatic handling popups before swipe + if err := dExt.ClosePopups(); err != nil { + log.Error().Err(err).Msg("auto handle popup failed") + } + // swipe to first screen for i := 0; i < 5; i++ { dExt.SwipeRight() From c22dc0836c067cc29fe803ed53e1fcff1222472c Mon Sep 17 00:00:00 2001 From: buyuxiang Date: Fri, 20 Oct 2023 12:03:11 +0800 Subject: [PATCH 6/7] fix: unpredefined tap_cv not found error --- hrp/pkg/uixt/service_vedem.go | 2 +- hrp/testcase.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index 74baef99..d91e3950 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -537,7 +537,7 @@ func (u UIResultMap) FilterUIResults(uiTypes []string) (uiResults UIResults, err return } } - err = errors.Errorf("UI types %v not detected", uiTypes) + err = errors.Wrap(code.CVResultNotFoundError, fmt.Sprintf("UI types %v not detected", uiTypes)) return } diff --git a/hrp/testcase.go b/hrp/testcase.go index 9b301465..9cbccce0 100644 --- a/hrp/testcase.go +++ b/hrp/testcase.go @@ -371,6 +371,7 @@ func convertCompatMobileStep(mobileStep *MobileStep) { if ma.Method == uixt.ACTION_TapByCV { uiTypes, _ := builtin.ConvertToStringSlice(ma.Params) ma.ActionOptions.ScreenShotWithUITypes = append(ma.ActionOptions.ScreenShotWithUITypes, uiTypes...) + ma.ActionOptions.ScreenShotWithUpload = true } // set default max_retry_times to 10 for swipe_to_tap_texts if ma.Method == uixt.ACTION_SwipeToTapTexts && actionOptions.MaxRetryTimes == 0 { From a772f836ec546c4d1ed9b1c4020a8831a61422c6 Mon Sep 17 00:00:00 2001 From: buyuxiang Date: Thu, 26 Oct 2023 22:05:04 +0800 Subject: [PATCH 7/7] fix: add lost options --- hrp/pkg/uixt/action.go | 16 ++++++++++++++++ hrp/pkg/uixt/service_vedem.go | 3 +++ 2 files changed, 19 insertions(+) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 242b59ea..cf28c4b8 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -121,6 +121,7 @@ type ActionOptions struct { ScreenShotWithLiveType bool `json:"screenshot_with_live_type,omitempty" yaml:"screenshot_with_live_type,omitempty"` ScreenShotWithUITypes []string `json:"screenshot_with_ui_types,omitempty" yaml:"screenshot_with_ui_types,omitempty"` ScreenShotWithClosePopups bool `json:"screenshot_with_close_popups,omitempty" yaml:"screenshot_with_close_popups,omitempty"` + ScreenShotWithOCRCluster string `json:"screenshot_with_ocr_cluster,omitempty" yaml:"screenshot_with_ocr_cluster,omitempty"` } func (o *ActionOptions) Options() []ActionOption { @@ -180,6 +181,9 @@ func (o *ActionOptions) Options() []ActionOption { if len(o.AbsScope) == 4 { options = append(options, WithAbsScope( o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3])) + } else if len(o.Scope) == 4 { + options = append(options, WithScope( + o.Scope[0], o.Scope[1], o.Scope[2], o.Scope[3])) } if len(o.Offset) == 2 { // for tap [x,y] offset @@ -224,6 +228,12 @@ func (o *ActionOptions) Options() []ActionOption { if len(o.ScreenShotWithUITypes) > 0 { options = append(options, WithScreenShotUITypes(o.ScreenShotWithUITypes...)) } + if o.ScreenShotWithClosePopups { + options = append(options, WithScreenShotClosePopups(true)) + } + if o.ScreenShotWithOCRCluster != "" { + options = append(options, WithScreenOCRCluster(o.ScreenShotWithOCRCluster)) + } return options } @@ -472,6 +482,12 @@ func WithScreenShotClosePopups(closeOn bool) ActionOption { } } +func WithScreenOCRCluster(ocrCluster string) ActionOption { + return func(o *ActionOptions) { + o.ScreenShotWithOCRCluster = ocrCluster + } +} + func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption { actionOptions := NewActionOptions(options...) diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index d91e3950..e2c4330b 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -217,6 +217,9 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp for _, uiType := range actionOptions.ScreenShotWithUITypes { bodyWriter.WriteField("uiTypes", uiType) } + if actionOptions.ScreenShotWithOCRCluster != "" { + bodyWriter.WriteField("ocrCluster", actionOptions.ScreenShotWithOCRCluster) + } if actionOptions.Timeout > 0 { bodyWriter.WriteField("timeout", fmt.Sprintf("%v", actionOptions.Timeout))