From d6db673962e7f9f12b65dfe7bb0ec8b8f1d35f49 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 11:22:45 +0800 Subject: [PATCH 01/67] change: update changelog --- docs/CHANGELOG.md | 2 ++ go.mod | 2 +- go.sum | 4 ++-- hrp/internal/scaffold/templates/plugin/debugtalk_gen.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 524aca34..ca27fbc5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,6 +20,8 @@ - fix: fast fail not closing the websocket connection - fix #1467: failed to parse parameters with plugin functions - fix #1549: avoid duplicate creating plugins +- fix #1547: generate html report failed for referenced testcases +- fix: setup hooks compatible with v3 ## v4.3.2 (2022-12-26) diff --git a/go.mod b/go.mod index f6975946..39dc3ec5 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 - gocv.io/x/gocv v0.31.0 + gocv.io/x/gocv v0.32.1 golang.org/x/net v0.7.0 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 google.golang.org/grpc v1.49.0 diff --git a/go.sum b/go.sum index 979a82b0..58a172df 100644 --- a/go.sum +++ b/go.sum @@ -421,8 +421,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -gocv.io/x/gocv v0.31.0 h1:BHDtK8v+YPvoSPQTTiZB2fM/7BLg6511JqkruY2z6LQ= -gocv.io/x/gocv v0.31.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= +gocv.io/x/gocv v0.32.1 h1:BC9hHs5+47nVgySUFVKntc6RsF3SULFzqk6OV9xz+C0= +gocv.io/x/gocv v0.32.1/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index 9d08c9a0..1590229f 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.3.0, DO NOT EDIT! +// NOTE: Generated By hrp v4.3.3, DO NOT EDIT! package main import ( From 7a255e6ef527ffcf5f051672e030ec56ca36ed49 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 13:26:56 +0800 Subject: [PATCH 02/67] feat: add video crawler --- hrp/pkg/uixt/ext.go | 100 +++++++++++++++++++--------------- hrp/pkg/uixt/video_crawler.go | 5 ++ hrp/step_mobile_ui.go | 10 +++- 3 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 hrp/pkg/uixt/video_crawler.go diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index ca09aaed..a7f80195 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -39,6 +39,7 @@ const ( CtlStopCamera MobileMethod = "camera_stop" // alias for app_terminate camera RecordStart MobileMethod = "record_start" RecordStop MobileMethod = "record_stop" + VideoCrawler MobileMethod = "video_crawler" // UI validation // selectors @@ -657,50 +658,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { if !ok { return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) } - // append default weight 1 - if len(params) == 2 { - params = append(params, 1.0) - } - - var sections []struct { - min, max, weight float64 - } - totalProb := 0.0 - for i := 0; i+3 <= len(params); i += 3 { - min, err := convertToFloat64(params[i]) - if err != nil { - return errors.Wrapf(err, "invalid minimum time: %v", params[i]) - } - max, err := convertToFloat64(params[i+1]) - if err != nil { - return errors.Wrapf(err, "invalid maximum time: %v", params[i+1]) - } - weight, err := convertToFloat64(params[i+2]) - if err != nil { - return errors.Wrapf(err, "invalid weight value: %v", params[i+2]) - } - totalProb += weight - sections = append(sections, - struct{ min, max, weight float64 }{min, max, weight}, - ) - } - - if totalProb == 0 { - log.Warn().Msg("total weight is 0, skip sleep") - return nil - } - - r := rand.Float64() - accProb := 0.0 - for _, s := range sections { - accProb += s.weight / totalProb - if r < accProb { - n := s.min + rand.Float64()*(s.max-s.min) - log.Info().Float64("duration", n).Msg("sleep random seconds") - time.Sleep(time.Duration(n*1000) * time.Millisecond) - return nil - } - } + return sleepRandom(params) case CtlScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") @@ -710,6 +668,60 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return dExt.Driver.StartCamera() case CtlStopCamera: return dExt.Driver.StopCamera() + case VideoCrawler: + params, ok := action.Params.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params) + } + return dExt.VideoCrawler(params) + } + return nil +} + +func sleepRandom(params []interface{}) error { + // append default weight 1 + if len(params) == 2 { + params = append(params, 1.0) + } + + var sections []struct { + min, max, weight float64 + } + totalProb := 0.0 + for i := 0; i+3 <= len(params); i += 3 { + min, err := convertToFloat64(params[i]) + if err != nil { + return errors.Wrapf(err, "invalid minimum time: %v", params[i]) + } + max, err := convertToFloat64(params[i+1]) + if err != nil { + return errors.Wrapf(err, "invalid maximum time: %v", params[i+1]) + } + weight, err := convertToFloat64(params[i+2]) + if err != nil { + return errors.Wrapf(err, "invalid weight value: %v", params[i+2]) + } + totalProb += weight + sections = append(sections, + struct{ min, max, weight float64 }{min, max, weight}, + ) + } + + if totalProb == 0 { + log.Warn().Msg("total weight is 0, skip sleep") + return nil + } + + r := rand.Float64() + accProb := 0.0 + for _, s := range sections { + accProb += s.weight / totalProb + if r < accProb { + n := s.min + rand.Float64()*(s.max-s.min) + log.Info().Float64("duration", n).Msg("sleep random seconds") + time.Sleep(time.Duration(n*1000) * time.Millisecond) + return nil + } } return nil } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go new file mode 100644 index 00000000..064507cb --- /dev/null +++ b/hrp/pkg/uixt/video_crawler.go @@ -0,0 +1,5 @@ +package uixt + +func (dExt *DriverExt) VideoCrawler(params map[string]interface{}) error { + return nil +} diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index ff8b3a2c..368546d9 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -12,7 +12,7 @@ import ( ) type MobileStep struct { - Serial string `json:"serial,omitempty" yaml:"serial,omitempty"` + Serial string `json:"serial,omitempty" yaml:"serial,omitempty"` // android serial or ios udid uixt.MobileAction `yaml:",inline"` Actions []uixt.MobileAction `json:"actions,omitempty" yaml:"actions,omitempty"` } @@ -293,6 +293,14 @@ func (s *StepMobile) SleepRandom(params ...float64) *StepMobile { return &StepMobile{step: s.step} } +func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile { + s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ + Method: uixt.VideoCrawler, + Params: params, + }) + return &StepMobile{step: s.step} +} + func (s *StepMobile) ScreenShot() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ Method: uixt.CtlScreenShot, From 556050367e05e3f74dc680808b9e04c42117d6e9 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 14:37:21 +0800 Subject: [PATCH 03/67] refactor: rename ActionMethod --- hrp/pkg/uixt/ext.go | 78 +++++++++++++++++------------------ hrp/pkg/uixt/video_crawler.go | 4 ++ hrp/step_mobile_ui.go | 18 ++++---- 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index a7f80195..4f1c2ed5 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -23,23 +23,20 @@ import ( "github.com/rs/zerolog/log" ) -type MobileMethod string +type ActionMethod string const ( - AppInstall MobileMethod = "install" - AppUninstall MobileMethod = "uninstall" - AppStart MobileMethod = "app_start" - AppLaunch MobileMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成 - AppTerminate MobileMethod = "app_terminate" - AppStop MobileMethod = "app_stop" - CtlScreenShot MobileMethod = "screenshot" - CtlSleep MobileMethod = "sleep" - CtlSleepRandom MobileMethod = "sleep_random" - CtlStartCamera MobileMethod = "camera_start" // alias for app_launch camera - CtlStopCamera MobileMethod = "camera_stop" // alias for app_terminate camera - RecordStart MobileMethod = "record_start" - RecordStop MobileMethod = "record_stop" - VideoCrawler MobileMethod = "video_crawler" + ACTION_AppInstall ActionMethod = "install" + ACTION_AppUninstall ActionMethod = "uninstall" + ACTION_AppStart ActionMethod = "app_start" + ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成 + ACTION_AppTerminate ActionMethod = "app_terminate" + ACTION_AppStop ActionMethod = "app_stop" + ACTION_ScreenShot ActionMethod = "screenshot" + ACTION_Sleep ActionMethod = "sleep" + ACTION_SleepRandom ActionMethod = "sleep_random" + ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera + ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera // UI validation // selectors @@ -55,26 +52,27 @@ const ( AssertionNotExists string = "not_exists" // UI handling - ACTION_Home MobileMethod = "home" - ACTION_TapXY MobileMethod = "tap_xy" - ACTION_TapAbsXY MobileMethod = "tap_abs_xy" - ACTION_TapByOCR MobileMethod = "tap_ocr" - ACTION_TapByCV MobileMethod = "tap_cv" - ACTION_Tap MobileMethod = "tap" - ACTION_DoubleTapXY MobileMethod = "double_tap_xy" - ACTION_DoubleTap MobileMethod = "double_tap" - ACTION_Swipe MobileMethod = "swipe" - ACTION_Input MobileMethod = "input" - ACTION_Back MobileMethod = "back" + ACTION_Home ActionMethod = "home" + ACTION_TapXY ActionMethod = "tap_xy" + ACTION_TapAbsXY ActionMethod = "tap_abs_xy" + ACTION_TapByOCR ActionMethod = "tap_ocr" + ACTION_TapByCV ActionMethod = "tap_cv" + ACTION_Tap ActionMethod = "tap" + ACTION_DoubleTapXY ActionMethod = "double_tap_xy" + ACTION_DoubleTap ActionMethod = "double_tap" + ACTION_Swipe ActionMethod = "swipe" + ACTION_Input ActionMethod = "input" + ACTION_Back ActionMethod = "back" // custom actions - ACTION_SwipeToTapApp MobileMethod = "swipe_to_tap_app" // swipe left & right to find app and tap - ACTION_SwipeToTapText MobileMethod = "swipe_to_tap_text" // swipe up & down to find text and tap - ACTION_SwipeToTapTexts MobileMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap + ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap + ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap + ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap + ACTION_VideoCrawler ActionMethod = "video_crawler" ) type MobileAction struct { - Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"` + Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log @@ -397,15 +395,15 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action") switch action.Method { - case AppInstall: + case ACTION_AppInstall: // TODO return errActionNotImplemented - case AppLaunch: + case ACTION_AppLaunch: if bundleId, ok := action.Params.(string); ok { return dExt.Driver.AppLaunch(bundleId) } return fmt.Errorf("invalid %s params, should be bundleId(string), got %v", - AppLaunch, action.Params) + ACTION_AppLaunch, action.Params) case ACTION_SwipeToTapApp: if appName, ok := action.Params.(string); ok { return dExt.swipeToTapApp(appName, action) @@ -515,7 +513,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid %s params, should be app text([]string), got %v", ACTION_SwipeToTapText, action.Params) - case AppTerminate: + case ACTION_AppTerminate: if bundleId, ok := action.Params.(string); ok { success, err := dExt.Driver.AppTerminate(bundleId) if err != nil { @@ -640,7 +638,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return dExt.Driver.Input(param, options...) case ACTION_Back: return dExt.Driver.PressBack() - case CtlSleep: + case ACTION_Sleep: if param, ok := action.Params.(json.Number); ok { seconds, _ := param.Float64() time.Sleep(time.Duration(seconds*1000) * time.Millisecond) @@ -653,22 +651,22 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return nil } return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params) - case CtlSleepRandom: + case ACTION_SleepRandom: params, ok := action.Params.([]interface{}) if !ok { return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) } return sleepRandom(params) - case CtlScreenShot: + case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot")) return err - case CtlStartCamera: + case ACTION_StartCamera: return dExt.Driver.StartCamera() - case CtlStopCamera: + case ACTION_StopCamera: return dExt.Driver.StopCamera() - case VideoCrawler: + case ACTION_VideoCrawler: params, ok := action.Params.(map[string]interface{}) if !ok { return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 064507cb..1cd757b0 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -1,5 +1,9 @@ package uixt +type VideoCrawlerConfigs struct { + Target struct{} +} + func (dExt *DriverExt) VideoCrawler(params map[string]interface{}) error { return nil } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 368546d9..344c15bb 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -36,7 +36,7 @@ func (s *StepMobile) Serial(serial string) *StepMobile { func (s *StepMobile) InstallApp(path string) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.AppInstall, + Method: uixt.ACTION_AppInstall, Params: path, }) return s @@ -44,7 +44,7 @@ func (s *StepMobile) InstallApp(path string) *StepMobile { func (s *StepMobile) AppLaunch(bundleId string) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.AppLaunch, + Method: uixt.ACTION_AppLaunch, Params: bundleId, }) return s @@ -52,7 +52,7 @@ func (s *StepMobile) AppLaunch(bundleId string) *StepMobile { func (s *StepMobile) AppTerminate(bundleId string) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.AppTerminate, + Method: uixt.ACTION_AppTerminate, Params: bundleId, }) return s @@ -275,7 +275,7 @@ func (s *StepMobile) Input(text string, options ...uixt.ActionOption) *StepMobil // Sleep specify sleep seconds after last action func (s *StepMobile) Sleep(n float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.CtlSleep, + Method: uixt.ACTION_Sleep, Params: n, }) return &StepMobile{step: s.step} @@ -287,7 +287,7 @@ func (s *StepMobile) Sleep(n float64) *StepMobile { // 2. [min1, max1, weight1, min2, max2, weight2, ...] : weight is the probability of the time range func (s *StepMobile) SleepRandom(params ...float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.CtlSleepRandom, + Method: uixt.ACTION_SleepRandom, Params: params, }) return &StepMobile{step: s.step} @@ -295,7 +295,7 @@ func (s *StepMobile) SleepRandom(params ...float64) *StepMobile { func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.VideoCrawler, + Method: uixt.ACTION_VideoCrawler, Params: params, }) return &StepMobile{step: s.step} @@ -303,7 +303,7 @@ func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile { func (s *StepMobile) ScreenShot() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.CtlScreenShot, + Method: uixt.ACTION_ScreenShot, Params: nil, }) return &StepMobile{step: s.step} @@ -311,7 +311,7 @@ func (s *StepMobile) ScreenShot() *StepMobile { func (s *StepMobile) StartCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.CtlStartCamera, + Method: uixt.ACTION_StartCamera, Params: nil, }) return &StepMobile{step: s.step} @@ -319,7 +319,7 @@ func (s *StepMobile) StartCamera() *StepMobile { func (s *StepMobile) StopCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.CtlStopCamera, + Method: uixt.ACTION_StopCamera, Params: nil, }) return &StepMobile{step: s.step} From f9a488c7e7dc5d68f351f4541c8539d42b7f0c45 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 14:47:26 +0800 Subject: [PATCH 04/67] refactor: split action to separate file --- hrp/pkg/uixt/action.go | 524 +++++++++++++++++++++++++++++++++++++++++ hrp/pkg/uixt/ext.go | 517 +--------------------------------------- 2 files changed, 527 insertions(+), 514 deletions(-) create mode 100644 hrp/pkg/uixt/action.go diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go new file mode 100644 index 00000000..6ad76c3e --- /dev/null +++ b/hrp/pkg/uixt/action.go @@ -0,0 +1,524 @@ +package uixt + +import ( + "encoding/json" + "fmt" + "math/rand" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" +) + +type ActionMethod string + +const ( + ACTION_AppInstall ActionMethod = "install" + ACTION_AppUninstall ActionMethod = "uninstall" + ACTION_AppStart ActionMethod = "app_start" + ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成 + ACTION_AppTerminate ActionMethod = "app_terminate" + ACTION_AppStop ActionMethod = "app_stop" + ACTION_ScreenShot ActionMethod = "screenshot" + ACTION_Sleep ActionMethod = "sleep" + ACTION_SleepRandom ActionMethod = "sleep_random" + ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera + ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera + + // UI validation + // selectors + SelectorName string = "ui_name" + SelectorLabel string = "ui_label" + SelectorOCR string = "ui_ocr" + SelectorImage string = "ui_image" + SelectorForegroundApp string = "ui_foreground_app" + // assertions + AssertionEqual string = "equal" + AssertionNotEqual string = "not_equal" + AssertionExists string = "exists" + AssertionNotExists string = "not_exists" + + // UI handling + ACTION_Home ActionMethod = "home" + ACTION_TapXY ActionMethod = "tap_xy" + ACTION_TapAbsXY ActionMethod = "tap_abs_xy" + ACTION_TapByOCR ActionMethod = "tap_ocr" + ACTION_TapByCV ActionMethod = "tap_cv" + ACTION_Tap ActionMethod = "tap" + ACTION_DoubleTapXY ActionMethod = "double_tap_xy" + ACTION_DoubleTap ActionMethod = "double_tap" + ACTION_Swipe ActionMethod = "swipe" + ACTION_Input ActionMethod = "input" + ACTION_Back ActionMethod = "back" + + // custom actions + ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap + ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap + ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap + ACTION_VideoCrawler ActionMethod = "video_crawler" +) + +type MobileAction struct { + Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` + Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` + + Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log + MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second + Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action + Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action + Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action + IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found + Text string `json:"text,omitempty" yaml:"text,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` +} + +type ActionOption func(o *MobileAction) + +func WithIdentifier(identifier string) ActionOption { + return func(o *MobileAction) { + o.Identifier = identifier + } +} + +func WithIndex(index int) ActionOption { + return func(o *MobileAction) { + o.Index = index + } +} + +func WithWaitTime(sec float64) ActionOption { + return func(o *MobileAction) { + o.WaitTime = sec + } +} + +func WithDuration(duration float64) ActionOption { + return func(o *MobileAction) { + o.Duration = duration + } +} + +func WithSteps(steps int) ActionOption { + return func(o *MobileAction) { + o.Steps = steps + } +} + +// WithDirection inputs direction (up, down, left, right) +func WithDirection(direction string) ActionOption { + return func(o *MobileAction) { + o.Direction = direction + } +} + +// WithCustomDirection inputs sx, sy, ex, ey +func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { + return func(o *MobileAction) { + o.Direction = []float64{sx, sy, ex, ey} + } +} + +// WithScope inputs area of [(x1,y1), (x2,y2)] +func WithScope(x1, y1, x2, y2 float64) ActionOption { + return func(o *MobileAction) { + o.Scope = []float64{x1, y1, x2, y2} + } +} + +func WithOffset(offsetX, offsetY int) ActionOption { + return func(o *MobileAction) { + o.Offset = []int{offsetX, offsetY} + } +} + +func WithText(text string) ActionOption { + return func(o *MobileAction) { + o.Text = text + } +} + +func WithID(id string) ActionOption { + return func(o *MobileAction) { + o.ID = id + } +} + +func WithDescription(description string) ActionOption { + return func(o *MobileAction) { + o.Description = description + } +} + +func WithMaxRetryTimes(maxRetryTimes int) ActionOption { + return func(o *MobileAction) { + o.MaxRetryTimes = maxRetryTimes + } +} + +func WithTimeout(timeout int) ActionOption { + return func(o *MobileAction) { + o.Timeout = timeout + } +} + +func WithIgnoreNotFoundError(ignoreError bool) ActionOption { + return func(o *MobileAction) { + o.IgnoreNotFoundError = ignoreError + } +} + +func (dExt *DriverExt) DoAction(action MobileAction) error { + log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action") + + switch action.Method { + case ACTION_AppInstall: + // TODO + return errActionNotImplemented + case ACTION_AppLaunch: + if bundleId, ok := action.Params.(string); ok { + return dExt.Driver.AppLaunch(bundleId) + } + return fmt.Errorf("invalid %s params, should be bundleId(string), got %v", + ACTION_AppLaunch, action.Params) + case ACTION_SwipeToTapApp: + if appName, ok := action.Params.(string); ok { + return dExt.swipeToTapApp(appName, action) + } + return fmt.Errorf("invalid %s params, should be app name(string), got %v", + ACTION_SwipeToTapApp, action.Params) + case ACTION_SwipeToTapText: + // TODO: merge to LoopUntil + if text, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + if len(action.Offset) != 2 { + action.Offset = []int{0, 0} + } + + identifierOption := WithDataIdentifier(action.Identifier) + offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) + indexOption := WithDataIndex(action.Index) + scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + + // default to retry 10 times + if action.MaxRetryTimes == 0 { + action.MaxRetryTimes = 10 + } + maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) + waitTimeOption := WithDataWaitTime(action.WaitTime) + + var point PointF + // findTextAction := func(d *DriverExt) error { + // return nil + // } + findTextCondition := func(d *DriverExt) error { + var err error + point, err = d.GetTextXY(text, indexOption, scopeOption) + return err + } + foundTextAction := func(d *DriverExt) error { + // tap text + return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) + } + + if action.Direction != nil { + return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) + } + // swipe until found + return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) + } + return fmt.Errorf("invalid %s params, should be app text(string), got %v", + ACTION_SwipeToTapText, action.Params) + case ACTION_SwipeToTapTexts: + // TODO: merge to LoopUntil + if texts, ok := action.Params.([]interface{}); ok { + var textList []string + for _, t := range texts { + textList = append(textList, t.(string)) + } + action.Params = textList + } + if texts, ok := action.Params.([]string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + if len(action.Offset) != 2 { + action.Offset = []int{0, 0} + } + + identifierOption := WithDataIdentifier(action.Identifier) + offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) + scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + // default to retry 10 times + if action.MaxRetryTimes == 0 { + action.MaxRetryTimes = 10 + } + maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) + waitTimeOption := WithDataWaitTime(action.WaitTime) + + var point PointF + findTexts := func(d *DriverExt) error { + var err error + points, err := d.GetTextXYs(texts, scopeOption) + if err != nil { + return err + } + for _, point = range points { + if point != (PointF{X: 0, Y: 0}) { + return nil + } + } + return errors.New("failed to find text position") + } + foundTextAction := func(d *DriverExt) error { + // tap text + return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) + } + + // default to retry 10 times + if action.MaxRetryTimes == 0 { + action.MaxRetryTimes = 10 + } + + if action.Direction != nil { + return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption) + } + // swipe until found + return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption) + } + return fmt.Errorf("invalid %s params, should be app text([]string), got %v", + ACTION_SwipeToTapText, action.Params) + case ACTION_AppTerminate: + if bundleId, ok := action.Params.(string); ok { + success, err := dExt.Driver.AppTerminate(bundleId) + if err != nil { + return errors.Wrap(err, "failed to terminate app") + } + if !success { + log.Warn().Str("bundleId", bundleId).Msg("app was not running") + } + return nil + } + return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params) + case ACTION_Home: + return dExt.Driver.Homescreen() + case ACTION_TapXY: + if location, ok := action.Params.([]interface{}); ok { + // relative x,y of window size: [0.5, 0.5] + if len(location) != 2 { + return fmt.Errorf("invalid tap location params: %v", location) + } + x, _ := location[0].(float64) + y, _ := location[1].(float64) + return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier)) + } + return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) + case ACTION_TapAbsXY: + if location, ok := action.Params.([]interface{}); ok { + // absolute coordinates x,y of window size: [100, 300] + if len(location) != 2 { + return fmt.Errorf("invalid tap location params: %v", location) + } + x, _ := location[0].(float64) + y, _ := location[1].(float64) + if len(action.Offset) != 2 { + action.Offset = []int{0, 0} + } + return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier), WithDataOffset(action.Offset[0], action.Offset[1])) + } + return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) + case ACTION_Tap: + if param, ok := action.Params.(string); ok { + return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) + } + return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) + case ACTION_TapByOCR: + if ocrText, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + if len(action.Offset) != 2 { + action.Offset = []int{0, 0} + } + + indexOption := WithDataIndex(action.Index) + offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) + scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + identifierOption := WithDataIdentifier(action.Identifier) + IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError) + return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption, offsetOption) + } + return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) + case ACTION_TapByCV: + if imagePath, ok := action.Params.(string); ok { + return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) + } + return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) + case ACTION_DoubleTapXY: + if location, ok := action.Params.([]interface{}); ok { + // relative x,y of window size: [0.5, 0.5] + if len(location) != 2 { + return fmt.Errorf("invalid tap location params: %v", location) + } + x, _ := location[0].(float64) + y, _ := location[1].(float64) + return dExt.DoubleTapXY(x, y) + } + return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params) + case ACTION_DoubleTap: + if param, ok := action.Params.(string); ok { + return dExt.DoubleTap(param) + } + return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) + case ACTION_Swipe: + identifierOption := WithDataIdentifier(action.Identifier) + durationOption := WithDataPressDuration(action.Duration) + if action.Steps == 0 { + action.Steps = 10 + } + stepsOption := WithDataSteps(action.Steps) + if positions, ok := action.Params.([]interface{}); ok { + // relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1] + if len(positions) != 4 { + return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions) + } + fromX, _ := positions[0].(float64) + fromY, _ := positions[1].(float64) + toX, _ := positions[2].(float64) + toY, _ := positions[3].(float64) + return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption) + } + if direction, ok := action.Params.(string); ok { + return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption) + } + return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params) + case ACTION_Input: + // input text on current active element + // append \n to send text with enter + // send \b\b\b to delete 3 chars + param := fmt.Sprintf("%v", action.Params) + options := []DataOption{} + if action.Text != "" { + options = append(options, WithCustomOption("textview", action.Text)) + } + if action.ID != "" { + options = append(options, WithCustomOption("id", action.ID)) + } + if action.Description != "" { + options = append(options, WithCustomOption("description", action.Description)) + } + if action.Identifier != "" { + options = append(options, WithDataIdentifier(action.Identifier)) + } + return dExt.Driver.Input(param, options...) + case ACTION_Back: + return dExt.Driver.PressBack() + case ACTION_Sleep: + if param, ok := action.Params.(json.Number); ok { + seconds, _ := param.Float64() + time.Sleep(time.Duration(seconds*1000) * time.Millisecond) + return nil + } else if param, ok := action.Params.(float64); ok { + time.Sleep(time.Duration(param*1000) * time.Millisecond) + return nil + } else if param, ok := action.Params.(int64); ok { + time.Sleep(time.Duration(param) * time.Second) + return nil + } + return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params) + case ACTION_SleepRandom: + params, ok := action.Params.([]interface{}) + if !ok { + return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) + } + return sleepRandom(params) + case ACTION_ScreenShot: + // take screenshot + log.Info().Msg("take screenshot for current screen") + _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot")) + return err + case ACTION_StartCamera: + return dExt.Driver.StartCamera() + case ACTION_StopCamera: + return dExt.Driver.StopCamera() + case ACTION_VideoCrawler: + params, ok := action.Params.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params) + } + return dExt.VideoCrawler(params) + } + return nil +} + +var errActionNotImplemented = errors.New("UI action not implemented") + +func convertToFloat64(val interface{}) (float64, error) { + switch v := val.(type) { + case float64: + return v, nil + case int: + return float64(v), nil + case int64: + return float64(v), nil + default: + return 0, fmt.Errorf("invalid type for conversion to float64: %T, value: %+v", val, val) + } +} + +func sleepRandom(params []interface{}) error { + // append default weight 1 + if len(params) == 2 { + params = append(params, 1.0) + } + + var sections []struct { + min, max, weight float64 + } + totalProb := 0.0 + for i := 0; i+3 <= len(params); i += 3 { + min, err := convertToFloat64(params[i]) + if err != nil { + return errors.Wrapf(err, "invalid minimum time: %v", params[i]) + } + max, err := convertToFloat64(params[i+1]) + if err != nil { + return errors.Wrapf(err, "invalid maximum time: %v", params[i+1]) + } + weight, err := convertToFloat64(params[i+2]) + if err != nil { + return errors.Wrapf(err, "invalid weight value: %v", params[i+2]) + } + totalProb += weight + sections = append(sections, + struct{ min, max, weight float64 }{min, max, weight}, + ) + } + + if totalProb == 0 { + log.Warn().Msg("total weight is 0, skip sleep") + return nil + } + + r := rand.Float64() + accProb := 0.0 + for _, s := range sections { + accProb += s.weight / totalProb + if r < accProb { + n := s.min + rand.Float64()*(s.max-s.min) + log.Info().Float64("duration", n).Msg("sleep random seconds") + time.Sleep(time.Duration(n*1000) * time.Millisecond) + return nil + } + } + return nil +} diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 4f1c2ed5..6f4af2aa 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -2,7 +2,6 @@ package uixt import ( "bytes" - "encoding/json" "fmt" "image" "image/gif" @@ -17,175 +16,13 @@ import ( "strings" "time" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/env" "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/env" ) -type ActionMethod string - -const ( - ACTION_AppInstall ActionMethod = "install" - ACTION_AppUninstall ActionMethod = "uninstall" - ACTION_AppStart ActionMethod = "app_start" - ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成 - ACTION_AppTerminate ActionMethod = "app_terminate" - ACTION_AppStop ActionMethod = "app_stop" - ACTION_ScreenShot ActionMethod = "screenshot" - ACTION_Sleep ActionMethod = "sleep" - ACTION_SleepRandom ActionMethod = "sleep_random" - ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera - ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera - - // UI validation - // selectors - SelectorName string = "ui_name" - SelectorLabel string = "ui_label" - SelectorOCR string = "ui_ocr" - SelectorImage string = "ui_image" - SelectorForegroundApp string = "ui_foreground_app" - // assertions - AssertionEqual string = "equal" - AssertionNotEqual string = "not_equal" - AssertionExists string = "exists" - AssertionNotExists string = "not_exists" - - // UI handling - ACTION_Home ActionMethod = "home" - ACTION_TapXY ActionMethod = "tap_xy" - ACTION_TapAbsXY ActionMethod = "tap_abs_xy" - ACTION_TapByOCR ActionMethod = "tap_ocr" - ACTION_TapByCV ActionMethod = "tap_cv" - ACTION_Tap ActionMethod = "tap" - ACTION_DoubleTapXY ActionMethod = "double_tap_xy" - ACTION_DoubleTap ActionMethod = "double_tap" - ACTION_Swipe ActionMethod = "swipe" - ACTION_Input ActionMethod = "input" - ACTION_Back ActionMethod = "back" - - // custom actions - ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap - ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap - ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap - ACTION_VideoCrawler ActionMethod = "video_crawler" -) - -type MobileAction struct { - Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` - Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` - - Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log - MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times - WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second - Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action - Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action - Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action - IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` -} - -type ActionOption func(o *MobileAction) - -func WithIdentifier(identifier string) ActionOption { - return func(o *MobileAction) { - o.Identifier = identifier - } -} - -func WithIndex(index int) ActionOption { - return func(o *MobileAction) { - o.Index = index - } -} - -func WithWaitTime(sec float64) ActionOption { - return func(o *MobileAction) { - o.WaitTime = sec - } -} - -func WithDuration(duration float64) ActionOption { - return func(o *MobileAction) { - o.Duration = duration - } -} - -func WithSteps(steps int) ActionOption { - return func(o *MobileAction) { - o.Steps = steps - } -} - -// WithDirection inputs direction (up, down, left, right) -func WithDirection(direction string) ActionOption { - return func(o *MobileAction) { - o.Direction = direction - } -} - -// WithCustomDirection inputs sx, sy, ex, ey -func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { - return func(o *MobileAction) { - o.Direction = []float64{sx, sy, ex, ey} - } -} - -// WithScope inputs area of [(x1,y1), (x2,y2)] -func WithScope(x1, y1, x2, y2 float64) ActionOption { - return func(o *MobileAction) { - o.Scope = []float64{x1, y1, x2, y2} - } -} - -func WithOffset(offsetX, offsetY int) ActionOption { - return func(o *MobileAction) { - o.Offset = []int{offsetX, offsetY} - } -} - -func WithText(text string) ActionOption { - return func(o *MobileAction) { - o.Text = text - } -} - -func WithID(id string) ActionOption { - return func(o *MobileAction) { - o.ID = id - } -} - -func WithDescription(description string) ActionOption { - return func(o *MobileAction) { - o.Description = description - } -} - -func WithMaxRetryTimes(maxRetryTimes int) ActionOption { - return func(o *MobileAction) { - o.MaxRetryTimes = maxRetryTimes - } -} - -func WithTimeout(timeout int) ActionOption { - return func(o *MobileAction) { - o.Timeout = timeout - } -} - -func WithIgnoreNotFoundError(ignoreError bool) ActionOption { - return func(o *MobileAction) { - o.IgnoreNotFoundError = ignoreError - } -} - // TemplateMatchMode is the type of the template matching operation. type TemplateMatchMode int @@ -376,354 +213,6 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool { return true } -var errActionNotImplemented = errors.New("UI action not implemented") - -func convertToFloat64(val interface{}) (float64, error) { - switch v := val.(type) { - case float64: - return v, nil - case int: - return float64(v), nil - case int64: - return float64(v), nil - default: - return 0, fmt.Errorf("invalid type for conversion to float64: %T, value: %+v", val, val) - } -} - -func (dExt *DriverExt) DoAction(action MobileAction) error { - log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action") - - switch action.Method { - case ACTION_AppInstall: - // TODO - return errActionNotImplemented - case ACTION_AppLaunch: - if bundleId, ok := action.Params.(string); ok { - return dExt.Driver.AppLaunch(bundleId) - } - return fmt.Errorf("invalid %s params, should be bundleId(string), got %v", - ACTION_AppLaunch, action.Params) - case ACTION_SwipeToTapApp: - if appName, ok := action.Params.(string); ok { - return dExt.swipeToTapApp(appName, action) - } - return fmt.Errorf("invalid %s params, should be app name(string), got %v", - ACTION_SwipeToTapApp, action.Params) - case ACTION_SwipeToTapText: - // TODO: merge to LoopUntil - if text, ok := action.Params.(string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - identifierOption := WithDataIdentifier(action.Identifier) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - indexOption := WithDataIndex(action.Index) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) - waitTimeOption := WithDataWaitTime(action.WaitTime) - - var point PointF - // findTextAction := func(d *DriverExt) error { - // return nil - // } - findTextCondition := func(d *DriverExt) error { - var err error - point, err = d.GetTextXY(text, indexOption, scopeOption) - return err - } - foundTextAction := func(d *DriverExt) error { - // tap text - return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) - } - - if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) - } - // swipe until found - return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) - } - return fmt.Errorf("invalid %s params, should be app text(string), got %v", - ACTION_SwipeToTapText, action.Params) - case ACTION_SwipeToTapTexts: - // TODO: merge to LoopUntil - if texts, ok := action.Params.([]interface{}); ok { - var textList []string - for _, t := range texts { - textList = append(textList, t.(string)) - } - action.Params = textList - } - if texts, ok := action.Params.([]string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - identifierOption := WithDataIdentifier(action.Identifier) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) - waitTimeOption := WithDataWaitTime(action.WaitTime) - - var point PointF - findTexts := func(d *DriverExt) error { - var err error - points, err := d.GetTextXYs(texts, scopeOption) - if err != nil { - return err - } - for _, point = range points { - if point != (PointF{X: 0, Y: 0}) { - return nil - } - } - return errors.New("failed to find text position") - } - foundTextAction := func(d *DriverExt) error { - // tap text - return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) - } - - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - - if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption) - } - // swipe until found - return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption) - } - return fmt.Errorf("invalid %s params, should be app text([]string), got %v", - ACTION_SwipeToTapText, action.Params) - case ACTION_AppTerminate: - if bundleId, ok := action.Params.(string); ok { - success, err := dExt.Driver.AppTerminate(bundleId) - if err != nil { - return errors.Wrap(err, "failed to terminate app") - } - if !success { - log.Warn().Str("bundleId", bundleId).Msg("app was not running") - } - return nil - } - return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params) - case ACTION_Home: - return dExt.Driver.Homescreen() - case ACTION_TapXY: - if location, ok := action.Params.([]interface{}); ok { - // relative x,y of window size: [0.5, 0.5] - if len(location) != 2 { - return fmt.Errorf("invalid tap location params: %v", location) - } - x, _ := location[0].(float64) - y, _ := location[1].(float64) - return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier)) - } - return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) - case ACTION_TapAbsXY: - if location, ok := action.Params.([]interface{}); ok { - // absolute coordinates x,y of window size: [100, 300] - if len(location) != 2 { - return fmt.Errorf("invalid tap location params: %v", location) - } - x, _ := location[0].(float64) - y, _ := location[1].(float64) - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier), WithDataOffset(action.Offset[0], action.Offset[1])) - } - return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) - case ACTION_Tap: - if param, ok := action.Params.(string); ok { - return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) - } - return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) - case ACTION_TapByOCR: - if ocrText, ok := action.Params.(string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - indexOption := WithDataIndex(action.Index) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - identifierOption := WithDataIdentifier(action.Identifier) - IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError) - return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption, offsetOption) - } - return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) - case ACTION_TapByCV: - if imagePath, ok := action.Params.(string); ok { - return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) - } - return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) - case ACTION_DoubleTapXY: - if location, ok := action.Params.([]interface{}); ok { - // relative x,y of window size: [0.5, 0.5] - if len(location) != 2 { - return fmt.Errorf("invalid tap location params: %v", location) - } - x, _ := location[0].(float64) - y, _ := location[1].(float64) - return dExt.DoubleTapXY(x, y) - } - return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params) - case ACTION_DoubleTap: - if param, ok := action.Params.(string); ok { - return dExt.DoubleTap(param) - } - return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) - case ACTION_Swipe: - identifierOption := WithDataIdentifier(action.Identifier) - durationOption := WithDataPressDuration(action.Duration) - if action.Steps == 0 { - action.Steps = 10 - } - stepsOption := WithDataSteps(action.Steps) - if positions, ok := action.Params.([]interface{}); ok { - // relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1] - if len(positions) != 4 { - return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions) - } - fromX, _ := positions[0].(float64) - fromY, _ := positions[1].(float64) - toX, _ := positions[2].(float64) - toY, _ := positions[3].(float64) - return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption) - } - if direction, ok := action.Params.(string); ok { - return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption) - } - return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params) - case ACTION_Input: - // input text on current active element - // append \n to send text with enter - // send \b\b\b to delete 3 chars - param := fmt.Sprintf("%v", action.Params) - options := []DataOption{} - if action.Text != "" { - options = append(options, WithCustomOption("textview", action.Text)) - } - if action.ID != "" { - options = append(options, WithCustomOption("id", action.ID)) - } - if action.Description != "" { - options = append(options, WithCustomOption("description", action.Description)) - } - if action.Identifier != "" { - options = append(options, WithDataIdentifier(action.Identifier)) - } - return dExt.Driver.Input(param, options...) - case ACTION_Back: - return dExt.Driver.PressBack() - case ACTION_Sleep: - if param, ok := action.Params.(json.Number); ok { - seconds, _ := param.Float64() - time.Sleep(time.Duration(seconds*1000) * time.Millisecond) - return nil - } else if param, ok := action.Params.(float64); ok { - time.Sleep(time.Duration(param*1000) * time.Millisecond) - return nil - } else if param, ok := action.Params.(int64); ok { - time.Sleep(time.Duration(param) * time.Second) - return nil - } - return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params) - case ACTION_SleepRandom: - params, ok := action.Params.([]interface{}) - if !ok { - return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) - } - return sleepRandom(params) - case ACTION_ScreenShot: - // take screenshot - log.Info().Msg("take screenshot for current screen") - _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot")) - return err - case ACTION_StartCamera: - return dExt.Driver.StartCamera() - case ACTION_StopCamera: - return dExt.Driver.StopCamera() - case ACTION_VideoCrawler: - params, ok := action.Params.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params) - } - return dExt.VideoCrawler(params) - } - return nil -} - -func sleepRandom(params []interface{}) error { - // append default weight 1 - if len(params) == 2 { - params = append(params, 1.0) - } - - var sections []struct { - min, max, weight float64 - } - totalProb := 0.0 - for i := 0; i+3 <= len(params); i += 3 { - min, err := convertToFloat64(params[i]) - if err != nil { - return errors.Wrapf(err, "invalid minimum time: %v", params[i]) - } - max, err := convertToFloat64(params[i+1]) - if err != nil { - return errors.Wrapf(err, "invalid maximum time: %v", params[i+1]) - } - weight, err := convertToFloat64(params[i+2]) - if err != nil { - return errors.Wrapf(err, "invalid weight value: %v", params[i+2]) - } - totalProb += weight - sections = append(sections, - struct{ min, max, weight float64 }{min, max, weight}, - ) - } - - if totalProb == 0 { - log.Warn().Msg("total weight is 0, skip sleep") - return nil - } - - r := rand.Float64() - accProb := 0.0 - for _, s := range sections { - accProb += s.weight / totalProb - if r < accProb { - n := s.min + rand.Float64()*(s.max-s.min) - log.Info().Float64("duration", n).Msg("sleep random seconds") - time.Sleep(time.Duration(n*1000) * time.Millisecond) - return nil - } - } - return nil -} - func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { return int(x1 * float64(dExt.windowSize.Width) * dExt.scale), int(y1 * float64(dExt.windowSize.Height) * dExt.scale), From f9bebf7202b499d072bfe8a8c90e6a578b88c6eb Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 15:51:38 +0800 Subject: [PATCH 05/67] feat: add code MobileUILaunchAppError --- hrp/internal/code/code.go | 2 ++ hrp/pkg/uixt/android_adb_driver.go | 6 ++++-- hrp/pkg/uixt/ios_driver.go | 14 ++++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 3479c8b6..71f9226c 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -68,6 +68,7 @@ var ( // UI automation related: [70, 80) var ( MobileUIDriverError = errors.New("mobile UI driver error") // 70 + MobileUILaunchAppError = errors.New("mobile UI launch app error") // 71 MobileUIValidationError = errors.New("mobile UI validation error") // 75 MobileUIAppNotInForegroundError = errors.New("mobile UI app not in foreground error") // 76 ) @@ -125,6 +126,7 @@ var errorsMap = map[error]int{ // UI automation related MobileUIDriverError: 70, + MobileUILaunchAppError: 71, MobileUIValidationError: 75, MobileUIAppNotInForegroundError: 76, diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 68f8c28b..388c444e 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -160,10 +160,12 @@ func (ad *adbDriver) AppLaunch(packageName string) (err error) { "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1", ) if err != nil { - return err + return errors.Wrap(code.MobileUILaunchAppError, + fmt.Sprintf("monkey launch failed: %v", err)) } if strings.Contains(sOutput, "monkey aborted") { - return fmt.Errorf("app launch: %s", strings.TrimSpace(sOutput)) + return errors.Wrap(code.MobileUILaunchAppError, + fmt.Sprintf("monkey aborted: %s", strings.TrimSpace(sOutput))) } ad.lastLaunchedPackageName = packageName return nil diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 33bbe0ed..830d78cc 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -308,17 +308,23 @@ func (wd *wdaDriver) AppLaunch(bundleId string) (err error) { data := make(map[string]interface{}) data["bundleId"] = bundleId _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/launch") - if err == nil { - wd.lastLaunchedPackageName = bundleId + if err != nil { + return errors.Wrap(code.MobileUILaunchAppError, + fmt.Sprintf("wda launch failed: %v", err)) } - return + wd.lastLaunchedPackageName = bundleId + return nil } func (wd *wdaDriver) AppLaunchUnattached(bundleId string) (err error) { // [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)] data := map[string]interface{}{"bundleId": bundleId} _, err = wd.httpPOST(data, "/wda/apps/launchUnattached") - return + if err != nil { + return errors.Wrap(code.MobileUILaunchAppError, + fmt.Sprintf("wda launchUnattached failed: %v", err)) + } + return nil } func (wd *wdaDriver) AppTerminate(bundleId string) (successful bool, err error) { From 3ec27b9afc6f93ee5378c905188156be3147fab8 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 25 Apr 2023 22:03:55 +0800 Subject: [PATCH 06/67] refactor: simplify OCR APIs --- examples/worldcup/main.go | 8 +- hrp/pkg/uixt/action.go | 15 +- hrp/pkg/uixt/drag.go | 10 +- hrp/pkg/uixt/ext.go | 14 +- hrp/pkg/uixt/ocr_test.go | 6 +- hrp/pkg/uixt/ocr_vedem.go | 272 ++++++++++++----------------- hrp/pkg/uixt/opencv.go | 16 +- hrp/pkg/uixt/opencv_off.go | 2 +- hrp/pkg/uixt/swipe.go | 2 +- hrp/pkg/uixt/swipe_test.go | 4 +- hrp/pkg/uixt/tap.go | 59 +------ hrp/pkg/uixt/video_crawler.go | 47 ++++- hrp/pkg/uixt/video_crawler_test.go | 23 +++ 13 files changed, 231 insertions(+), 247 deletions(-) create mode 100644 hrp/pkg/uixt/video_crawler_test.go diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index 697bcd57..fc84d996 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { utcTimeStr := utcTime.Format("15:04:05") - ocrTexts, err := wc.driver.GetTextsByOCR() + ocrTexts, err := wc.driver.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("get ocr texts failed") return err @@ -212,8 +212,10 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { time.Sleep(5 * time.Second) // 青少年弹窗处理 - if points, err := wc.driver.GetTextXYs([]string{"青少年模式", "我知道了"}); err == nil { - _ = wc.driver.TapAbsXY(points[1].X, points[1].Y) + if ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { + if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { + _ = wc.driver.TapAbsXY(points[1].X, points[1].Y) + } } // 进入世界杯 tab diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 6ad76c3e..942610f6 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -222,7 +222,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { // } findTextCondition := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(text, indexOption, scopeOption) + point, err = d.FindScreenTextByOCR(text, indexOption, scopeOption) return err } foundTextAction := func(d *DriverExt) error { @@ -268,7 +268,11 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findTexts := func(d *DriverExt) error { var err error - points, err := d.GetTextXYs(texts, scopeOption) + ocrTexts, err := d.GetScreenTextsByOCR(scopeOption) + if err != nil { + return err + } + points, err := ocrTexts.FindTexts(texts, scopeOption) if err != nil { return err } @@ -455,7 +459,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { if !ok { return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params) } - return dExt.VideoCrawler(params) + data, _ := json.Marshal(params) + configs := &VideoCrawlerConfigs{} + if err := json.Unmarshal(data, configs); err != nil { + return errors.Wrapf(err, "invalid video crawler params: %v(%T)", action.Params, action.Params) + } + return dExt.VideoCrawler(configs) } return nil } diff --git a/hrp/pkg/uixt/drag.go b/hrp/pkg/uixt/drag.go index 31d33b1d..f583f9cf 100644 --- a/hrp/pkg/uixt/drag.go +++ b/hrp/pkg/uixt/drag.go @@ -17,14 +17,12 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs pressForDuration = []float64{1.0} } - var x, y, width, height float64 - if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + point, err := dExt.FindUIRectInUIKit(pathname) + if err != nil { return err } - fromX := x + width*xOffset - fromY := y + height*yOffset - - return dExt.Driver.DragFloat(fromX, fromY, toX, toY, + // FIXME: handle offset + return dExt.Driver.DragFloat(point.X+xOffset, point.Y+yOffset, toX, toY, WithDataPressDuration(pressForDuration[0])) } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 6f4af2aa..ce1aeb24 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -52,8 +52,8 @@ type DriverExt struct { frame *bytes.Buffer doneMjpegStream chan bool scale float64 - ocrService OCRService // used to get text from image - screenShots []string // cache screenshot paths + OCRService IOCRService // used to get text from image + screenShots []string // cache screenshot paths CVArgs } @@ -75,7 +75,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, err } - if dExt.ocrService, err = newVEDEMOCRService(); err != nil { + if dExt.OCRService, err = newVEDEMOCRService(); err != nil { return nil, err } @@ -178,10 +178,10 @@ func init() { rand.Seed(time.Now().UnixNano()) } -func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (point PointF, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search, options...) + return dExt.FindScreenTextByOCR(search, options...) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, options...) @@ -194,12 +194,12 @@ func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, _, _, _, err := dExt.FindTextByOCR(text) + _, err := dExt.FindScreenTextByOCR(text) return err == nil } func (dExt *DriverExt) IsImageExist(text string) bool { - _, _, _, _, err := dExt.FindImageRectInUIKit(text) + _, err := dExt.FindImageRectInUIKit(text) return err == nil } diff --git a/hrp/pkg/uixt/ocr_test.go b/hrp/pkg/uixt/ocr_test.go index da868c27..053ccaee 100644 --- a/hrp/pkg/uixt/ocr_test.go +++ b/hrp/pkg/uixt/ocr_test.go @@ -10,9 +10,9 @@ func TestDriverExtOCR(t *testing.T) { driverExt, err := iosDevice.NewDriver(nil) checkErr(t, err) - x, y, width, height, err := driverExt.FindTextByOCR("抖音") + point, err := driverExt.FindScreenTextByOCR("抖音") checkErr(t, err) - t.Logf("x: %v, y: %v, width: %v, height: %v", x, y, width, height) - driverExt.Driver.TapFloat(x+width*0.5, y+height*0.5-20) + t.Logf("point.X: %v, point.Y: %v", point.X, point.Y) + driverExt.Driver.TapFloat(point.X, point.Y-20) } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 15db742d..db230b6d 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -34,7 +34,85 @@ type ResponseOCR struct { OCRResult []OCRResult `json:"ocrResult"` } -type veDEMOCRService struct{} +type OCRText struct { + Text string + Rect image.Rectangle +} + +type OCRTexts []OCRText + +func (t OCRTexts) texts() (texts []string) { + for _, text := range t { + texts = append(texts, text.Text) + } + return texts +} + +func (t OCRTexts) FindText(text string, options ...DataOption) ( + point PointF, err error) { + + dataOptions := NewDataOptions(options...) + + var rects []image.Rectangle + for _, ocrText := range t { + rect := ocrText.Rect + + // not contains text + if !strings.Contains(ocrText.Text, text) { + continue + } + + rects = append(rects, rect) + + // contains text while not match exactly + if ocrText.Text != text { + continue + } + + // match exactly, and not specify index, return the first one + if dataOptions.Index == 0 { + return getRectangleCenterPoint(rect), nil + } + } + + if len(rects) == 0 { + return PointF{}, errors.Wrap(code.OCRTextNotFoundError, + fmt.Sprintf("text %s not found in %v", text, t.texts())) + } + + // get index + idx := dataOptions.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return PointF{}, errors.Wrap(code.OCRTextNotFoundError, + fmt.Sprintf("text %s found %d, index %d out of range", text, len(rects), idx)) + } + + return getRectangleCenterPoint(rects[idx]), nil +} + +func (t OCRTexts) FindTexts(texts []string, options ...DataOption) (points []PointF, err error) { + for _, text := range texts { + point, err := t.FindText(text, options...) + if err != nil { + continue + } + points = append(points, point) + } + + if len(points) != len(texts) { + return nil, errors.Wrap(code.OCRTextNotFoundError, + fmt.Sprintf("texts %s not found in %v", texts, t.texts())) + } + return points, nil +} func newVEDEMOCRService() (*veDEMOCRService, error) { if err := checkEnv(); err != nil { @@ -43,18 +121,7 @@ func newVEDEMOCRService() (*veDEMOCRService, error) { return &veDEMOCRService{}, nil } -func checkEnv() error { - if env.VEDEM_OCR_URL == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_URL missed") - } - if env.VEDEM_OCR_AK == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_AK missed") - } - if env.VEDEM_OCR_SK == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_SK missed") - } - return nil -} +type veDEMOCRService struct{} func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, error) { bodyBuf := &bytes.Buffer{} @@ -138,32 +205,6 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err return ocrResult.OCRResult, nil } -func getLogID(header http.Header) string { - if len(header) == 0 { - return "" - } - - logID, ok := header["X-Tt-Logid"] - if !ok || len(logID) == 0 { - return "" - } - return logID[0] -} - -type OCRText struct { - Text string - Rect image.Rectangle -} - -type OCRTexts []OCRText - -func (t OCRTexts) Texts() (texts []string) { - for _, text := range t { - texts = append(texts, text.Text) - } - return texts -} - func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption) ( ocrTexts OCRTexts, err error) { @@ -203,157 +244,74 @@ func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption return } -func (s *veDEMOCRService) FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) ( - rect image.Rectangle, err error) { - - ocrTexts, err := s.GetTexts(imageBuf, options...) - if err != nil { - log.Error().Err(err).Msg("GetTexts failed") - return +func checkEnv() error { + if env.VEDEM_OCR_URL == "" { + return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_URL missed") } - - dataOptions := NewDataOptions(options...) - - var rects []image.Rectangle - for _, ocrText := range ocrTexts { - rect = ocrText.Rect - - // not contains text - if !strings.Contains(ocrText.Text, text) { - continue - } - - rects = append(rects, rect) - - // contains text while not match exactly - if ocrText.Text != text { - continue - } - - // match exactly, and not specify index, return the first one - if dataOptions.Index == 0 { - return rect, nil - } + if env.VEDEM_OCR_AK == "" { + return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_AK missed") } - - if len(rects) == 0 { - return image.Rectangle{}, errors.Wrap(code.OCRTextNotFoundError, - fmt.Sprintf("text %s not found in %v", text, ocrTexts.Texts())) + if env.VEDEM_OCR_SK == "" { + return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_SK missed") } - - // get index - idx := dataOptions.Index - if idx > 0 { - // NOTICE: index start from 1 - idx = idx - 1 - } else if idx < 0 { - idx = len(rects) + idx - } - - // index out of range - if idx >= len(rects) { - return image.Rectangle{}, errors.Wrap(code.OCRTextNotFoundError, - fmt.Sprintf("text %s found %d, index %d out of range", text, len(rects), idx)) - } - - return rects[idx], nil + return nil } -func (s *veDEMOCRService) FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) ( - rects []image.Rectangle, err error) { - - ocrTexts, err := s.GetTexts(imageBuf, options...) - if err != nil { - log.Error().Err(err).Msg("GetTexts failed") - return +func getLogID(header http.Header) string { + if len(header) == 0 { + return "" } - var success bool - for _, text := range texts { - var found bool - for _, ocrText := range ocrTexts { - rect := ocrText.Rect - - // not contains text - if !strings.Contains(ocrText.Text, text) { - continue - } - - found = true - rects = append(rects, rect) - break - } - if !found { - rects = append(rects, image.Rectangle{}) - } - success = found || success + logID, ok := header["X-Tt-Logid"] + if !ok || len(logID) == 0 { + return "" } - - if !success { - return rects, errors.Wrap(code.OCRTextNotFoundError, - fmt.Sprintf("texts %s not found in %v", texts, ocrTexts.Texts())) - } - - return rects, nil + return logID[0] } -type OCRService interface { +type IOCRService interface { GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error) - FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) (rect image.Rectangle, err error) - FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) (rects []image.Rectangle, err error) } -func (dExt *DriverExt) GetTextsByOCR(options ...DataOption) (texts OCRTexts, err error) { +func (dExt *DriverExt) GetScreenTextsByOCR(options ...DataOption) (texts OCRTexts, err error) { var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { + if bufSource, err = dExt.TakeScreenShot( + builtin.GenNameWithTimestamp("screenshot_%d_ocr")); err != nil { return } - ocrTexts, err := dExt.ocrService.GetTexts(bufSource, options...) + ocrTexts, err := dExt.OCRService.GetTexts(bufSource, options...) if err != nil { - log.Error().Err(err).Msg("GetTexts failed") + log.Error().Err(err).Msg("GetScreenTextsByOCR failed") return } + log.Debug().Interface("texts", ocrTexts).Msg("get screen texts by OCR") return ocrTexts, nil } -func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { - var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { +func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...DataOption) (point PointF, err error) { + ocrTexts, err := dExt.GetScreenTextsByOCR(options...) + if err != nil { return } - - rect, err := dExt.ocrService.FindText(ocrText, bufSource, options...) + point, err = ocrTexts.FindText(text, options...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return } - log.Info().Str("ocrText", ocrText). - Interface("rect", rect).Msgf("FindTextByOCR success") - x, y, width, height = dExt.MappingToRectInUIKit(rect) + log.Info().Str("text", text). + Interface("point", point).Msgf("FindScreenTextByOCR success") return } -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) { - var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { - return +func getRectangleCenterPoint(rect image.Rectangle) (point PointF) { + x, y := float64(rect.Min.X), float64(rect.Min.Y) + width, height := float64(rect.Dx()), float64(rect.Dy()) + point = PointF{ + X: x + width*0.5, + Y: y + height*0.5, } - - rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource, options...) - if err != nil { - log.Warn().Msgf("FindTexts failed: %s", err.Error()) - return - } - - log.Info().Interface("ocrTexts", ocrTexts). - Interface("rects", rects).Msgf("FindTextsByOCR success") - for _, rect := range rects { - x, y, width, height := dExt.MappingToRectInUIKit(rect) - points = append(points, []float64{x, y, width, height}) - } - - return + return point } diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index 98f7286b..635e18e0 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -113,25 +113,19 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (point PointF, err error) { var bufSource, bufSearch *bytes.Buffer if bufSearch, err = getBufFromDisk(imagePath); err != nil { - return 0, 0, 0, 0, err - } - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil { - return 0, 0, 0, 0, err + return PointF{}, err } var rect image.Rectangle if rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil { - return 0, 0, 0, 0, err + return PointF{}, err } - // if rect, err = dExt.findImgRect(search); err != nil { - // return 0, 0, 0, 0, err - // } - x, y, width, height = dExt.MappingToRectInUIKit(rect) - return + point = getRectangleCenterPoint(rect) + return point, nil } func getBufFromDisk(name string) (*bytes.Buffer, error) { diff --git a/hrp/pkg/uixt/opencv_off.go b/hrp/pkg/uixt/opencv_off.go index 445c4350..394f9666 100644 --- a/hrp/pkg/uixt/opencv_off.go +++ b/hrp/pkg/uixt/opencv_off.go @@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (point PointF, err error) { log.Fatal().Msg("opencv is not supported") return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index ddb919fb..96f16149 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -151,7 +151,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error } findAppCondition := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(appName, scopeOption, indexOption) + point, err = d.FindScreenTextByOCR(appName, scopeOption, indexOption) return err } foundAppAction := func(d *DriverExt) error { diff --git a/hrp/pkg/uixt/swipe_test.go b/hrp/pkg/uixt/swipe_test.go index 22950c44..0828d934 100644 --- a/hrp/pkg/uixt/swipe_test.go +++ b/hrp/pkg/uixt/swipe_test.go @@ -13,7 +13,7 @@ func TestSwipeUntil(t *testing.T) { var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextXY("抖音") + point, err = d.FindScreenTextByOCR("抖音") return err } foundAppAction := func(d *DriverExt) error { @@ -34,7 +34,7 @@ func TestSwipeUntil(t *testing.T) { findLive := func(d *DriverExt) error { var err error - point, err = d.GetTextXY("点击进入直播间") + point, err = d.FindScreenTextByOCR("点击进入直播间") return err } foundLiveAction := func(d *DriverExt) error { diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 5e5cdb70..d5d80cbe 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -21,53 +21,10 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error { return dExt.TapAbsXY(x, y, options...) } -func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) { - x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...) - if err != nil { - return PointF{}, err - } - - point = PointF{ - X: x + width*0.5, - Y: y + height*0.5, - } - return point, nil -} - -func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) { - ps, err := dExt.FindTextsByOCR(ocrText, options...) - if err != nil { - return nil, err - } - - for _, point := range ps { - pointF := PointF{ - X: point[0] + point[2]*0.5, - Y: point[1] + point[3]*0.5, - } - points = append(points, pointF) - } - - return points, nil -} - -func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) { - x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...) - if err != nil { - return PointF{}, err - } - - point = PointF{ - X: x + width*0.5, - Y: y + height*0.5, - } - return point, nil -} - func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { dataOptions := NewDataOptions(options...) - point, err := dExt.GetTextXY(ocrText, options...) + point, err := dExt.FindScreenTextByOCR(ocrText, options...) if err != nil { if dataOptions.IgnoreNotFoundError { return nil @@ -81,7 +38,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { dataOptions := NewDataOptions(options...) - point, err := dExt.GetImageXY(imagePath, options...) + point, err := dExt.FindImageRectInUIKit(imagePath, options...) if err != nil { if dataOptions.IgnoreNotFoundError { return nil @@ -99,7 +56,7 @@ func (dExt *DriverExt) Tap(param string, options ...DataOption) error { func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...DataOption) (err error) { dataOptions := NewDataOptions(options...) - x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...) + point, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { if dataOptions.IgnoreNotFoundError { return nil @@ -107,7 +64,8 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options return err } - return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, options...) + // FIXME: handle offset + return dExt.TapAbsXY(point.X+xOffset, point.Y+yOffset, options...) } func (dExt *DriverExt) DoubleTapXY(x, y float64) error { @@ -126,10 +84,11 @@ func (dExt *DriverExt) DoubleTap(param string) (err error) { } func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) { - var x, y, width, height float64 - if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { + point, err := dExt.FindUIRectInUIKit(param) + if err != nil { return err } - return dExt.Driver.DoubleTapFloat(x+width*xOffset, y+height*yOffset) + // FIXME: handle offset + return dExt.Driver.DoubleTapFloat(point.X+xOffset, point.Y+yOffset) } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 1cd757b0..ad1644cd 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -1,9 +1,50 @@ package uixt +import ( + "time" + + "github.com/rs/zerolog/log" +) + type VideoCrawlerConfigs struct { - Target struct{} + AppPackageName string `json:"app_package_name"` + + TargetFeedCount int `json:"target_feed_count"` + TargetLiveCount int `json:"target_live_count"` } -func (dExt *DriverExt) VideoCrawler(params map[string]interface{}) error { - return nil +func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { + // launch app + if configs.AppPackageName != "" { + if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { + return err + } + time.Sleep(5 * time.Second) + } + + // loop until target count achieved + for { + // take screenshot and get screen texts by OCR + _, err := dExt.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + return err + } + + // TODO: check if text popup exists + + // TODO: check if live video + + // assert feed video type + + // swipe to next video + if err = dExt.SwipeUp(); err != nil { + log.Error().Err(err).Msg("swipe up failed") + return err + } + + time.Sleep(5 * time.Second) + } + + // return nil } diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go new file mode 100644 index 00000000..78edb206 --- /dev/null +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -0,0 +1,23 @@ +//go:build localtest + +package uixt + +import "testing" + +func TestVideoCrawler(t *testing.T) { + device, err := NewAndroidDevice() + if err != nil { + t.Fatal(err) + } + driver, err := device.NewDriver(nil) + if err != nil { + t.Fatal(err) + } + configs := &VideoCrawlerConfigs{ + AppPackageName: "com.ss.android.ugc.aweme", + } + err = driver.VideoCrawler(configs) + if err != nil { + t.Fatal(err) + } +} From fcab5ece587a50fa24f059ae37cff3ca5d7e9dee Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 26 Apr 2023 21:46:24 +0800 Subject: [PATCH 07/67] refactor: OCRService.GetTexts remove options --- hrp/pkg/uixt/action.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 942610f6..d5f12c19 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -268,7 +268,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findTexts := func(d *DriverExt) error { var err error - ocrTexts, err := d.GetScreenTextsByOCR(scopeOption) + ocrTexts, err := d.GetScreenTextsByOCR() if err != nil { return err } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index db230b6d..0b1ca58e 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -62,6 +62,13 @@ func (t OCRTexts) FindText(text string, options ...DataOption) ( continue } + // check if text in scope + if rect.Min.X < dataOptions.Scope[0] || rect.Max.X > dataOptions.Scope[2] || + rect.Min.Y < dataOptions.Scope[1] || rect.Max.Y > dataOptions.Scope[3] { + // not in scope + continue + } + rects = append(rects, rect) // contains text while not match exactly @@ -121,6 +128,7 @@ func newVEDEMOCRService() (*veDEMOCRService, error) { return &veDEMOCRService{}, nil } +// veDEMOCRService implements IOCRService interface type veDEMOCRService struct{} func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, error) { @@ -205,7 +213,7 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err return ocrResult.OCRResult, nil } -func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption) ( +func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer) ( ocrTexts OCRTexts, err error) { ocrResults, err := s.getOCRResult(imageBuf) @@ -214,8 +222,6 @@ func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption return } - dataOptions := NewDataOptions(options...) - for _, ocrResult := range ocrResults { rect := image.Rectangle{ // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 @@ -229,18 +235,13 @@ func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption }, } - // check if text in scope - if rect.Min.X < dataOptions.Scope[0] || rect.Max.X > dataOptions.Scope[2] || - rect.Min.Y < dataOptions.Scope[1] || rect.Max.Y > dataOptions.Scope[3] { - // not in scope - continue - } - ocrTexts = append(ocrTexts, OCRText{ Text: ocrResult.Text, Rect: rect, }) } + + log.Debug().Interface("texts", ocrTexts).Msg("get screen texts by veDEM OCR") return } @@ -270,28 +271,27 @@ func getLogID(header http.Header) string { } type IOCRService interface { - GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error) + GetTexts(imageBuf *bytes.Buffer) (texts OCRTexts, err error) } -func (dExt *DriverExt) GetScreenTextsByOCR(options ...DataOption) (texts OCRTexts, err error) { +func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.TakeScreenShot( builtin.GenNameWithTimestamp("screenshot_%d_ocr")); err != nil { return } - ocrTexts, err := dExt.OCRService.GetTexts(bufSource, options...) + ocrTexts, err := dExt.OCRService.GetTexts(bufSource) if err != nil { log.Error().Err(err).Msg("GetScreenTextsByOCR failed") return } - log.Debug().Interface("texts", ocrTexts).Msg("get screen texts by OCR") return ocrTexts, nil } func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...DataOption) (point PointF, err error) { - ocrTexts, err := dExt.GetScreenTextsByOCR(options...) + ocrTexts, err := dExt.GetScreenTextsByOCR() if err != nil { return } From 45bdfa36f2c15327a568ec25ccd30b0a6b11ad98 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 26 Apr 2023 22:47:00 +0800 Subject: [PATCH 08/67] fix: handle screen scale for iOS --- hrp/pkg/uixt/android_test.go | 14 -------------- hrp/pkg/uixt/client.go | 1 + hrp/pkg/uixt/ext.go | 21 ++++++--------------- hrp/pkg/uixt/ios_device.go | 10 ++++++++++ hrp/pkg/uixt/ios_driver.go | 35 ++++++++++++++++++++--------------- hrp/pkg/uixt/ios_test.go | 11 +++++++++++ hrp/pkg/uixt/opencv.go | 2 -- 7 files changed, 48 insertions(+), 46 deletions(-) diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index b2a89037..be5740ce 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -153,20 +153,6 @@ func TestDriver_GetAppiumSettings(t *testing.T) { // t.Log(appiumSettings) } -func TestDriver_DeviceScaleRatio(t *testing.T) { - driver, err := NewUIADriver(nil, uiaServerURL) - if err != nil { - t.Fatal(err) - } - - scaleRatio, err := driver.Scale() - if err != nil { - t.Fatal(err) - } - - t.Log(scaleRatio) -} - func TestDriver_DeviceInfo(t *testing.T) { driver, err := NewUIADriver(nil, uiaServerURL) if err != nil { diff --git a/hrp/pkg/uixt/client.go b/hrp/pkg/uixt/client.go index 95a8b277..b8166da8 100644 --- a/hrp/pkg/uixt/client.go +++ b/hrp/pkg/uixt/client.go @@ -20,6 +20,7 @@ type Driver struct { urlPrefix *url.URL sessionId string client *http.Client + scale float64 // cache the last launched package name lastLaunchedPackageName string } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index ce1aeb24..ffe61f64 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -51,7 +51,6 @@ type DriverExt struct { windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - scale float64 OCRService IOCRService // used to get text from image screenShots []string // cache screenshot paths @@ -71,10 +70,6 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, errors.Wrap(err, "failed to get windows size") } - if dExt.scale, err = dExt.Driver.Scale(); err != nil { - return nil, err - } - if dExt.OCRService, err = newVEDEMOCRService(); err != nil { return nil, err } @@ -187,12 +182,6 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) ( return dExt.FindImageRectInUIKit(search, options...) } -func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { - x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale - width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale - return -} - func (dExt *DriverExt) IsOCRExist(text string) bool { _, err := dExt.FindScreenTextByOCR(text) return err == nil @@ -213,11 +202,13 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool { return true } +// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner +// the value of (x, y) is between 0 and 1, which means the percentage of the screen func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { - return int(x1 * float64(dExt.windowSize.Width) * dExt.scale), - int(y1 * float64(dExt.windowSize.Height) * dExt.scale), - int(x2 * float64(dExt.windowSize.Width) * dExt.scale), - int(y2 * float64(dExt.windowSize.Height) * dExt.scale) + return int(x1 * float64(dExt.windowSize.Width)), + int(y1 * float64(dExt.windowSize.Height)), + int(x2 * float64(dExt.windowSize.Width)), + int(y2 * float64(dExt.windowSize.Height)) } func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { diff --git a/hrp/pkg/uixt/ios_device.go b/hrp/pkg/uixt/ios_device.go index 93828a8d..cac6bfce 100644 --- a/hrp/pkg/uixt/ios_device.go +++ b/hrp/pkg/uixt/ios_device.go @@ -631,6 +631,11 @@ func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver WebDriver } wd.mjpegClient = convertToHTTPClient(wd.mjpegHTTPConn) + // init WDA scale + if wd.scale, err = wd.Scale(); err != nil { + return nil, err + } + return wd, nil } @@ -659,6 +664,11 @@ func (dev *IOSDevice) NewUSBDriver(capabilities Capabilities) (driver WebDriver, return nil, errors.Wrap(code.IOSDeviceUSBDriverError, err.Error()) } + // init WDA scale + if wd.scale, err = wd.Scale(); err != nil { + return nil, err + } + return wd, nil } diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 830d78cc..64cb8f24 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -164,11 +164,16 @@ func (wd *wdaDriver) Screen() (screen Screen, err error) { func (wd *wdaDriver) Scale() (float64, error) { screen, err := wd.Screen() if err != nil { - return 0, err + return 0, errors.Wrap(code.MobileUIDriverError, + fmt.Sprintf("get screen info failed: %v", err)) } return screen.Scale, nil } +func (wd *wdaDriver) toScale(x float64) float64 { + return x / wd.scale +} + func (wd *wdaDriver) ActiveAppInfo() (info AppInfo, err error) { // [[FBRoute GET:@"/wda/activeAppInfo"] respondWithTarget:self action:@selector(handleActiveAppInfo:)] // [[FBRoute GET:@"/wda/activeAppInfo"].withoutSession @@ -375,8 +380,8 @@ func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } // new data options in post data for extra WDA configurations newData := NewData(data, options...) @@ -392,8 +397,8 @@ func (wd *wdaDriver) DoubleTap(x, y int) error { func (wd *wdaDriver) DoubleTapFloat(x, y float64) (err error) { // [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/doubleTap") return @@ -406,8 +411,8 @@ func (wd *wdaDriver) TouchAndHold(x, y int, second ...float64) error { func (wd *wdaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err error) { // [[FBRoute POST:@"/wda/touchAndHold"] respondWithTarget:self action:@selector(handleTouchAndHoldCoordinate:)] data := map[string]interface{}{ - "x": x, - "y": y, + "x": wd.toScale(x), + "y": wd.toScale(y), } if len(second) == 0 || second[0] <= 0 { second = []float64{1.0} @@ -424,10 +429,10 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) err func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) { // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] data := map[string]interface{}{ - "fromX": fromX, - "fromY": fromY, - "toX": toX, - "toY": toY, + "fromX": wd.toScale(fromX), + "fromY": wd.toScale(fromY), + "toX": wd.toScale(toX), + "toY": wd.toScale(toY), } // new data options in post data for extra WDA configurations @@ -491,10 +496,10 @@ func (wd *wdaDriver) PressBack(options ...DataOption) (err error) { } data := map[string]interface{}{ - "fromX": float64(windowSize.Width) * 0, - "fromY": float64(windowSize.Height) * 0.5, - "toX": float64(windowSize.Width) * 0.6, - "toY": float64(windowSize.Height) * 0.5, + "fromX": wd.toScale(float64(windowSize.Width) * 0), + "fromY": wd.toScale(float64(windowSize.Height) * 0.5), + "toX": wd.toScale(float64(windowSize.Width) * 0.6), + "toY": wd.toScale(float64(windowSize.Height) * 0.5), } // new data options in post data for extra WDA configurations diff --git a/hrp/pkg/uixt/ios_test.go b/hrp/pkg/uixt/ios_test.go index c894f5aa..3865953a 100644 --- a/hrp/pkg/uixt/ios_test.go +++ b/hrp/pkg/uixt/ios_test.go @@ -68,6 +68,17 @@ func TestNewUSBDriver(t *testing.T) { // t.Log(driver.IsWdaHealthy()) } +func TestDriver_DeviceScaleRatio(t *testing.T) { + setup(t) + + scaleRatio, err := driver.Scale() + if err != nil { + t.Fatal(err) + } + + t.Log(scaleRatio) +} + func Test_remoteWD_DeleteSession(t *testing.T) { setup(t) diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index 635e18e0..e1b3cbdb 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -71,7 +71,6 @@ func (dExt *DriverExt) Debug(dm DebugMode) { func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { newExt = new(DriverExt) newExt.Driver = dExt.Driver - newExt.scale = dExt.scale newExt.matchMode = dExt.matchMode newExt.threshold = threshold return @@ -80,7 +79,6 @@ func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { newExt = new(DriverExt) newExt.Driver = dExt.Driver - newExt.scale = dExt.scale newExt.matchMode = matchMode newExt.threshold = dExt.threshold return From df5731aafd0c602edf938ba3e538378b0edc4618 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 27 Apr 2023 13:38:02 +0800 Subject: [PATCH 09/67] change: log ocr service request elapsed seconds --- hrp/pkg/uixt/action.go | 146 +++++--------------------------------- hrp/pkg/uixt/ocr_vedem.go | 5 +- hrp/pkg/uixt/swipe.go | 134 ++++++++++++++++++++-------------- 3 files changed, 105 insertions(+), 180 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index d5f12c19..b5c7162a 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -64,20 +64,19 @@ type MobileAction struct { Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` - Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log - MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times - WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second - Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action - Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action - Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action - IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` + Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log + MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second + Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action + Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action + IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found + Text string `json:"text,omitempty" yaml:"text,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` } type ActionOption func(o *MobileAction) @@ -115,14 +114,14 @@ func WithSteps(steps int) ActionOption { // WithDirection inputs direction (up, down, left, right) func WithDirection(direction string) ActionOption { return func(o *MobileAction) { - o.Direction = direction + o.Params = direction } } // WithCustomDirection inputs sx, sy, ex, ey func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { return func(o *MobileAction) { - o.Direction = []float64{sx, sy, ex, ey} + o.Params = []float64{sx, sy, ex, ey} } } @@ -195,51 +194,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params, should be app name(string), got %v", ACTION_SwipeToTapApp, action.Params) case ACTION_SwipeToTapText: - // TODO: merge to LoopUntil if text, ok := action.Params.(string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - identifierOption := WithDataIdentifier(action.Identifier) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - indexOption := WithDataIndex(action.Index) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) - waitTimeOption := WithDataWaitTime(action.WaitTime) - - var point PointF - // findTextAction := func(d *DriverExt) error { - // return nil - // } - findTextCondition := func(d *DriverExt) error { - var err error - point, err = d.FindScreenTextByOCR(text, indexOption, scopeOption) - return err - } - foundTextAction := func(d *DriverExt) error { - // tap text - return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) - } - - if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) - } - // swipe until found - return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption) + return dExt.swipeToTapTexts([]string{text}, action) } return fmt.Errorf("invalid %s params, should be app text(string), got %v", ACTION_SwipeToTapText, action.Params) case ACTION_SwipeToTapTexts: - // TODO: merge to LoopUntil if texts, ok := action.Params.([]interface{}); ok { var textList []string for _, t := range texts { @@ -248,56 +208,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Params = textList } if texts, ok := action.Params.([]string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - identifierOption := WithDataIdentifier(action.Identifier) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) - waitTimeOption := WithDataWaitTime(action.WaitTime) - - var point PointF - findTexts := func(d *DriverExt) error { - var err error - ocrTexts, err := d.GetScreenTextsByOCR() - if err != nil { - return err - } - points, err := ocrTexts.FindTexts(texts, scopeOption) - if err != nil { - return err - } - for _, point = range points { - if point != (PointF{X: 0, Y: 0}) { - return nil - } - } - return errors.New("failed to find text position") - } - foundTextAction := func(d *DriverExt) error { - // tap text - return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) - } - - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - - if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption) - } - // swipe until found - return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption) + return dExt.swipeToTapTexts(texts, action) } return fmt.Errorf("invalid %s params, should be app text([]string), got %v", ACTION_SwipeToTapText, action.Params) @@ -384,27 +295,8 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) case ACTION_Swipe: - identifierOption := WithDataIdentifier(action.Identifier) - durationOption := WithDataPressDuration(action.Duration) - if action.Steps == 0 { - action.Steps = 10 - } - stepsOption := WithDataSteps(action.Steps) - if positions, ok := action.Params.([]interface{}); ok { - // relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1] - if len(positions) != 4 { - return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions) - } - fromX, _ := positions[0].(float64) - fromY, _ := positions[1].(float64) - toX, _ := positions[2].(float64) - toY, _ := positions[3].(float64) - return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption) - } - if direction, ok := action.Params.(string); ok { - return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption) - } - return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params) + swipeAction := dExt.prepareSwipeAction(action) + return swipeAction(dExt) case ACTION_Input: // input text on current active element // append \n to send text with enter diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 0b1ca58e..2d8821b6 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -167,7 +167,9 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err var resp *http.Response // retry 3 times for i := 1; i <= 3; i++ { + start := time.Now() resp, err = client.Do(req) + elapsed := time.Since(start) var logID string if resp != nil { logID = getLogID(resp.Header) @@ -175,7 +177,8 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err if err == nil && resp.StatusCode == http.StatusOK { log.Debug(). Str("X-TT-LOGID", logID). - Int("imageBufSize", size). + Int("image_bytes", size). + Float64("elapsed_seconds", elapsed.Seconds()). Msg("request OCR service success") break } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 96f16149..a056554c 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -66,42 +66,6 @@ func (dExt *DriverExt) SwipeRight(options ...DataOption) (err error) { type Action func(driver *DriverExt) error -// findCondition indicates the condition to find a UI element -// foundAction indicates the action to do after a UI element is found -func (dExt *DriverExt) SwipeUntil(direction interface{}, findCondition Action, foundAction Action, options ...DataOption) error { - dataOptions := NewDataOptions(options...) - maxRetryTimes := dataOptions.MaxRetryTimes - interval := dataOptions.Interval - - for i := 0; i < maxRetryTimes; i++ { - if err := findCondition(dExt); err == nil { - // do action after found - return foundAction(dExt) - } - if d, ok := direction.(string); ok { - if err := dExt.SwipeTo(d); err != nil { - log.Error().Err(err).Msgf("swipe %s failed", d) - } - } else if d, ok := direction.([]float64); ok { - if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3]); err != nil { - log.Error().Err(err).Msgf("swipe %v failed", d) - } - } else if d, ok := direction.([]interface{}); ok { - sx, _ := builtin.Interface2Float64(d[0]) - sy, _ := builtin.Interface2Float64(d[1]) - ex, _ := builtin.Interface2Float64(d[2]) - ey, _ := builtin.Interface2Float64(d[3]) - if err := dExt.SwipeRelative(sx, sy, ex, ey); err != nil { - log.Error().Err(err).Msgf("swipe (%v, %v) to (%v, %v) failed", sx, sy, ex, ey) - } - } - // wait for swipe action to completed and content to load completely - time.Sleep(time.Duration(1000*interval) * time.Millisecond) - } - return errors.Wrap(code.OCRTextNotFoundError, - fmt.Sprintf("swipe %v %d times, match condition failed", direction, maxRetryTimes)) -} - func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...DataOption) error { dataOptions := NewDataOptions(options...) maxRetryTimes := dataOptions.MaxRetryTimes @@ -125,40 +89,104 @@ func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, fmt.Sprintf("loop %d times, match find condition failed", maxRetryTimes)) } -func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error { +func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt) error { + identifierOption := WithDataIdentifier(action.Identifier) + durationOption := WithDataPressDuration(action.Duration) + + if action.Steps == 0 { + action.Steps = 10 + } + stepsOption := WithDataSteps(action.Steps) + + dataOptions := make([]DataOption, 3) + dataOptions = append(dataOptions, identifierOption, durationOption, stepsOption) + + return func(d *DriverExt) error { + defer func() { + // wait for swipe action to completed and content to load completely + time.Sleep(time.Duration(1000*action.WaitTime) * time.Millisecond) + }() + + if d, ok := action.Params.(string); ok { + // enum direction: up, down, left, right + if err := dExt.SwipeTo(d, dataOptions...); err != nil { + log.Error().Err(err).Msgf("swipe %s failed", d) + return err + } + } else if d, ok := action.Params.([]float64); ok { + // custom direction: [fromX, fromY, toX, toY] + if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], dataOptions...); err != nil { + log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", + d[0], d[1], d[2], d[3]) + return err + } + } else if d, ok := action.Params.([]interface{}); ok { + // loaded from json case + // custom direction: [fromX, fromY, toX, toY] + sx, _ := builtin.Interface2Float64(d[0]) + sy, _ := builtin.Interface2Float64(d[1]) + ex, _ := builtin.Interface2Float64(d[2]) + ey, _ := builtin.Interface2Float64(d[3]) + if err := dExt.SwipeRelative(sx, sy, ex, ey, dataOptions...); err != nil { + log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", + sx, sy, ex, ey) + return err + } + } else { + return fmt.Errorf("invalid swipe params %v", action.Params) + } + return nil + } +} + +func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) error { if len(action.Scope) != 4 { action.Scope = []float64{0, 0, 1, 1} } if len(action.Offset) != 2 { - action.Offset = []int{0, -25} + action.Offset = []int{0, 0} } identifierOption := WithDataIdentifier(action.Identifier) - indexOption := WithDataIndex(action.Index) offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) + indexOption := WithDataIndex(action.Index) scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - - // default to retry 5 times + // default to retry 10 times if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 5 + action.MaxRetryTimes = 10 } maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) waitTimeOption := WithDataWaitTime(action.WaitTime) var point PointF - findAppAction := func(d *DriverExt) error { - return dExt.SwipeLeft() - } - findAppCondition := func(d *DriverExt) error { + findTexts := func(d *DriverExt) error { var err error - point, err = d.FindScreenTextByOCR(appName, scopeOption, indexOption) - return err + ocrTexts, err := d.GetScreenTextsByOCR() + if err != nil { + return err + } + points, err := ocrTexts.FindTexts(texts, indexOption, scopeOption) + if err != nil { + return err + } + // FIXME: handle index + for _, point = range points { + if point != (PointF{X: 0, Y: 0}) { + return nil + } + } + return errors.New("failed to find text position") } - foundAppAction := func(d *DriverExt) error { - // click app to launch + foundTextAction := func(d *DriverExt) error { + // tap text return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) } + findAction := dExt.prepareSwipeAction(action) + return dExt.LoopUntil(findAction, findTexts, foundTextAction, maxRetryOption, waitTimeOption) +} + +func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error { // go to home screen if err := dExt.Driver.Homescreen(); err != nil { return errors.Wrap(err, "go to home screen failed") @@ -169,6 +197,8 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error dExt.SwipeRight() } - // swipe next screen until app found - return dExt.LoopUntil(findAppAction, findAppCondition, foundAppAction, maxRetryOption, waitTimeOption) + action.Offset = []int{0, -25} + action.Params = "left" + + return dExt.swipeToTapTexts([]string{appName}, action) } From 0794ef8296bf54cef7512b3fe195e40612277eb1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 27 Apr 2023 14:08:24 +0800 Subject: [PATCH 10/67] refactor: swipe actions --- hrp/pkg/uixt/android_test.go | 12 ++++++- hrp/pkg/uixt/ocr_vedem.go | 2 +- hrp/pkg/uixt/swipe.go | 15 ++++---- hrp/pkg/uixt/swipe_test.go | 69 ++++++++++++++++++------------------ 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index be5740ce..433cc38a 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -10,7 +10,17 @@ import ( "time" ) -var uiaServerURL = "http://localhost:6790/wd/hub" +var ( + uiaServerURL = "http://localhost:6790/wd/hub" + driverExt *DriverExt +) + +func setupAndroid(t *testing.T) { + device, err := NewAndroidDevice() + checkErr(t, err) + driverExt, err = device.NewDriver(nil) + checkErr(t, err) +} func TestDriver_NewSession(t *testing.T) { driver, err := NewUIADriver(nil, uiaServerURL) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 2d8821b6..e50018bd 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -280,7 +280,7 @@ type IOCRService interface { func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.TakeScreenShot( - builtin.GenNameWithTimestamp("screenshot_%d_ocr")); err != nil { + builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index a056554c..3b7ad7f3 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -140,6 +140,10 @@ func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt } func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) error { + if len(texts) == 0 { + return errors.New("no text to tap") + } + if len(action.Scope) != 4 { action.Scope = []float64{0, 0, 1, 1} } @@ -169,13 +173,8 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) erro if err != nil { return err } - // FIXME: handle index - for _, point = range points { - if point != (PointF{X: 0, Y: 0}) { - return nil - } - } - return errors.New("failed to find text position") + point = points[0] // FIXME + return nil } foundTextAction := func(d *DriverExt) error { // tap text @@ -197,7 +196,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error dExt.SwipeRight() } - action.Offset = []int{0, -25} + action.Offset = []int{0, -25} // tap app icon above the text action.Params = "left" return dExt.swipeToTapTexts([]string{appName}, action) diff --git a/hrp/pkg/uixt/swipe_test.go b/hrp/pkg/uixt/swipe_test.go index 0828d934..32cb3033 100644 --- a/hrp/pkg/uixt/swipe_test.go +++ b/hrp/pkg/uixt/swipe_test.go @@ -6,43 +6,44 @@ import ( "testing" ) -func TestSwipeUntil(t *testing.T) { - driverExt, err := iosDevice.NewDriver(nil) +func TestAndroidSwipeAction(t *testing.T) { + setupAndroid(t) + + action := MobileAction{ + Method: ACTION_Swipe, + Params: "up", + } + swipeAction := driverExt.prepareSwipeAction(action) + + err := swipeAction(driverExt) checkErr(t, err) - var point PointF - findApp := func(d *DriverExt) error { - var err error - point, err = d.FindScreenTextByOCR("抖音") - return err - } - foundAppAction := func(d *DriverExt) error { - // click app, launch douyin - return d.TapAbsXY(point.X, point.Y) + action = MobileAction{ + Method: ACTION_Swipe, + Params: []float64{0.5, 0.5, 0.5, 0.9}, } + swipeAction = driverExt.prepareSwipeAction(action) - driverExt.Driver.Homescreen() - - // swipe to first screen - for i := 0; i < 5; i++ { - driverExt.SwipeRight() - } - - // swipe until app found - err = driverExt.SwipeUntil("left", findApp, foundAppAction, WithDataMaxRetryTimes(10)) - checkErr(t, err) - - findLive := func(d *DriverExt) error { - var err error - point, err = d.FindScreenTextByOCR("点击进入直播间") - return err - } - foundLiveAction := func(d *DriverExt) error { - // enter live room - return d.TapAbsXY(point.X, point.Y) - } - - // swipe until live room found - err = driverExt.SwipeUntil("up", findLive, foundLiveAction, WithDataMaxRetryTimes(20)) + err = swipeAction(driverExt) + checkErr(t, err) +} + +func TestAndroidSwipeToTapApp(t *testing.T) { + setupAndroid(t) + + err := driverExt.swipeToTapApp("抖音", MobileAction{}) + checkErr(t, err) +} + +func TestAndroidSwipeToTapTexts(t *testing.T) { + setupAndroid(t) + + err := driverExt.Driver.AppLaunch("com.ss.android.ugc.aweme") + checkErr(t, err) + + action := MobileAction{ + Params: "up", + } + err = driverExt.swipeToTapTexts([]string{"点击进入直播间", "直播中"}, action) checkErr(t, err) } From d0f3218c189de625a6020d274acb7f874595cccc Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 27 Apr 2023 19:37:59 +0800 Subject: [PATCH 11/67] change: add TODO compress image data --- hrp/pkg/uixt/ext.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index ffe61f64..8e402d06 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -87,9 +87,6 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) // TakeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder // if fileName is empty, it will not save image file and only return raw image data func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, err error) { - // wait for action done - time.Sleep(500 * time.Millisecond) - // iOS 优先使用 MJPEG 流进行截图，性能最优 // 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口 if dExt.frame != nil { @@ -100,10 +97,17 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, er return nil, err } + // compress image data + compressed, err := compressImageBuffer(raw) + if err != nil { + log.Error().Err(err).Msg("compress screenshot data failed") + return nil, err + } + // save screenshot to file if len(fileName) > 0 && fileName[0] != "" { path := filepath.Join(env.ScreenShotsPath, fileName[0]) - path, err := dExt.saveScreenShot(raw, path) + path, err := dExt.saveScreenShot(compressed, path) if err != nil { log.Error().Err(err).Msg("save screenshot file failed") return nil, err @@ -112,6 +116,11 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, er log.Info().Str("path", path).Msg("save screenshot file success") } + return compressed, nil +} + +func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error) { + // TODO: compress image data return raw, nil } From c0ea52d02ad65d00b2c7a16eb83b447eaf2db009 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 27 Apr 2023 21:43:22 +0800 Subject: [PATCH 12/67] change: rename screenshot filenames --- hrp/pkg/uixt/action.go | 2 +- hrp/pkg/uixt/opencv.go | 2 +- hrp/step_mobile_ui.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index b5c7162a..696f35fc 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -340,7 +340,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") - _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot")) + _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot")) return err case ACTION_StartCamera: return dExt.Driver.StartCamera() diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index e1b3cbdb..06a6122d 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -101,7 +101,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, if bufSearch, err = getBufFromDisk(search); err != nil { return nil, err } - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil { + if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil { return nil, err } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 344c15bb..83d8f5f4 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -610,7 +610,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // take screenshot after each step _, err := uiDriver.TakeScreenShot( - builtin.GenNameWithTimestamp("step_%d_") + step.Name) + builtin.GenNameWithTimestamp("%d_step_") + step.Name) if err != nil { log.Error().Err(err).Str("step", step.Name).Msg("take screenshot failed on step finished") } From 9981dad1f8229c82f18eaa0e3311d75e1d85d97f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 11:27:18 +0800 Subject: [PATCH 13/67] feat: cache screenshot ocr texts --- examples/uitest/demo_android_live_swipe.json | 4 +- .../uitest/demo_android_live_swipe_test.go | 4 +- hrp/pkg/uixt/action.go | 2 +- hrp/pkg/uixt/ext.go | 39 ++++++++++++------- hrp/pkg/uixt/ocr_vedem.go | 4 +- hrp/pkg/uixt/opencv.go | 2 +- hrp/pkg/uixt/video_crawler.go | 22 +++++++++-- hrp/pkg/uixt/video_crawler_test.go | 12 ++---- hrp/step_mobile_ui.go | 2 +- 9 files changed, 56 insertions(+), 35 deletions(-) diff --git a/examples/uitest/demo_android_live_swipe.json b/examples/uitest/demo_android_live_swipe.json index 3912f1ca..bd79ad18 100644 --- a/examples/uitest/demo_android_live_swipe.json +++ b/examples/uitest/demo_android_live_swipe.json @@ -97,7 +97,7 @@ }, { "method": "sleep", - "params": 10 + "params": 5 }, { "method": "screenshot" @@ -109,7 +109,7 @@ }, { "method": "sleep", - "params": 10 + "params": 5 }, { "method": "screenshot" diff --git a/examples/uitest/demo_android_live_swipe_test.go b/examples/uitest/demo_android_live_swipe_test.go index 500aff1d..f8d9448b 100644 --- a/examples/uitest/demo_android_live_swipe_test.go +++ b/examples/uitest/demo_android_live_swipe_test.go @@ -37,8 +37,8 @@ func TestAndroidLiveSwipeTest(t *testing.T) { SleepRandom(0, 5, 0.6, 5, 10, 0.4), hrp.NewStep("向上滑动，等待 10s"). Android(). - SwipeUp(uixt.WithIdentifier("第一次上划")).Sleep(10).ScreenShot(). // 上划 1 次，等待 10s，截图保存 - SwipeUp(uixt.WithIdentifier("第二次上划")).Sleep(10).ScreenShot(), // 再上划 1 次，等待 10s，截图保存 + SwipeUp(uixt.WithIdentifier("第一次上划")).Sleep(5).ScreenShot(). // 上划 1 次，等待 5s，截图保存 + SwipeUp(uixt.WithIdentifier("第二次上划")).Sleep(5).ScreenShot(), // 再上划 1 次，等待 5s，截图保存 hrp.NewStep("exit"). Android(). AppTerminate("com.ss.android.ugc.aweme"). diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 696f35fc..31c11c20 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -340,7 +340,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") - _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot")) + _, _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot")) return err case ACTION_StartCamera: return dExt.Driver.StartCamera() diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 8e402d06..9f6fcf44 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -51,16 +51,17 @@ type DriverExt struct { windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - OCRService IOCRService // used to get text from image - screenShots []string // cache screenshot paths + OCRService IOCRService // used to get texts from image + stepScreenShots map[string]OCRTexts // cache screenshot ocr results, key is image path CVArgs } func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) { dExt = &DriverExt{ - Device: device, - Driver: driver, + Device: device, + Driver: driver, + stepScreenShots: make(map[string]OCRTexts), } dExt.doneMjpegStream = make(chan bool, 1) @@ -86,22 +87,22 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) // TakeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder // if fileName is empty, it will not save image file and only return raw image data -func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, err error) { +func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, path string, err error) { // iOS 优先使用 MJPEG 流进行截图，性能最优 // 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口 if dExt.frame != nil { - return dExt.frame, nil + return dExt.frame, "", nil } if raw, err = dExt.Driver.Screenshot(); err != nil { log.Error().Err(err).Msg("capture screenshot data failed") - return nil, err + return nil, "", err } // compress image data compressed, err := compressImageBuffer(raw) if err != nil { log.Error().Err(err).Msg("compress screenshot data failed") - return nil, err + return nil, "", err } // save screenshot to file @@ -110,13 +111,12 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, er path, err := dExt.saveScreenShot(compressed, path) if err != nil { log.Error().Err(err).Msg("save screenshot file failed") - return nil, err + return nil, "", err } - dExt.screenShots = append(dExt.screenShots, path) - log.Info().Str("path", path).Msg("save screenshot file success") + return compressed, path, nil } - return compressed, nil + return compressed, "", nil } func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error) { @@ -160,14 +160,23 @@ func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (strin return "", errors.Wrap(err, "encode screenshot image failed") } + dExt.stepScreenShots[screenshotPath] = nil + log.Info().Str("path", screenshotPath).Msg("save screenshot file success") return screenshotPath, nil } -func (dExt *DriverExt) GetScreenShots() []string { +func (dExt *DriverExt) GetScreenShots() map[string]OCRTexts { defer func() { - dExt.screenShots = nil + for key := range dExt.stepScreenShots { + delete(dExt.stepScreenShots, key) + } }() - return dExt.screenShots + + copied := make(map[string]OCRTexts) + for key, value := range dExt.stepScreenShots { + copied[key] = value + } + return copied } // isPathExists returns true if path exists, whether path is file or dir diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index e50018bd..28538adc 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -279,7 +279,8 @@ type IOCRService interface { func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot( + var imagePath string + if bufSource, imagePath, err = dExt.TakeScreenShot( builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } @@ -290,6 +291,7 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { return } + dExt.stepScreenShots[imagePath] = ocrTexts return ocrTexts, nil } diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index 06a6122d..d3888128 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -101,7 +101,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, if bufSearch, err = getBufFromDisk(search); err != nil { return nil, err } - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil { + if bufSource, _, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil { return nil, err } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index ad1644cd..450fc744 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -25,15 +25,21 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // loop until target count achieved for { // take screenshot and get screen texts by OCR - _, err := dExt.GetScreenTextsByOCR() + texts, err := dExt.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") return err } - // TODO: check if text popup exists + // check if text popup exists + if isTextPopup(texts) { + log.Info().Msg("text popup found") + } - // TODO: check if live video + // check if live video + if isLiveVideo(texts) { + log.Info().Msg("live video found") + } // assert feed video type @@ -48,3 +54,13 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // return nil } + +func isTextPopup(texts OCRTexts) bool { + texts.FindTexts([]string{"确定", "取消"}) + return false +} + +func isLiveVideo(texts OCRTexts) bool { + _, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) + return err == nil +} diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index 78edb206..d849395e 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -5,18 +5,12 @@ package uixt import "testing" func TestVideoCrawler(t *testing.T) { - device, err := NewAndroidDevice() - if err != nil { - t.Fatal(err) - } - driver, err := device.NewDriver(nil) - if err != nil { - t.Fatal(err) - } + setupAndroid(t) + configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", } - err = driver.VideoCrawler(configs) + err := driverExt.VideoCrawler(configs) if err != nil { t.Fatal(err) } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 83d8f5f4..c34ea4ae 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -609,7 +609,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err } // take screenshot after each step - _, err := uiDriver.TakeScreenShot( + _, _, err := uiDriver.TakeScreenShot( builtin.GenNameWithTimestamp("%d_step_") + step.Name) if err != nil { log.Error().Err(err).Str("step", step.Name).Msg("take screenshot failed on step finished") From 37a984c243c2c31782d51fd6a3771cde63e1cd6d Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 12:05:19 +0800 Subject: [PATCH 14/67] fix: swipe to find texts --- .../uitest/demo_douyin_follow_live_test.go | 3 -- hrp/pkg/uixt/action.go | 31 ++++++++++--------- hrp/pkg/uixt/swipe.go | 17 +++++++--- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/examples/uitest/demo_douyin_follow_live_test.go b/examples/uitest/demo_douyin_follow_live_test.go index 59b223ef..d87f1da9 100644 --- a/examples/uitest/demo_douyin_follow_live_test.go +++ b/examples/uitest/demo_douyin_follow_live_test.go @@ -47,9 +47,6 @@ func TestIOSDouyinFollowLive(t *testing.T) { if err := testCase.Dump2JSON("demo_douyin_follow_live.json"); err != nil { t.Fatal(err) } - if err := testCase.Dump2YAML("demo_douyin_follow_live.yaml"); err != nil { - t.Fatal(err) - } runner := hrp.NewRunner(t).SetSaveTests(true) err := runner.Run(testCase) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 31c11c20..d80126f2 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -64,19 +64,20 @@ type MobileAction struct { Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` - Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log - MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times - WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second - Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action - Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action - IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` + Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log + MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second + Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action + Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action + Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action + IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found + Text string `json:"text,omitempty" yaml:"text,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` } type ActionOption func(o *MobileAction) @@ -114,14 +115,14 @@ func WithSteps(steps int) ActionOption { // WithDirection inputs direction (up, down, left, right) func WithDirection(direction string) ActionOption { return func(o *MobileAction) { - o.Params = direction + o.Direction = direction } } // WithCustomDirection inputs sx, sy, ex, ey func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { return func(o *MobileAction) { - o.Params = []float64{sx, sy, ex, ey} + o.Direction = []float64{sx, sy, ex, ey} } } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 3b7ad7f3..2636f86d 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -101,26 +101,33 @@ func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt dataOptions := make([]DataOption, 3) dataOptions = append(dataOptions, identifierOption, durationOption, stepsOption) + var swipeDirection interface{} + if action.Direction != nil { + swipeDirection = action.Direction + } else { + swipeDirection = "up" // default swipe up + } + return func(d *DriverExt) error { defer func() { // wait for swipe action to completed and content to load completely time.Sleep(time.Duration(1000*action.WaitTime) * time.Millisecond) }() - if d, ok := action.Params.(string); ok { + if d, ok := swipeDirection.(string); ok { // enum direction: up, down, left, right if err := dExt.SwipeTo(d, dataOptions...); err != nil { log.Error().Err(err).Msgf("swipe %s failed", d) return err } - } else if d, ok := action.Params.([]float64); ok { + } else if d, ok := swipeDirection.([]float64); ok { // custom direction: [fromX, fromY, toX, toY] if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], dataOptions...); err != nil { log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", d[0], d[1], d[2], d[3]) return err } - } else if d, ok := action.Params.([]interface{}); ok { + } else if d, ok := swipeDirection.([]interface{}); ok { // loaded from json case // custom direction: [fromX, fromY, toX, toY] sx, _ := builtin.Interface2Float64(d[0]) @@ -133,7 +140,7 @@ func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt return err } } else { - return fmt.Errorf("invalid swipe params %v", action.Params) + return fmt.Errorf("invalid swipe params %v", swipeDirection) } return nil } @@ -197,7 +204,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error } action.Offset = []int{0, -25} // tap app icon above the text - action.Params = "left" + action.Direction = "left" return dExt.swipeToTapTexts([]string{appName}, action) } From 23230157551326f27a362e1b9859ac19e4330c3c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 13:23:52 +0800 Subject: [PATCH 15/67] change: dump ocr texts as string in summary --- hrp/pkg/uixt/ext.go | 12 ++++++------ hrp/pkg/uixt/ocr_vedem.go | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 9f6fcf44..2112e168 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -51,8 +51,8 @@ type DriverExt struct { windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - OCRService IOCRService // used to get texts from image - stepScreenShots map[string]OCRTexts // cache screenshot ocr results, key is image path + OCRService IOCRService // used to get texts from image + stepScreenShots map[string]string // cache screenshot ocr results, key is image path, value is dumped OCRTexts CVArgs } @@ -61,7 +61,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) dExt = &DriverExt{ Device: device, Driver: driver, - stepScreenShots: make(map[string]OCRTexts), + stepScreenShots: make(map[string]string), } dExt.doneMjpegStream = make(chan bool, 1) @@ -160,19 +160,19 @@ func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (strin return "", errors.Wrap(err, "encode screenshot image failed") } - dExt.stepScreenShots[screenshotPath] = nil + dExt.stepScreenShots[screenshotPath] = "" log.Info().Str("path", screenshotPath).Msg("save screenshot file success") return screenshotPath, nil } -func (dExt *DriverExt) GetScreenShots() map[string]OCRTexts { +func (dExt *DriverExt) GetScreenShots() map[string]string { defer func() { for key := range dExt.stepScreenShots { delete(dExt.stepScreenShots, key) } }() - copied := make(map[string]OCRTexts) + copied := make(map[string]string) for key, value := range dExt.stepScreenShots { copied[key] = value } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 28538adc..1558b95f 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -291,7 +291,8 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { return } - dExt.stepScreenShots[imagePath] = ocrTexts + o, _ := json.Marshal(ocrTexts) + dExt.stepScreenShots[imagePath] = string(o) return ocrTexts, nil } From 1d41d276abf7544d276d8de9925406f61121dfed Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 14:12:31 +0800 Subject: [PATCH 16/67] feat: cache screenshot ocr texts --- hrp/pkg/uixt/ext.go | 41 +++++++++++++++++++++++---------------- hrp/pkg/uixt/ocr_vedem.go | 2 +- hrp/step_mobile_ui.go | 4 +++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 2112e168..59c658a7 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -45,23 +45,34 @@ func WithThreshold(threshold float64) CVOption { } } +type cacheStepData struct { + // cache step screenshot paths + ScreenShots []string + // cache step screenshot ocr results, key is image path, value is dumped OCRTexts + OcrResults map[string]string +} + type DriverExt struct { + CVArgs Device Device Driver WebDriver windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - OCRService IOCRService // used to get texts from image - stepScreenShots map[string]string // cache screenshot ocr results, key is image path, value is dumped OCRTexts + OCRService IOCRService // used to get texts from image - CVArgs + // cache step data + cacheStepData cacheStepData } func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) { dExt = &DriverExt{ - Device: device, - Driver: driver, - stepScreenShots: make(map[string]string), + Device: device, + Driver: driver, + cacheStepData: cacheStepData{ + ScreenShots: make([]string, 0), + OcrResults: make(map[string]string), + }, } dExt.doneMjpegStream = make(chan bool, 1) @@ -160,21 +171,17 @@ func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (strin return "", errors.Wrap(err, "encode screenshot image failed") } - dExt.stepScreenShots[screenshotPath] = "" + dExt.cacheStepData.ScreenShots = append(dExt.cacheStepData.ScreenShots, screenshotPath) log.Info().Str("path", screenshotPath).Msg("save screenshot file success") return screenshotPath, nil } -func (dExt *DriverExt) GetScreenShots() map[string]string { - defer func() { - for key := range dExt.stepScreenShots { - delete(dExt.stepScreenShots, key) - } - }() - - copied := make(map[string]string) - for key, value := range dExt.stepScreenShots { - copied[key] = value +func (dExt *DriverExt) GetStepCacheData() cacheStepData { + copied := dExt.cacheStepData + // clear cache + dExt.cacheStepData = cacheStepData{ + ScreenShots: []string{}, + OcrResults: make(map[string]string), } return copied } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 1558b95f..939ffcbd 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -292,7 +292,7 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { } o, _ := json.Marshal(ocrTexts) - dExt.stepScreenShots[imagePath] = string(o) + dExt.cacheStepData.OcrResults[imagePath] = string(o) return ocrTexts, nil } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index c34ea4ae..35877e3b 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -616,7 +616,9 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err } // save attachments - attachments["screenshots"] = uiDriver.GetScreenShots() + cacheData := uiDriver.GetStepCacheData() + attachments["screenshots"] = cacheData.ScreenShots + attachments["ocr_results"] = cacheData.OcrResults stepResult.Attachments = attachments }() From 6067d5ddbd56ee4a2df04f939eff17bc2abfda48 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 16:15:53 +0800 Subject: [PATCH 17/67] refactor: AssertAppForeground --- docs/CHANGELOG.md | 2 +- hrp/internal/code/code.go | 2 ++ hrp/pkg/uixt/android_adb_driver.go | 11 +++++------ hrp/pkg/uixt/android_test.go | 9 +++++---- hrp/pkg/uixt/ext.go | 12 +----------- hrp/pkg/uixt/interface.go | 4 ++-- hrp/pkg/uixt/ios_driver.go | 4 ++-- hrp/pkg/uixt/video_crawler.go | 11 +++++++++++ hrp/step_mobile_ui.go | 6 +++--- 9 files changed, 32 insertions(+), 29 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ca27fbc5..8f18b940 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,7 +7,7 @@ - feat: add `sleep_random` to sleep random seconds, with weight for multiple time ranges - feat: input text with adb - feat: add adb `screencap` sub command -- feat: add `IsAppInForeground` to check if the given package is in foreground +- feat: add `AssertAppInForeground` to check if the given package is in foreground - feat: check if app is in foreground when step failed - feat: add validator AssertAppInForeground and AssertAppNotInForeground - feat: save screenshots of all steps including ocr and cv recognition process data diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 71f9226c..23d79bea 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -61,6 +61,7 @@ var ( var ( AndroidDeviceConnectionError = errors.New("android device connection error") // 60 AndroidDeviceUSBDriverError = errors.New("android device USB driver error") // 61 + AndroidShellExecError = errors.New("android shell exec error") // 62 AndroidScreenShotError = errors.New("android screenshot error") // 65 AndroidCaptureLogError = errors.New("android capture log error") // 66 ) @@ -121,6 +122,7 @@ var errorsMap = map[error]int{ // android related AndroidDeviceConnectionError: 60, AndroidDeviceUSBDriverError: 61, + AndroidShellExecError: 62, AndroidScreenShotError: 65, AndroidCaptureLogError: 66, diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 388c444e..b509d3b3 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -362,30 +362,29 @@ func (ad *adbDriver) GetLastLaunchedApp() (packageName string) { return ad.lastLaunchedPackageName } -func (ad *adbDriver) IsAppInForeground(packageName string) (bool, error) { +func (ad *adbDriver) AssertAppForeground(packageName string) error { if packageName == "" { - return false, errors.New("package name is not given") + return errors.New("package name is not given") } // adb shell dumpsys activity activities | grep mResumedActivity output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") - return false, err + return errors.Wrap(code.AndroidShellExecError, err.Error()) } lines := strings.Split(string(output), "\n") - isInForeground := false for _, line := range lines { trimmedLine := strings.TrimSpace(line) if strings.HasPrefix(trimmedLine, "mResumedActivity:") { if strings.Contains(trimmedLine, packageName) { - isInForeground = true + return nil } break } } - return isInForeground, nil + return errors.New("app is not in foreground") } diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index 433cc38a..af916d3e 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -361,18 +361,19 @@ func TestDriver_IsAppInForeground(t *testing.T) { t.Fatal(err) } - yes, err := driver.Driver.IsAppInForeground(driver.Driver.GetLastLaunchedApp()) - if err != nil || !yes { + err = driver.Driver.AssertAppForeground(driver.Driver.GetLastLaunchedApp()) + if err != nil { t.Fatal(err) } + time.Sleep(2 * time.Second) _, err = driver.Driver.AppTerminate("com.android.settings") if err != nil { t.Fatal(err) } - yes, err = driver.Driver.IsAppInForeground("com.android.settings") - if err != nil || yes { + err = driver.Driver.AssertAppForeground("com.android.settings") + if err == nil { t.Fatal(err) } } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 59c658a7..3c57ffb6 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -217,16 +217,6 @@ func (dExt *DriverExt) IsImageExist(text string) bool { return err == nil } -func (dExt *DriverExt) IsAppInForeground(packageName string) bool { - // check if app is in foreground - yes, err := dExt.Driver.IsAppInForeground(packageName) - if !yes || err != nil { - log.Info().Str("packageName", packageName).Msg("app is not in foreground") - return false - } - return true -} - // (x1, y1) is the top left corner, (x2, y2) is the bottom right corner // the value of (x, y) is between 0 and 1, which means the percentage of the screen func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { @@ -250,7 +240,7 @@ func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...s case SelectorImage: result = (dExt.IsImageExist(expected) == exp) case SelectorForegroundApp: - result = (dExt.IsAppInForeground(expected) == exp) + result = ((dExt.Driver.AssertAppForeground(expected) == nil) == exp) } if !result { diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 7d6145b3..180f2066 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -621,8 +621,8 @@ type WebDriver interface { AppTerminate(packageName string) (bool, error) // GetLastLaunchedApp returns the package name of the last launched app GetLastLaunchedApp() string - // IsAppInForeground returns true if the given package is in foreground - IsAppInForeground(packageName string) (bool, error) + // AssertAppForeground returns nil if the given package is in foreground + AssertAppForeground(packageName string) error // StartCamera Starts a new camera for recording StartCamera() error diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 64cb8f24..29902268 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -369,8 +369,8 @@ func (wd *wdaDriver) GetLastLaunchedApp() (packageName string) { return wd.lastLaunchedPackageName } -func (wd *wdaDriver) IsAppInForeground(packageName string) (bool, error) { - return false, errors.New("not implemented") +func (wd *wdaDriver) AssertAppForeground(packageName string) error { + return nil } func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 450fc744..777c4d72 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -3,7 +3,10 @@ package uixt import ( "time" + "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/code" ) type VideoCrawlerConfigs struct { @@ -24,6 +27,14 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // loop until target count achieved for { + // check if app in foreground + err := dExt.Driver.AssertAppForeground(configs.AppPackageName) + if err != nil { + log.Error().Err(err).Str("packageName", configs.AppPackageName).Msg("app is not in foreground") + err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) + return err + } + // take screenshot and get screen texts by OCR texts, err := dExt.GetScreenTextsByOCR() if err != nil { diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 35877e3b..2ba33467 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -601,9 +601,9 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // check if app is in foreground packageName := uiDriver.Driver.GetLastLaunchedApp() - yes, err2 := uiDriver.Driver.IsAppInForeground(packageName) - if packageName != "" && (!yes || err2 != nil) { - log.Error().Err(err2).Str("packageName", packageName).Msg("app is not in foreground") + err := uiDriver.Driver.AssertAppForeground(packageName) + if packageName != "" && err != nil { + log.Error().Err(err).Str("packageName", packageName).Msg("app is not in foreground") err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) } } From 8498fe10e319feb0e9dcc0233336563dbefe79b3 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 16:59:02 +0800 Subject: [PATCH 18/67] feat: add GetForegroundApp for android --- hrp/pkg/uixt/android_adb_driver.go | 26 ++++++++++++++++++++------ hrp/pkg/uixt/android_test.go | 24 ++++++++++++------------ hrp/pkg/uixt/interface.go | 2 ++ hrp/pkg/uixt/ios_driver.go | 4 ++++ hrp/pkg/uixt/video_crawler.go | 3 +-- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index b509d3b3..d26abffe 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -367,24 +367,38 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { return errors.New("package name is not given") } + foreApp, err := ad.GetForegroundApp() + if err != nil { + return err + } + if foreApp != packageName { + return errors.New("app is not in foreground") + } + return nil +} + +func (ad *adbDriver) GetForegroundApp() (packageName string, err error) { // adb shell dumpsys activity activities | grep mResumedActivity output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") - return errors.Wrap(code.AndroidShellExecError, err.Error()) + return "", errors.Wrap(code.AndroidShellExecError, err.Error()) } lines := strings.Split(string(output), "\n") - for _, line := range lines { trimmedLine := strings.TrimSpace(line) if strings.HasPrefix(trimmedLine, "mResumedActivity:") { - if strings.Contains(trimmedLine, packageName) { - return nil + // mResumedActivity: ActivityRecord{9656d74 u0 com.android.settings/.Settings t407} + strs := strings.Split(trimmedLine, " ") + for _, str := range strs { + if strings.Contains(str, "/") { + // com.android.settings/.Settings + return strings.Split(str, "/")[0], nil + } } - break } } - return errors.New("app is not in foreground") + return "", errors.New("get foreground app failed") } diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index af916d3e..34ba6d10 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -350,29 +350,29 @@ func TestDriver_AppLaunch(t *testing.T) { } func TestDriver_IsAppInForeground(t *testing.T) { - device, _ := NewAndroidDevice() - driver, err := device.NewDriver(nil) - if err != nil { - t.Fatal(err) + setupAndroid(t) + + err := driverExt.Driver.AppLaunch("com.android.settings") + checkErr(t, err) + + foreApp, err := driverExt.Driver.GetForegroundApp() + checkErr(t, err) + if foreApp != "com.android.settings" { + t.FailNow() } - err = driver.Driver.AppLaunch("com.android.settings") - if err != nil { - t.Fatal(err) - } - - err = driver.Driver.AssertAppForeground(driver.Driver.GetLastLaunchedApp()) + err = driverExt.Driver.AssertAppForeground(driverExt.Driver.GetLastLaunchedApp()) if err != nil { t.Fatal(err) } time.Sleep(2 * time.Second) - _, err = driver.Driver.AppTerminate("com.android.settings") + _, err = driverExt.Driver.AppTerminate("com.android.settings") if err != nil { t.Fatal(err) } - err = driver.Driver.AssertAppForeground("com.android.settings") + err = driverExt.Driver.AssertAppForeground("com.android.settings") if err == nil { t.Fatal(err) } diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 180f2066..e2e56b24 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -623,6 +623,8 @@ type WebDriver interface { GetLastLaunchedApp() string // AssertAppForeground returns nil if the given package is in foreground AssertAppForeground(packageName string) error + // GetForegroundApp returns current foreground app package name + GetForegroundApp() (string, error) // StartCamera Starts a new camera for recording StartCamera() error diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 29902268..be8eecee 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -373,6 +373,10 @@ func (wd *wdaDriver) AssertAppForeground(packageName string) error { return nil } +func (wd *wdaDriver) GetForegroundApp() (string, error) { + return "", nil +} + func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { return wd.TapFloat(float64(x), float64(y), options...) } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 777c4d72..edcd3090 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -28,8 +28,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // loop until target count achieved for { // check if app in foreground - err := dExt.Driver.AssertAppForeground(configs.AppPackageName) - if err != nil { + if err := dExt.Driver.AssertAppForeground(configs.AppPackageName); err != nil { log.Error().Err(err).Str("packageName", configs.AppPackageName).Msg("app is not in foreground") err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) return err From 68dc545f35d371885b227206df12f50b3db0f139 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 21:48:59 +0800 Subject: [PATCH 19/67] feat: implement video crawler --- hrp/pkg/uixt/android_adb_driver.go | 19 ++- hrp/pkg/uixt/android_test.go | 7 +- hrp/pkg/uixt/demo/main_test.go | 11 +- hrp/pkg/uixt/interface.go | 14 ++- hrp/pkg/uixt/ios_driver.go | 4 +- hrp/pkg/uixt/video_crawler.go | 191 ++++++++++++++++++++++++----- hrp/pkg/uixt/video_crawler_test.go | 8 +- 7 files changed, 208 insertions(+), 46 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index d26abffe..988b3c76 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -367,22 +367,22 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { return errors.New("package name is not given") } - foreApp, err := ad.GetForegroundApp() + app, err := ad.GetForegroundApp() if err != nil { return err } - if foreApp != packageName { + if app.BundleId != packageName { return errors.New("app is not in foreground") } return nil } -func (ad *adbDriver) GetForegroundApp() (packageName string, err error) { +func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { // adb shell dumpsys activity activities | grep mResumedActivity output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") - return "", errors.Wrap(code.AndroidShellExecError, err.Error()) + return AppInfo{}, errors.Wrap(code.AndroidShellExecError, err.Error()) } lines := strings.Split(string(output), "\n") @@ -394,11 +394,18 @@ func (ad *adbDriver) GetForegroundApp() (packageName string, err error) { for _, str := range strs { if strings.Contains(str, "/") { // com.android.settings/.Settings - return strings.Split(str, "/")[0], nil + s := strings.Split(str, "/") + app := AppInfo{ + AppBaseInfo: AppBaseInfo{ + BundleId: s[0], + Activity: s[1], + }, + } + return app, nil } } } } - return "", errors.New("get foreground app failed") + return AppInfo{}, errors.New("get foreground app failed") } diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index 34ba6d10..a5061ee9 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -355,9 +355,12 @@ func TestDriver_IsAppInForeground(t *testing.T) { err := driverExt.Driver.AppLaunch("com.android.settings") checkErr(t, err) - foreApp, err := driverExt.Driver.GetForegroundApp() + app, err := driverExt.Driver.GetForegroundApp() checkErr(t, err) - if foreApp != "com.android.settings" { + if app.BundleId != "com.android.settings" { + t.FailNow() + } + if app.Activity != ".Settings" { t.FailNow() } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 1ff036ce..b2610e0e 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/rs/zerolog/log" + "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" ) @@ -32,7 +34,14 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) + // take screenshot and get screen texts by OCR + texts, err := driverExt.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + t.Fatal(err) + } + + points, err := texts.FindTexts([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index e2e56b24..95d71611 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -251,8 +251,9 @@ type AppInfo struct { } type AppBaseInfo struct { - Pid int `json:"pid"` - BundleId string `json:"bundleId"` + Pid int `json:"pid,omitempty"` + BundleId string `json:"bundleId"` // package name for android + Activity string // view controller for ios } type AppState int @@ -579,6 +580,11 @@ type Device interface { StopPcap() string } +type ForegroundApp struct { + PackageName string + Activity string +} + // WebDriver defines methods supported by WebDriver drivers. type WebDriver interface { // NewSession starts a new session and returns the SessionInfo. @@ -623,8 +629,8 @@ type WebDriver interface { GetLastLaunchedApp() string // AssertAppForeground returns nil if the given package is in foreground AssertAppForeground(packageName string) error - // GetForegroundApp returns current foreground app package name - GetForegroundApp() (string, error) + // GetForegroundApp returns current foreground app package name and activity name + GetForegroundApp() (app AppInfo, err error) // StartCamera Starts a new camera for recording StartCamera() error diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index be8eecee..6acd6a99 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -373,8 +373,8 @@ func (wd *wdaDriver) AssertAppForeground(packageName string) error { return nil } -func (wd *wdaDriver) GetForegroundApp() (string, error) { - return "", nil +func (wd *wdaDriver) GetForegroundApp() (app AppInfo, err error) { + return AppInfo{}, nil } func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index edcd3090..8f86a809 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -1,36 +1,138 @@ package uixt import ( + "fmt" + "strings" "time" "github.com/pkg/errors" "github.com/rs/zerolog/log" - - "github.com/httprunner/httprunner/v4/hrp/internal/code" ) +type VideoStat struct { + FeedCount int `json:"feed_count"` + LiveCount int `json:"live_count"` +} + +func (s *VideoStat) IsTargetAchieved(target *VideoStat) bool { + log.Info(). + Interface("current", s). + Interface("target", target). + Msg("current video stat") + if s.FeedCount < target.FeedCount { + return false + } + if s.LiveCount < target.LiveCount { + return false + } + return true +} + type VideoCrawlerConfigs struct { AppPackageName string `json:"app_package_name"` - TargetFeedCount int `json:"target_feed_count"` - TargetLiveCount int `json:"target_live_count"` + TargetCount VideoStat `json:"target_count"` +} + +var androidActivities = map[string]map[string]string{ + // DY + "com.ss.android.ugc.aweme": { + "feed": ".splash.SplashActivity", + "live": ".live.LivePlayActivity", + }, + // KS + "com.smile.gifmaker": { + "feed": "com.yxcorp.gifshow.HomeActivity", + "live": "com.kuaishou.live.core.basic.activity.LiveSlideActivity", + }, +} + +type LiveCrawler struct { + driver *DriverExt + configs *VideoCrawlerConfigs // target video count + currentStat *VideoStat // current video stat +} + +func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) { + // 预览流入口 + points, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) + if err == nil { + return points[0], true + } + + // TODO: 头像入口 + + return PointF{}, false +} + +// run live video crawler +func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { + log.Info().Msg("enter live room") + if err := driver.TapAbsXY(enterPoint.X, enterPoint.Y); err != nil { + log.Error().Err(err).Msg("tap live video failed") + return err + } + time.Sleep(5 * time.Second) + + for l.currentStat.LiveCount < l.configs.TargetCount.LiveCount { + // check if entered live room + if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { + return err + } + + log.Info(). + Int("count", l.currentStat.LiveCount). + Int("target", l.configs.TargetCount.LiveCount). + Msg("current live count") + + // swipe to next live video + err := l.driver.SwipeUp() + if err != nil { + log.Error().Err(err).Msg("swipe up failed") + return err + } + time.Sleep(2 * time.Second) + l.currentStat.LiveCount++ + } + + log.Info().Msg("live count achieved, exit live room") + + return l.exitLiveRoom() +} + +func (l *LiveCrawler) exitLiveRoom() error { + // FIXME: exit live room + for i := 0; i < 5; i++ { + l.driver.SwipeRelative(0, 0.5, 0.5, 0.5) + time.Sleep(2 * time.Second) + + // check if back to feed page + if err := l.driver.assertActivity(l.configs.AppPackageName, "feed"); err == nil { + return nil + } + } + return errors.New("exit live room failed") } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // launch app - if configs.AppPackageName != "" { - if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { - return err - } - time.Sleep(5 * time.Second) + if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { + return err + } + time.Sleep(5 * time.Second) + + currVideoStat := &VideoStat{} + liveCrawler := LiveCrawler{ + driver: dExt, + configs: configs, + currentStat: currVideoStat, } // loop until target count achieved + // the main loop is feed crawler for { - // check if app in foreground - if err := dExt.Driver.AssertAppForeground(configs.AppPackageName); err != nil { - log.Error().Err(err).Str("packageName", configs.AppPackageName).Msg("app is not in foreground") - err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) + // check if feed page + if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { return err } @@ -41,36 +143,69 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return err } - // check if text popup exists - if isTextPopup(texts) { - log.Info().Msg("text popup found") + // automatic handling of pop-up windows + if err := dExt.autoPopupHandler(texts); err != nil { + log.Error().Err(err).Msg("auto handle popup failed") + return err } - // check if live video - if isLiveVideo(texts) { + // check if live video && run live crawler + if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") + if liveCrawler.currentStat.LiveCount < configs.TargetCount.LiveCount { + if err := liveCrawler.Run(dExt, enterPoint); err != nil { + return err + } + } } - // assert feed video type + // check if target count achieved + if currVideoStat.IsTargetAchieved(&configs.TargetCount) { + log.Info().Msg("target count achieved, exit crawler") + break + } - // swipe to next video + // swipe to next feed video + log.Info().Msg("swipe to next feed video") if err = dExt.SwipeUp(); err != nil { log.Error().Err(err).Msg("swipe up failed") return err } - time.Sleep(5 * time.Second) + currVideoStat.FeedCount++ } - // return nil + return nil } -func isTextPopup(texts OCRTexts) bool { +func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error { + log.Debug().Str("pacakge_name", pacakgeName). + Str("activity_type", activityType).Msg("assert activity") + app, err := dExt.Driver.GetForegroundApp() + if err != nil { + log.Error().Err(err).Msg("get foreground app failed") + return err + } + + if app.BundleId != pacakgeName { + return fmt.Errorf("app %s is not in foreground", pacakgeName) + } + + if activities, ok := androidActivities[app.BundleId]; ok { + if activity, ok := activities[activityType]; ok { + if strings.HasSuffix(app.Activity, activity) { + return nil + } + } + } + + log.Error().Interface("app", app.AppBaseInfo).Msg("app activity not match") + return fmt.Errorf("%s activity is not in foreground", activityType) +} + +func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { texts.FindTexts([]string{"确定", "取消"}) - return false -} -func isLiveVideo(texts OCRTexts) bool { - _, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) - return err == nil + // log.Warn().Msg("text popup found") + return nil } diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index d849395e..1a40e429 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -9,9 +9,11 @@ func TestVideoCrawler(t *testing.T) { configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", + TargetCount: VideoStat{ + FeedCount: 5, + LiveCount: 3, + }, } err := driverExt.VideoCrawler(configs) - if err != nil { - t.Fatal(err) - } + checkErr(t, err) } From 824fbe393d65f118ef1cb20fa845f2c785d30535 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 22:23:50 +0800 Subject: [PATCH 20/67] feat: add auto popup handler --- hrp/pkg/uixt/video_crawler.go | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 8f86a809..db4072c0 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -45,6 +45,7 @@ var androidActivities = map[string]map[string]string{ "feed": "com.yxcorp.gifshow.HomeActivity", "live": "com.kuaishou.live.core.basic.activity.LiveSlideActivity", }, + // TODO: SPH, XHS } type LiveCrawler struct { @@ -111,6 +112,12 @@ func (l *LiveCrawler) exitLiveRoom() error { return nil } } + + // exit live room failed, while video count achieved + if l.currentStat.IsTargetAchieved(&l.configs.TargetCount) { + return nil + } + return errors.New("exit live room failed") } @@ -203,9 +210,31 @@ func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error { return fmt.Errorf("%s activity is not in foreground", activityType) } -func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { - texts.FindTexts([]string{"确定", "取消"}) +// TODO: add more popup texts +var popups = [][]string{ + {"青少年", "我知道了"}, // 青少年弹窗 + {"允许", "拒绝"}, + {"确定", "取消"}, +} - // log.Warn().Msg("text popup found") +func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { + for _, popup := range popups { + if len(popup) != 2 { + continue + } + + points, err := texts.FindTexts([]string{"确定", "取消"}) + if err == nil { + log.Warn().Msg("text popup found") + if err := dExt.TapAbsXY(points[1].X, points[1].Y); err != nil { + log.Error().Err(err).Msg("tap popup failed") + return err + } + // tap popup success + return nil + } + } + + // no popup found return nil } From 6b764c5649f41def8e06233c33ace363e7642b89 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 29 Apr 2023 14:53:13 +0800 Subject: [PATCH 21/67] fix: video crawler --- hrp/pkg/uixt/video_crawler.go | 65 +++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index db4072c0..9394f2fb 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -14,18 +14,26 @@ type VideoStat struct { LiveCount int `json:"live_count"` } -func (s *VideoStat) IsTargetAchieved(target *VideoStat) bool { +func (s *VideoStat) isFeedTargetAchieved(target *VideoStat) bool { log.Info(). - Interface("current", s). - Interface("target", target). - Msg("current video stat") - if s.FeedCount < target.FeedCount { - return false - } - if s.LiveCount < target.LiveCount { - return false - } - return true + Int("count", s.FeedCount). + Int("target", target.FeedCount). + Msg("current feed count") + + return s.FeedCount >= target.FeedCount +} + +func (s *VideoStat) isLiveTargetAchieved(target *VideoStat) bool { + log.Info(). + Int("count", s.LiveCount). + Int("target", target.LiveCount). + Msg("current live count") + + return s.LiveCount >= target.LiveCount +} + +func (s *VideoStat) isTargetAchieved(target *VideoStat) bool { + return s.isFeedTargetAchieved(target) && s.isLiveTargetAchieved(target) } type VideoCrawlerConfigs struct { @@ -75,24 +83,24 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } time.Sleep(5 * time.Second) - for l.currentStat.LiveCount < l.configs.TargetCount.LiveCount { - // check if entered live room + for !l.currentStat.isLiveTargetAchieved(&l.configs.TargetCount) { + // check if live room if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { return err } - log.Info(). - Int("count", l.currentStat.LiveCount). - Int("target", l.configs.TargetCount.LiveCount). - Msg("current live count") - // swipe to next live video err := l.driver.SwipeUp() + // TODO: sleep custom random time + time.Sleep(15 * time.Second) if err != nil { log.Error().Err(err).Msg("swipe up failed") - return err + // TODO: retry maximum 3 times + continue } - time.Sleep(2 * time.Second) + + // TODO: check live type + l.currentStat.LiveCount++ } @@ -114,7 +122,7 @@ func (l *LiveCrawler) exitLiveRoom() error { } // exit live room failed, while video count achieved - if l.currentStat.IsTargetAchieved(&l.configs.TargetCount) { + if l.currentStat.isTargetAchieved(&l.configs.TargetCount) { return nil } @@ -159,15 +167,22 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // check if live video && run live crawler if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") - if liveCrawler.currentStat.LiveCount < configs.TargetCount.LiveCount { + if !liveCrawler.currentStat.isLiveTargetAchieved(&configs.TargetCount) { if err := liveCrawler.Run(dExt, enterPoint); err != nil { - return err + log.Error().Err(err).Msg("run live crawler failed, continue") + continue } } } + // TODO: check feed type + + currVideoStat.FeedCount++ + // TODO: sleep custom random time + time.Sleep(5 * time.Second) + // check if target count achieved - if currVideoStat.IsTargetAchieved(&configs.TargetCount) { + if currVideoStat.isTargetAchieved(&configs.TargetCount) { log.Info().Msg("target count achieved, exit crawler") break } @@ -178,8 +193,6 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Error().Err(err).Msg("swipe up failed") return err } - time.Sleep(5 * time.Second) - currVideoStat.FeedCount++ } return nil From a69db8500cfb4870aebfc372efe2390e2701507a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 30 Apr 2023 00:13:18 +0800 Subject: [PATCH 22/67] feat: save video stat to summary --- hrp/pkg/uixt/ext.go | 2 ++ hrp/pkg/uixt/video_crawler.go | 6 +++++- hrp/step_mobile_ui.go | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 3c57ffb6..901d2553 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -50,6 +50,8 @@ type cacheStepData struct { ScreenShots []string // cache step screenshot ocr results, key is image path, value is dumped OCRTexts OcrResults map[string]string + // cache feed/live video stat + VideoStat *VideoStat } type DriverExt struct { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 9394f2fb..2b129c2a 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -130,13 +130,17 @@ func (l *LiveCrawler) exitLiveRoom() error { } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { + currVideoStat := &VideoStat{} + defer func() { + dExt.cacheStepData.VideoStat = currVideoStat + }() + // launch app if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { return err } time.Sleep(5 * time.Second) - currVideoStat := &VideoStat{} liveCrawler := LiveCrawler{ driver: dExt, configs: configs, diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 2ba33467..2a4f1f6a 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -619,6 +619,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err cacheData := uiDriver.GetStepCacheData() attachments["screenshots"] = cacheData.ScreenShots attachments["ocr_results"] = cacheData.OcrResults + attachments["video_stat"] = cacheData.VideoStat stepResult.Attachments = attachments }() From 6bc5e11e2deb98709929077beb772867affc6fef Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 30 Apr 2023 00:40:44 +0800 Subject: [PATCH 23/67] fix: exit live room --- .../uitest/demo_android_video_crawler.json | 59 +++++++++++++++++++ .../uitest/demo_android_video_crawler_test.go | 47 +++++++++++++++ hrp/pkg/uixt/android_adb_driver.go | 6 +- hrp/pkg/uixt/android_test.go | 2 +- hrp/pkg/uixt/interface.go | 8 ++- hrp/pkg/uixt/video_crawler.go | 28 +++++---- 6 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 examples/uitest/demo_android_video_crawler.json create mode 100644 examples/uitest/demo_android_video_crawler_test.go diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json new file mode 100644 index 00000000..380a6377 --- /dev/null +++ b/examples/uitest/demo_android_video_crawler.json @@ -0,0 +1,59 @@ +{ + "config": { + "name": "抓取抖音视频信息", + "variables": { + "device": "${ENV(SerialNumber)}" + }, + "android": [ + { + "serial": "$device" + } + ] + }, + "teststeps": [ + { + "name": "滑动消费 feed 至少 10 个，live 至少 3 个；滑动过程中，70% 随机间隔 0-5s，30% 随机间隔 5-10s", + "android": { + "actions": [ + { + "method": "video_crawler", + "params": { + "app_package_name": "com.ss.android.ugc.aweme", + "sleep_random": [ + 0, + 5, + 0.7, + 5, + 10, + 0.3 +], + "target_count": { + "feed_count": 5, + "live_count": 3 +} + } + } + ] + } + }, + { + "name": "exit", + "android": { + "actions": [ + { + "method": "app_terminate", + "params": "com.ss.android.ugc.aweme" + } + ] + }, + "validate": [ + { + "check": "ui_foreground_app", + "assert": "not_equal", + "expect": "com.ss.android.ugc.aweme", + "msg": "app [com.ss.android.ugc.aweme] should not be in foreground" + } + ] + } + ] +} diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go new file mode 100644 index 00000000..c6f81bc0 --- /dev/null +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -0,0 +1,47 @@ +//go:build localtest + +package uitest + +import ( + "testing" + + "github.com/httprunner/httprunner/v4/hrp" + "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" +) + +func TestAndroidVideoCrawlerTest(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("抓取抖音视频信息"). + WithVariables(map[string]interface{}{ + "device": "${ENV(SerialNumber)}", + }). + SetAndroid(uixt.WithSerialNumber("$device")), + TestSteps: []hrp.IStep{ + hrp.NewStep("滑动消费 feed 至少 10 个，live 至少 3 个；滑动过程中，70% 随机间隔 0-5s，30% 随机间隔 5-10s"). + Android(). + VideoCrawler(map[string]interface{}{ + "app_package_name": "com.ss.android.ugc.aweme", + "target_count": map[string]interface{}{ + "feed_count": 5, + "live_count": 3, + }, + "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, + }), + hrp.NewStep("exit"). + Android(). + AppTerminate("com.ss.android.ugc.aweme"). + Validate(). + AssertAppNotInForeground("com.ss.android.ugc.aweme"), + }, + } + + if err := testCase.Dump2JSON("demo_android_video_crawler.json"); err != nil { + t.Fatal(err) + } + + runner := hrp.NewRunner(t).SetSaveTests(true) + err := runner.Run(testCase) + if err != nil { + t.Fatal(err) + } +} diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 988b3c76..9bc6992c 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -371,7 +371,7 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { if err != nil { return err } - if app.BundleId != packageName { + if app.PackageName != packageName { return errors.New("app is not in foreground") } return nil @@ -397,8 +397,8 @@ func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { s := strings.Split(str, "/") app := AppInfo{ AppBaseInfo: AppBaseInfo{ - BundleId: s[0], - Activity: s[1], + PackageName: s[0], + Activity: s[1], }, } return app, nil diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index a5061ee9..46ab5a0d 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -357,7 +357,7 @@ func TestDriver_IsAppInForeground(t *testing.T) { app, err := driverExt.Driver.GetForegroundApp() checkErr(t, err) - if app.BundleId != "com.android.settings" { + if app.PackageName != "com.android.settings" { t.FailNow() } if app.Activity != ".Settings" { diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 95d71611..d996dfb0 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -251,9 +251,11 @@ type AppInfo struct { } type AppBaseInfo struct { - Pid int `json:"pid,omitempty"` - BundleId string `json:"bundleId"` // package name for android - Activity string // view controller for ios + Pid int `json:"pid,omitempty"` + BundleId string `json:"bundleId,omitempty"` // ios package name + ViewController string `json:"viewController,omitempty"` // ios view controller + PackageName string `json:"packageName,omitempty"` // android package name + Activity string `json:"activity,omitempty"` // android activity } type AppState int diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 2b129c2a..8082f5a9 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -89,8 +89,17 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { return err } + // take screenshot and get screen texts by OCR + _, err := l.driver.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + continue + } + + // TODO: check live type + // swipe to next live video - err := l.driver.SwipeUp() + err = l.driver.SwipeUp() // TODO: sleep custom random time time.Sleep(15 * time.Second) if err != nil { @@ -110,9 +119,8 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } func (l *LiveCrawler) exitLiveRoom() error { - // FIXME: exit live room - for i := 0; i < 5; i++ { - l.driver.SwipeRelative(0, 0.5, 0.5, 0.5) + for i := 0; i < 3; i++ { + l.driver.SwipeRelative(0.1, 0.5, 0.9, 0.5) time.Sleep(2 * time.Second) // check if back to feed page @@ -159,7 +167,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { texts, err := dExt.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") - return err + continue } // automatic handling of pop-up windows @@ -202,8 +210,8 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return nil } -func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error { - log.Debug().Str("pacakge_name", pacakgeName). +func (dExt *DriverExt) assertActivity(packageName, activityType string) error { + log.Debug().Str("pacakge_name", packageName). Str("activity_type", activityType).Msg("assert activity") app, err := dExt.Driver.GetForegroundApp() if err != nil { @@ -211,11 +219,11 @@ func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error { return err } - if app.BundleId != pacakgeName { - return fmt.Errorf("app %s is not in foreground", pacakgeName) + if app.PackageName != packageName { + return fmt.Errorf("app %s is not in foreground", packageName) } - if activities, ok := androidActivities[app.BundleId]; ok { + if activities, ok := androidActivities[app.PackageName]; ok { if activity, ok := activities[activityType]; ok { if strings.HasSuffix(app.Activity, activity) { return nil From bb52abf5795eb1ccc0dab003d5694e336edcaeb4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 30 Apr 2023 01:17:40 +0800 Subject: [PATCH 24/67] refactor: VideoCrawlerConfigs struct --- .../uitest/demo_android_video_crawler.json | 14 +++-- .../uitest/demo_android_video_crawler_test.go | 11 ++-- hrp/pkg/uixt/action.go | 3 +- hrp/pkg/uixt/video_crawler.go | 56 +++++++++++++------ hrp/pkg/uixt/video_crawler_test.go | 11 +++- 5 files changed, 65 insertions(+), 30 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 380a6377..94f060a2 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -19,7 +19,8 @@ "method": "video_crawler", "params": { "app_package_name": "com.ss.android.ugc.aweme", - "sleep_random": [ + "feed": { + "sleep_random": [ 0, 5, 0.7, @@ -27,9 +28,14 @@ 10, 0.3 ], - "target_count": { - "feed_count": 5, - "live_count": 3 + "target_count": 5 +}, + "live": { + "sleep_random": [ + 15, + 20 +], + "target_count": 3 } } } diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index c6f81bc0..9655808b 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -21,11 +21,14 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { Android(). VideoCrawler(map[string]interface{}{ "app_package_name": "com.ss.android.ugc.aweme", - "target_count": map[string]interface{}{ - "feed_count": 5, - "live_count": 3, + "feed": map[string]interface{}{ + "target_count": 5, + "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, + }, + "live": map[string]interface{}{ + "target_count": 3, + "sleep_random": []float64{15, 20}, }, - "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, }), hrp.NewStep("exit"). Android(). diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index d80126f2..7518c327 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -417,7 +417,8 @@ func sleepRandom(params []interface{}) error { accProb += s.weight / totalProb if r < accProb { n := s.min + rand.Float64()*(s.max-s.min) - log.Info().Float64("duration", n).Msg("sleep random seconds") + log.Info().Float64("duration", n). + Interface("strategy_params", params).Msg("sleep random seconds") time.Sleep(time.Duration(n*1000) * time.Millisecond) return nil } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 8082f5a9..02e8324e 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -10,36 +10,49 @@ import ( ) type VideoStat struct { + configs *VideoCrawlerConfigs + FeedCount int `json:"feed_count"` LiveCount int `json:"live_count"` } -func (s *VideoStat) isFeedTargetAchieved(target *VideoStat) bool { +func (s *VideoStat) isFeedTargetAchieved() bool { log.Info(). Int("count", s.FeedCount). - Int("target", target.FeedCount). + Int("target", s.configs.Feed.TargetCount). Msg("current feed count") - return s.FeedCount >= target.FeedCount + return s.FeedCount >= s.configs.Feed.TargetCount } -func (s *VideoStat) isLiveTargetAchieved(target *VideoStat) bool { +func (s *VideoStat) isLiveTargetAchieved() bool { log.Info(). Int("count", s.LiveCount). - Int("target", target.LiveCount). + Int("target", s.configs.Live.TargetCount). Msg("current live count") - return s.LiveCount >= target.LiveCount + return s.LiveCount >= s.configs.Live.TargetCount } -func (s *VideoStat) isTargetAchieved(target *VideoStat) bool { - return s.isFeedTargetAchieved(target) && s.isLiveTargetAchieved(target) +func (s *VideoStat) isTargetAchieved() bool { + return s.isFeedTargetAchieved() && s.isLiveTargetAchieved() +} + +type FeedConfig struct { + TargetCount int `json:"target_count"` + SleepRandom []interface{} `json:"sleep_random"` +} + +type LiveConfig struct { + TargetCount int `json:"target_count"` + SleepRandom []interface{} `json:"sleep_random"` } type VideoCrawlerConfigs struct { AppPackageName string `json:"app_package_name"` - TargetCount VideoStat `json:"target_count"` + Feed FeedConfig `json:"feed"` + Live LiveConfig `json:"live"` } var androidActivities = map[string]map[string]string{ @@ -83,7 +96,7 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } time.Sleep(5 * time.Second) - for !l.currentStat.isLiveTargetAchieved(&l.configs.TargetCount) { + for !l.currentStat.isLiveTargetAchieved() { // check if live room if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { return err @@ -100,14 +113,17 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { // swipe to next live video err = l.driver.SwipeUp() - // TODO: sleep custom random time - time.Sleep(15 * time.Second) if err != nil { log.Error().Err(err).Msg("swipe up failed") // TODO: retry maximum 3 times continue } + // sleep custom random time + if err := sleepRandom(l.configs.Live.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } + // TODO: check live type l.currentStat.LiveCount++ @@ -130,7 +146,7 @@ func (l *LiveCrawler) exitLiveRoom() error { } // exit live room failed, while video count achieved - if l.currentStat.isTargetAchieved(&l.configs.TargetCount) { + if l.currentStat.isTargetAchieved() { return nil } @@ -138,7 +154,9 @@ func (l *LiveCrawler) exitLiveRoom() error { } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { - currVideoStat := &VideoStat{} + currVideoStat := &VideoStat{ + configs: configs, + } defer func() { dExt.cacheStepData.VideoStat = currVideoStat }() @@ -179,7 +197,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // check if live video && run live crawler if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") - if !liveCrawler.currentStat.isLiveTargetAchieved(&configs.TargetCount) { + if !liveCrawler.currentStat.isLiveTargetAchieved() { if err := liveCrawler.Run(dExt, enterPoint); err != nil { log.Error().Err(err).Msg("run live crawler failed, continue") continue @@ -190,11 +208,13 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // TODO: check feed type currVideoStat.FeedCount++ - // TODO: sleep custom random time - time.Sleep(5 * time.Second) + // sleep custom random time + if err := sleepRandom(configs.Feed.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } // check if target count achieved - if currVideoStat.isTargetAchieved(&configs.TargetCount) { + if currVideoStat.isTargetAchieved() { log.Info().Msg("target count achieved, exit crawler") break } diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index 1a40e429..e6f78c18 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -9,9 +9,14 @@ func TestVideoCrawler(t *testing.T) { configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", - TargetCount: VideoStat{ - FeedCount: 5, - LiveCount: 3, + + Feed: FeedConfig{ + TargetCount: 5, + SleepRandom: []interface{}{0, 5, 0.7, 5, 10, 0.3}, + }, + Live: LiveConfig{ + TargetCount: 3, + SleepRandom: []interface{}{15, 20}, }, } err := driverExt.VideoCrawler(configs) From 3a404c8372e625b2caac618fcd61338d9e07673a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 1 May 2023 15:09:28 +0800 Subject: [PATCH 25/67] refactor: merge ActionOption with DataOption --- examples/uitest/demo_kuaishou_test.go | 2 +- examples/worldcup/main_test.go | 2 +- hrp/pkg/uixt/action.go | 293 ++++++++++++++++++-------- hrp/pkg/uixt/android_adb_driver.go | 26 +-- hrp/pkg/uixt/android_uia2_driver.go | 26 +-- hrp/pkg/uixt/drag.go | 2 +- hrp/pkg/uixt/ext.go | 2 +- hrp/pkg/uixt/interface.go | 159 +------------- hrp/pkg/uixt/ios_driver.go | 26 +-- hrp/pkg/uixt/ocr_vedem.go | 16 +- hrp/pkg/uixt/opencv.go | 2 +- hrp/pkg/uixt/opencv_off.go | 2 +- hrp/pkg/uixt/swipe.go | 96 +++------ hrp/pkg/uixt/swipe_test.go | 21 +- hrp/pkg/uixt/tap.go | 24 +-- hrp/step_mobile_ui.go | 151 ++++++------- 16 files changed, 387 insertions(+), 463 deletions(-) diff --git a/examples/uitest/demo_kuaishou_test.go b/examples/uitest/demo_kuaishou_test.go index d0d47e27..2cc179aa 100644 --- a/examples/uitest/demo_kuaishou_test.go +++ b/examples/uitest/demo_kuaishou_test.go @@ -39,7 +39,7 @@ func TestAndroidKuaiShouFeedCardLive(t *testing.T) { uixt.WithCustomDirection(0.9, 0.7, 0.9, 0.3), uixt.WithScope(0.2, 0.5, 0.8, 0.8), uixt.WithMaxRetryTimes(20), - uixt.WithWaitTime(60), + uixt.WithInterval(60), uixt.WithIdentifier("click_live"), ), hrp.NewStep("等待1分钟"). diff --git a/examples/worldcup/main_test.go b/examples/worldcup/main_test.go index 03fa9e9c..ce54be59 100644 --- a/examples/worldcup/main_test.go +++ b/examples/worldcup/main_test.go @@ -89,7 +89,7 @@ func TestIOSDouyinWorldCupLive(t *testing.T) { uixt.WithMaxRetryTimes(5), uixt.WithCustomDirection(0.4, 0.07, 0.6, 0.07), // 滑动 tab，从左到右，解决「世界杯」被遮挡的问题 uixt.WithScope(0, 0, 1, 0.15), // 限定 tab 区域 - uixt.WithWaitTime(1), + uixt.WithInterval(1), ), hrp.NewStep("点击进入赛程晋级"). Loop(5). // 重复执行 5 次 diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 7518c327..9778b623 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -3,6 +3,7 @@ package uixt import ( "encoding/json" "fmt" + "math" "math/rand" "time" @@ -61,116 +62,260 @@ const ( ) type MobileAction struct { - Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` - Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` - - Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log - MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times - WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second - Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action - Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action - Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action - IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` + Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"` + Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` + Options *ActionOptions `json:"options,omitempty" yaml:"options,omitempty"` } -type ActionOption func(o *MobileAction) +type ActionOptions struct { + // log + Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log + + // control related + MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found + Interval float64 `json:"interval,omitempty" yaml:"interval,omitempty"` // interval between retries in seconds + PressDuration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action + Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action + Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action + Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty"` + + // scope related + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope + AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // used by ocr to get text position in the scope + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + + // element related + Text string `json:"text,omitempty" yaml:"text,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + + // set custiom options such as textview, id, description + Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"` +} + +func (o *ActionOptions) Options() []ActionOption { + options := make([]ActionOption, 0) + + if o.Identifier != "" { + options = append(options, WithIdentifier(o.Identifier)) + } + + if o.MaxRetryTimes != 0 { + options = append(options, WithMaxRetryTimes(o.MaxRetryTimes)) + } + if o.IgnoreNotFoundError { + options = append(options, WithIgnoreNotFoundError(true)) + } + if o.Interval != 0 { + options = append(options, WithInterval(o.Interval)) + } + if o.PressDuration != 0 { + options = append(options, WithPressDuration(o.PressDuration)) + } + if o.Steps != 0 { + options = append(options, WithSteps(o.Steps)) + } + + switch v := o.Direction.(type) { + case string: + options = append(options, WithDirection(v)) + case []float64: + options = append(options, WithCustomDirection( + v[0], v[1], + v[2], v[3], + )) + case []interface{}: + // loaded from json case + // custom direction: [fromX, fromY, toX, toY] + sx, _ := builtin.Interface2Float64(v[0]) + sy, _ := builtin.Interface2Float64(v[1]) + ex, _ := builtin.Interface2Float64(v[2]) + ey, _ := builtin.Interface2Float64(v[3]) + options = append(options, WithCustomDirection( + sx, sy, + ex, ey, + )) + } + + if o.Timeout != 0 { + options = append(options, WithTimeout(o.Timeout)) + } + if o.Frequency != 0 { + options = append(options, WithFrequency(o.Frequency)) + } + if len(o.Scope) != 4 { + options = append(options, WithScope(0, 0, 1, 1)) + } + if len(o.AbsScope) != 4 { + o.AbsScope = []int{0, 0, math.MaxInt64, math.MaxInt64} + } + if len(o.Offset) == 2 { + options = append(options, WithOffset(o.Offset[0], o.Offset[1])) + } + + // custom options + if o.Custom != nil { + for k, v := range o.Custom { + options = append(options, WithCustomOption(k, v)) + } + } + + return options +} + +func NewActionOptions(options ...ActionOption) *ActionOptions { + actionOptions := &ActionOptions{ + Custom: make(map[string]interface{}), + } + for _, option := range options { + option(actionOptions) + } + return actionOptions +} + +func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} { + actionOptions := NewActionOptions(options...) + + // custom options + for k, v := range actionOptions.Custom { + data[k] = v + } + + if actionOptions.Identifier != "" { + data["log"] = map[string]interface{}{ + "enable": true, + "data": actionOptions.Identifier, + } + } + + // handle point offset + if len(actionOptions.Offset) == 2 { + if x, ok := data["x"]; ok { + xf, _ := builtin.Interface2Float64(x) + data["x"] = xf + float64(actionOptions.Offset[0]) + } + if y, ok := data["y"]; ok { + yf, _ := builtin.Interface2Float64(y) + data["y"] = yf + float64(actionOptions.Offset[1]) + } + } + + if actionOptions.Steps > 0 { + data["steps"] = actionOptions.Steps + } + if _, ok := data["steps"]; !ok { + data["steps"] = 12 // default steps + } + + if actionOptions.PressDuration > 0 { + data["duration"] = actionOptions.PressDuration + } + if _, ok := data["duration"]; !ok { + data["duration"] = 0 // default duration + } + + if actionOptions.Frequency > 0 { + data["frequency"] = actionOptions.Frequency + } + if _, ok := data["frequency"]; !ok { + data["frequency"] = 60 // default frequency + } + + if _, ok := data["isReplace"]; !ok { + data["isReplace"] = true // default true + } + + return data +} + +type ActionOption func(o *ActionOptions) + +func WithCustomOption(key string, value interface{}) ActionOption { + return func(o *ActionOptions) { + o.Custom[key] = value + } +} func WithIdentifier(identifier string) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Identifier = identifier } } func WithIndex(index int) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Index = index } } -func WithWaitTime(sec float64) ActionOption { - return func(o *MobileAction) { - o.WaitTime = sec +func WithInterval(sec float64) ActionOption { + return func(o *ActionOptions) { + o.Interval = sec } } -func WithDuration(duration float64) ActionOption { - return func(o *MobileAction) { - o.Duration = duration +func WithPressDuration(duration float64) ActionOption { + return func(o *ActionOptions) { + o.PressDuration = duration } } func WithSteps(steps int) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Steps = steps } } // WithDirection inputs direction (up, down, left, right) func WithDirection(direction string) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Direction = direction } } // WithCustomDirection inputs sx, sy, ex, ey func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Direction = []float64{sx, sy, ex, ey} } } // WithScope inputs area of [(x1,y1), (x2,y2)] func WithScope(x1, y1, x2, y2 float64) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Scope = []float64{x1, y1, x2, y2} } } func WithOffset(offsetX, offsetY int) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Offset = []int{offsetX, offsetY} } } -func WithText(text string) ActionOption { - return func(o *MobileAction) { - o.Text = text - } -} - -func WithID(id string) ActionOption { - return func(o *MobileAction) { - o.ID = id - } -} - -func WithDescription(description string) ActionOption { - return func(o *MobileAction) { - o.Description = description +func WithFrequency(frequency int) ActionOption { + return func(o *ActionOptions) { + o.Frequency = frequency } } func WithMaxRetryTimes(maxRetryTimes int) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.MaxRetryTimes = maxRetryTimes } } func WithTimeout(timeout int) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.Timeout = timeout } } func WithIgnoreNotFoundError(ignoreError bool) ActionOption { - return func(o *MobileAction) { + return func(o *ActionOptions) { o.IgnoreNotFoundError = ignoreError } } @@ -190,13 +335,13 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { ACTION_AppLaunch, action.Params) case ACTION_SwipeToTapApp: if appName, ok := action.Params.(string); ok { - return dExt.swipeToTapApp(appName, action) + return dExt.swipeToTapApp(appName, action.Options.Options()...) } return fmt.Errorf("invalid %s params, should be app name(string), got %v", ACTION_SwipeToTapApp, action.Params) case ACTION_SwipeToTapText: if text, ok := action.Params.(string); ok { - return dExt.swipeToTapTexts([]string{text}, action) + return dExt.swipeToTapTexts([]string{text}, action.Options.Options()...) } return fmt.Errorf("invalid %s params, should be app text(string), got %v", ACTION_SwipeToTapText, action.Params) @@ -209,7 +354,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Params = textList } if texts, ok := action.Params.([]string); ok { - return dExt.swipeToTapTexts(texts, action) + return dExt.swipeToTapTexts(texts, action.Options.Options()...) } return fmt.Errorf("invalid %s params, should be app text([]string), got %v", ACTION_SwipeToTapText, action.Params) @@ -235,7 +380,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier)) + return dExt.TapXY(x, y, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) case ACTION_TapAbsXY: @@ -246,37 +391,32 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} + if len(action.Options.Offset) != 2 { + action.Options.Offset = []int{0, 0} } - return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier), WithDataOffset(action.Offset[0], action.Offset[1])) + return dExt.TapAbsXY(x, y, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) case ACTION_Tap: if param, ok := action.Params.(string); ok { - return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) + return dExt.Tap(param, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} + if len(action.Options.Scope) != 4 { + action.Options.Scope = []float64{0, 0, 1, 1} } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} + if len(action.Options.Offset) != 2 { + action.Options.Offset = []int{0, 0} } - indexOption := WithDataIndex(action.Index) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - identifierOption := WithDataIdentifier(action.Identifier) - IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError) - return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption, offsetOption) + return dExt.TapByOCR(ocrText, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: if imagePath, ok := action.Params.(string); ok { - return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) + return dExt.TapByCV(imagePath, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: @@ -296,27 +436,14 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) case ACTION_Swipe: - swipeAction := dExt.prepareSwipeAction(action) + swipeAction := dExt.prepareSwipeAction(action.Options.Options()...) return swipeAction(dExt) case ACTION_Input: // input text on current active element // append \n to send text with enter // send \b\b\b to delete 3 chars param := fmt.Sprintf("%v", action.Params) - options := []DataOption{} - if action.Text != "" { - options = append(options, WithCustomOption("textview", action.Text)) - } - if action.ID != "" { - options = append(options, WithCustomOption("id", action.ID)) - } - if action.Description != "" { - options = append(options, WithCustomOption("description", action.Description)) - } - if action.Identifier != "" { - options = append(options, WithDataIdentifier(action.Identifier)) - } - return dExt.Driver.Input(param, options...) + return dExt.Driver.Input(param, action.Options.Options()...) case ACTION_Back: return dExt.Driver.PressBack() case ACTION_Sleep: diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 9bc6992c..858ad229 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -81,7 +81,7 @@ func (ad *adbDriver) Scale() (scale float64, err error) { } // PressBack simulates a short press on the BACK button. -func (ad *adbDriver) PressBack(options ...DataOption) (err error) { +func (ad *adbDriver) PressBack(options ...ActionOption) (err error) { // adb shell input keyevent 4 _, err = ad.adbClient.RunShellCommand("input", "keyevent", fmt.Sprintf("%d", KCBack)) return @@ -185,16 +185,16 @@ func (ad *adbDriver) AppTerminate(packageName string) (successful bool, err erro return true, nil } -func (ad *adbDriver) Tap(x, y int, options ...DataOption) error { +func (ad *adbDriver) Tap(x, y int, options ...ActionOption) error { return ad.TapFloat(float64(x), float64(y), options...) } -func (ad *adbDriver) TapFloat(x, y float64, options ...DataOption) (err error) { - dataOptions := NewDataOptions(options...) +func (ad *adbDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) - if len(dataOptions.Offset) == 2 { - x += float64(dataOptions.Offset[0]) - y += float64(dataOptions.Offset[1]) + if len(actionOptions.Offset) == 2 { + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) } // adb shell input tap x y @@ -221,20 +221,20 @@ func (ad *adbDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err err return } -func (ad *adbDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) error { +func (ad *adbDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) error { return ad.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (ad *adbDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) { +func (ad *adbDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { err = errDriverNotImplemented return } -func (ad *adbDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { +func (ad *adbDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) error { return ad.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error { +func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { // adb shell input swipe fromX fromY toX toY _, err := ad.adbClient.RunShellCommand( "input", "swipe", @@ -263,13 +263,13 @@ func (ad *adbDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe return } -func (ad *adbDriver) SendKeys(text string, options ...DataOption) (err error) { +func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error) { // adb shell input text _, err = ad.adbClient.RunShellCommand("input", "text", text) return } -func (ad *adbDriver) Input(text string, options ...DataOption) (err error) { +func (ad *adbDriver) Input(text string, options ...ActionOption) (err error) { return ad.SendKeys(text, options...) } diff --git a/hrp/pkg/uixt/android_uia2_driver.go b/hrp/pkg/uixt/android_uia2_driver.go index daf9e8bc..5f426e04 100644 --- a/hrp/pkg/uixt/android_uia2_driver.go +++ b/hrp/pkg/uixt/android_uia2_driver.go @@ -174,7 +174,7 @@ func (ud *uiaDriver) WindowSize() (size Size, err error) { } // PressBack simulates a short press on the BACK button. -func (ud *uiaDriver) PressBack(options ...DataOption) (err error) { +func (ud *uiaDriver) PressBack(options ...ActionOption) (err error) { // register(postHandler, new PressBack("/wd/hub/session/:sessionId/back")) _, err = ud.httpPOST(nil, "/session", ud.sessionId, "back") return @@ -199,18 +199,18 @@ func (ud *uiaDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta, flags ...K return } -func (ud *uiaDriver) Tap(x, y int, options ...DataOption) error { +func (ud *uiaDriver) Tap(x, y int, options ...ActionOption) error { return ud.TapFloat(float64(x), float64(y), options...) } -func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { +func (ud *uiaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) data := map[string]interface{}{ "x": x, "y": y, } // new data options in post data for extra uiautomator configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = ud.httpPOST(newData, "/session", ud.sessionId, "appium/tap") return @@ -240,11 +240,11 @@ func (ud *uiaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err err // the smoothness and speed of the swipe by specifying the number of steps. // Each step execution is throttled to 5 milliseconds per step, so for a 100 // steps, the swipe will take around 0.5 seconds to complete. -func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) error { +func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) error { return ud.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) { +func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -253,7 +253,7 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra uiautomator configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) // register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag")) _, err = ud.httpPOST(newData, "/session", ud.sessionId, "touch/drag") @@ -264,11 +264,11 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp // to determine smoothness and speed. Each step execution is throttled to 5ms // per step. So for a 100 steps, the swipe will take about 1/2 second to complete. // `steps` is the number of move steps sent to the system -func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { +func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) error { return ud.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error { +func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { // register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform")) data := map[string]interface{}{ "startX": fromX, @@ -278,7 +278,7 @@ func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataO } // new data options in post data for extra uiautomator configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err := ud.httpPOST(newData, "/session", ud.sessionId, "touch/perform") return err @@ -327,20 +327,20 @@ func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe return } -func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) { +func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) { // register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys")) // https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85 data := map[string]interface{}{ "text": text, } // new data options in post data for extra uiautomator configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = ud.httpPOST(newData, "/session", ud.sessionId, "keys") return } -func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) { +func (ud *uiaDriver) Input(text string, options ...ActionOption) (err error) { return ud.SendKeys(text, options...) } diff --git a/hrp/pkg/uixt/drag.go b/hrp/pkg/uixt/drag.go index f583f9cf..3f3dd1f6 100644 --- a/hrp/pkg/uixt/drag.go +++ b/hrp/pkg/uixt/drag.go @@ -24,5 +24,5 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs // FIXME: handle offset return dExt.Driver.DragFloat(point.X+xOffset, point.Y+yOffset, toX, toY, - WithDataPressDuration(pressForDuration[0])) + WithPressDuration(pressForDuration[0])) } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 901d2553..40610319 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -200,7 +200,7 @@ func init() { rand.Seed(time.Now().UnixNano()) } -func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) { // click on text, using OCR if !isPathExists(search) { return dExt.FindScreenTextByOCR(search, options...) diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index d996dfb0..d19ea31b 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -2,11 +2,8 @@ package uixt import ( "bytes" - "math" "strings" "time" - - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" ) var ( @@ -431,144 +428,6 @@ type Rect struct { Size } -type DataOptions struct { - Data map[string]interface{} // configurations used by ios/android driver - Scope []int // used by ocr to get text position in the scope - Offset []int // used to tap offset of point - Index int // index of the target element, should start from 1 - IgnoreNotFoundError bool // ignore error if target element not found - MaxRetryTimes int // max retry times if target element not found - Interval float64 // interval between retries in seconds -} - -type DataOption func(data *DataOptions) - -func WithCustomOption(key string, value interface{}) DataOption { - return func(data *DataOptions) { - data.Data[key] = value - } -} - -func WithDataPressDuration(duration float64) DataOption { - return func(data *DataOptions) { - data.Data["duration"] = duration - } -} - -func WithDataSteps(steps int) DataOption { - return func(data *DataOptions) { - data.Data["steps"] = steps - } -} - -func WithDataFrequency(frequency int) DataOption { - return func(data *DataOptions) { - data.Data["frequency"] = frequency - } -} - -func WithDataIndex(index int) DataOption { - return func(data *DataOptions) { - data.Index = index - } -} - -func WithDataScope(x1, x2, y1, y2 int) DataOption { - return func(data *DataOptions) { - data.Scope = []int{x1, x2, y1, y2} - } -} - -func WithDataOffset(offsetX, offsetY int) DataOption { - return func(data *DataOptions) { - data.Offset = []int{offsetX, offsetY} - } -} - -func WithDataIdentifier(identifier string) DataOption { - if identifier == "" { - return func(data *DataOptions) {} - } - return func(data *DataOptions) { - data.Data["log"] = map[string]interface{}{ - "enable": true, - "data": identifier, - } - } -} - -func WithDataIgnoreNotFoundError(ignoreError bool) DataOption { - return func(data *DataOptions) { - data.IgnoreNotFoundError = ignoreError - } -} - -func WithDataMaxRetryTimes(maxRetryTimes int) DataOption { - return func(data *DataOptions) { - data.MaxRetryTimes = maxRetryTimes - } -} - -func WithDataWaitTime(sec float64) DataOption { - return func(data *DataOptions) { - data.Interval = sec - } -} - -func NewDataOptions(options ...DataOption) *DataOptions { - dataOptions := &DataOptions{ - Data: make(map[string]interface{}), - } - for _, option := range options { - option(dataOptions) - } - - if len(dataOptions.Scope) == 0 { - dataOptions.Scope = []int{0, 0, math.MaxInt64, math.MaxInt64} // default scope - } - return dataOptions -} - -func NewData(data map[string]interface{}, options ...DataOption) map[string]interface{} { - dataOptions := NewDataOptions(options...) - - // merge with data options - for k, v := range dataOptions.Data { - data[k] = v - } - - // handle point offset - if len(dataOptions.Offset) == 2 { - if x, ok := data["x"]; ok { - xf, _ := builtin.Interface2Float64(x) - data["x"] = xf + float64(dataOptions.Offset[0]) - } - if y, ok := data["y"]; ok { - yf, _ := builtin.Interface2Float64(y) - data["y"] = yf + float64(dataOptions.Offset[1]) - } - } - - // add default options - if _, ok := data["steps"]; !ok { - data["steps"] = 12 // default steps - } - - if _, ok := data["duration"]; !ok { - data["duration"] = 0 // default duration - } - - if _, ok := data["frequency"]; !ok { - data["frequency"] = 60 // default frequency - } - - if _, ok := data["isReplace"]; !ok { - data["isReplace"] = true // default true - } - - return data -} - // current implemeted device: IOSDevice, AndroidDevice type Device interface { UUID() string // ios udid or android serial @@ -640,8 +499,8 @@ type WebDriver interface { StopCamera() error // Tap Sends a tap event at the coordinate. - Tap(x, y int, options ...DataOption) error - TapFloat(x, y float64, options ...DataOption) error + Tap(x, y int, options ...ActionOption) error + TapFloat(x, y float64, options ...ActionOption) error // DoubleTap Sends a double tap event at the coordinate. DoubleTap(x, y int) error @@ -654,12 +513,12 @@ type WebDriver interface { // Drag Initiates a press-and-hold gesture at the coordinate, then drags to another coordinate. // WithPressDurationOption option can be used to set pressForDuration (default to 1 second). - Drag(fromX, fromY, toX, toY int, options ...DataOption) error - DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) error + Drag(fromX, fromY, toX, toY int, options ...ActionOption) error + DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error // Swipe works like Drag, but `pressForDuration` value is 0 - Swipe(fromX, fromY, toX, toY int, options ...DataOption) error - SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error + Swipe(fromX, fromY, toX, toY int, options ...ActionOption) error + SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error // SetPasteboard Sets data to the general pasteboard SetPasteboard(contentType PasteboardType, content string) error @@ -670,16 +529,16 @@ type WebDriver interface { // SendKeys Types a string into active element. There must be element with keyboard focus, // otherwise an error is raised. // WithFrequency option can be used to set frequency of typing (letters per sec). The default value is 60 - SendKeys(text string, options ...DataOption) error + SendKeys(text string, options ...ActionOption) error // Input works like SendKeys - Input(text string, options ...DataOption) error + Input(text string, options ...ActionOption) error // PressButton Presses the corresponding hardware button on the device PressButton(devBtn DeviceButton) error // PressBack Presses the back button - PressBack(options ...DataOption) error + PressBack(options ...ActionOption) error Screenshot() (*bytes.Buffer, error) diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 6acd6a99..5507de6e 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -377,18 +377,18 @@ func (wd *wdaDriver) GetForegroundApp() (app AppInfo, err error) { return AppInfo{}, nil } -func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { +func (wd *wdaDriver) Tap(x, y int, options ...ActionOption) error { return wd.TapFloat(float64(x), float64(y), options...) } -func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { +func (wd *wdaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] data := map[string]interface{}{ "x": wd.toScale(x), "y": wd.toScale(y), } // new data options in post data for extra WDA configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/tap/0") return @@ -426,11 +426,11 @@ func (wd *wdaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err err return } -func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) error { +func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) error { return wd.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) { +func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] data := map[string]interface{}{ "fromX": wd.toScale(fromX), @@ -440,17 +440,17 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra WDA configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } -func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { +func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) error { return wd.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } -func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error { +func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { return wd.DragFloat(fromX, fromY, toX, toY, options...) } @@ -477,23 +477,23 @@ func (wd *wdaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe return } -func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) { +func (wd *wdaDriver) SendKeys(text string, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] data := map[string]interface{}{"value": strings.Split(text, "")} // new data options in post data for extra WDA configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/keys") return } -func (wd *wdaDriver) Input(text string, options ...DataOption) (err error) { +func (wd *wdaDriver) Input(text string, options ...ActionOption) (err error) { return wd.SendKeys(text, options...) } // PressBack simulates a short press on the BACK button. -func (wd *wdaDriver) PressBack(options ...DataOption) (err error) { +func (wd *wdaDriver) PressBack(options ...ActionOption) (err error) { windowSize, err := wd.WindowSize() if err != nil { return @@ -507,7 +507,7 @@ func (wd *wdaDriver) PressBack(options ...DataOption) (err error) { } // new data options in post data for extra WDA configurations - newData := NewData(data, options...) + newData := mergeDataWithOptions(data, options...) _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") return diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 939ffcbd..b88cb85f 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -48,10 +48,10 @@ func (t OCRTexts) texts() (texts []string) { return texts } -func (t OCRTexts) FindText(text string, options ...DataOption) ( +func (t OCRTexts) FindText(text string, options ...ActionOption) ( point PointF, err error) { - dataOptions := NewDataOptions(options...) + actionOptions := NewActionOptions(options...) var rects []image.Rectangle for _, ocrText := range t { @@ -63,8 +63,8 @@ func (t OCRTexts) FindText(text string, options ...DataOption) ( } // check if text in scope - if rect.Min.X < dataOptions.Scope[0] || rect.Max.X > dataOptions.Scope[2] || - rect.Min.Y < dataOptions.Scope[1] || rect.Max.Y > dataOptions.Scope[3] { + if rect.Min.X < actionOptions.AbsScope[0] || rect.Min.Y < actionOptions.AbsScope[1] || + rect.Max.X > actionOptions.AbsScope[2] || rect.Max.Y > actionOptions.AbsScope[3] { // not in scope continue } @@ -77,7 +77,7 @@ func (t OCRTexts) FindText(text string, options ...DataOption) ( } // match exactly, and not specify index, return the first one - if dataOptions.Index == 0 { + if actionOptions.Index == 0 { return getRectangleCenterPoint(rect), nil } } @@ -88,7 +88,7 @@ func (t OCRTexts) FindText(text string, options ...DataOption) ( } // get index - idx := dataOptions.Index + idx := actionOptions.Index if idx > 0 { // NOTICE: index start from 1 idx = idx - 1 @@ -105,7 +105,7 @@ func (t OCRTexts) FindText(text string, options ...DataOption) ( return getRectangleCenterPoint(rects[idx]), nil } -func (t OCRTexts) FindTexts(texts []string, options ...DataOption) (points []PointF, err error) { +func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (points []PointF, err error) { for _, text := range texts { point, err := t.FindText(text, options...) if err != nil { @@ -296,7 +296,7 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { return ocrTexts, nil } -func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) { ocrTexts, err := dExt.GetScreenTextsByOCR() if err != nil { return diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index d3888128..e72f0cde 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...ActionOption) (point PointF, err error) { var bufSource, bufSearch *bytes.Buffer if bufSearch, err = getBufFromDisk(imagePath); err != nil { return PointF{}, err diff --git a/hrp/pkg/uixt/opencv_off.go b/hrp/pkg/uixt/opencv_off.go index 394f9666..981d97ea 100644 --- a/hrp/pkg/uixt/opencv_off.go +++ b/hrp/pkg/uixt/opencv_off.go @@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...ActionOption) (point PointF, err error) { log.Fatal().Msg("opencv is not supported") return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 2636f86d..a654a6c8 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/code" ) @@ -16,7 +15,7 @@ func assertRelative(p float64) bool { } // SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY] -func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...DataOption) error { +func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...ActionOption) error { width := dExt.windowSize.Width height := dExt.windowSize.Height @@ -34,7 +33,7 @@ func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ... return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, options...) } -func (dExt *DriverExt) SwipeTo(direction string, options ...DataOption) (err error) { +func (dExt *DriverExt) SwipeTo(direction string, options ...ActionOption) (err error) { switch direction { case "up": return dExt.SwipeUp(options...) @@ -48,28 +47,28 @@ func (dExt *DriverExt) SwipeTo(direction string, options ...DataOption) (err err return fmt.Errorf("unexpected direction: %s", direction) } -func (dExt *DriverExt) SwipeUp(options ...DataOption) (err error) { +func (dExt *DriverExt) SwipeUp(options ...ActionOption) (err error) { return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, options...) } -func (dExt *DriverExt) SwipeDown(options ...DataOption) (err error) { +func (dExt *DriverExt) SwipeDown(options ...ActionOption) (err error) { return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, options...) } -func (dExt *DriverExt) SwipeLeft(options ...DataOption) (err error) { +func (dExt *DriverExt) SwipeLeft(options ...ActionOption) (err error) { return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, options...) } -func (dExt *DriverExt) SwipeRight(options ...DataOption) (err error) { +func (dExt *DriverExt) SwipeRight(options ...ActionOption) (err error) { return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, options...) } type Action func(driver *DriverExt) error -func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...DataOption) error { - dataOptions := NewDataOptions(options...) - maxRetryTimes := dataOptions.MaxRetryTimes - interval := dataOptions.Interval +func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...ActionOption) error { + actionOptions := NewActionOptions(options...) + maxRetryTimes := actionOptions.MaxRetryTimes + interval := actionOptions.Interval for i := 0; i < maxRetryTimes; i++ { if err := findCondition(dExt); err == nil { @@ -89,56 +88,38 @@ func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, fmt.Sprintf("loop %d times, match find condition failed", maxRetryTimes)) } -func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt) error { - identifierOption := WithDataIdentifier(action.Identifier) - durationOption := WithDataPressDuration(action.Duration) - - if action.Steps == 0 { - action.Steps = 10 - } - stepsOption := WithDataSteps(action.Steps) - - dataOptions := make([]DataOption, 3) - dataOptions = append(dataOptions, identifierOption, durationOption, stepsOption) - +func (dExt *DriverExt) prepareSwipeAction(options ...ActionOption) func(d *DriverExt) error { + actionOptions := NewActionOptions(options...) var swipeDirection interface{} - if action.Direction != nil { - swipeDirection = action.Direction + if actionOptions.Direction != nil { + swipeDirection = actionOptions.Direction } else { swipeDirection = "up" // default swipe up } + if actionOptions.Steps == 0 { + actionOptions.Steps = 10 + } + return func(d *DriverExt) error { defer func() { // wait for swipe action to completed and content to load completely - time.Sleep(time.Duration(1000*action.WaitTime) * time.Millisecond) + time.Sleep(time.Duration(1000*actionOptions.Interval) * time.Millisecond) }() if d, ok := swipeDirection.(string); ok { // enum direction: up, down, left, right - if err := dExt.SwipeTo(d, dataOptions...); err != nil { + if err := dExt.SwipeTo(d, options...); err != nil { log.Error().Err(err).Msgf("swipe %s failed", d) return err } } else if d, ok := swipeDirection.([]float64); ok { // custom direction: [fromX, fromY, toX, toY] - if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], dataOptions...); err != nil { + if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], options...); err != nil { log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", d[0], d[1], d[2], d[3]) return err } - } else if d, ok := swipeDirection.([]interface{}); ok { - // loaded from json case - // custom direction: [fromX, fromY, toX, toY] - sx, _ := builtin.Interface2Float64(d[0]) - sy, _ := builtin.Interface2Float64(d[1]) - ex, _ := builtin.Interface2Float64(d[2]) - ey, _ := builtin.Interface2Float64(d[3]) - if err := dExt.SwipeRelative(sx, sy, ex, ey, dataOptions...); err != nil { - log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed", - sx, sy, ex, ey) - return err - } } else { return fmt.Errorf("invalid swipe params %v", swipeDirection) } @@ -146,28 +127,13 @@ func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt } } -func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) error { +func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) error { if len(texts) == 0 { return errors.New("no text to tap") } - if len(action.Scope) != 4 { - action.Scope = []float64{0, 0, 1, 1} - } - if len(action.Offset) != 2 { - action.Offset = []int{0, 0} - } - - identifierOption := WithDataIdentifier(action.Identifier) - offsetOption := WithDataOffset(action.Offset[0], action.Offset[1]) - indexOption := WithDataIndex(action.Index) - scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes) - waitTimeOption := WithDataWaitTime(action.WaitTime) + options = append(options, WithMaxRetryTimes(10)) var point PointF findTexts := func(d *DriverExt) error { @@ -176,7 +142,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) erro if err != nil { return err } - points, err := ocrTexts.FindTexts(texts, indexOption, scopeOption) + points, err := ocrTexts.FindTexts(texts, options...) if err != nil { return err } @@ -185,14 +151,14 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) erro } foundTextAction := func(d *DriverExt) error { // tap text - return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption) + return d.TapAbsXY(point.X, point.Y, options...) } - findAction := dExt.prepareSwipeAction(action) - return dExt.LoopUntil(findAction, findTexts, foundTextAction, maxRetryOption, waitTimeOption) + findAction := dExt.prepareSwipeAction(options...) + return dExt.LoopUntil(findAction, findTexts, foundTextAction, options...) } -func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error { +func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) error { // go to home screen if err := dExt.Driver.Homescreen(); err != nil { return errors.Wrap(err, "go to home screen failed") @@ -203,8 +169,8 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error dExt.SwipeRight() } - action.Offset = []int{0, -25} // tap app icon above the text - action.Direction = "left" + options = append(options, WithOffset(0, -25)) // tap app icon above the text + options = append(options, WithDirection("left")) - return dExt.swipeToTapTexts([]string{appName}, action) + return dExt.swipeToTapTexts([]string{appName}, options...) } diff --git a/hrp/pkg/uixt/swipe_test.go b/hrp/pkg/uixt/swipe_test.go index 32cb3033..e40c41c5 100644 --- a/hrp/pkg/uixt/swipe_test.go +++ b/hrp/pkg/uixt/swipe_test.go @@ -9,21 +9,11 @@ import ( func TestAndroidSwipeAction(t *testing.T) { setupAndroid(t) - action := MobileAction{ - Method: ACTION_Swipe, - Params: "up", - } - swipeAction := driverExt.prepareSwipeAction(action) - + swipeAction := driverExt.prepareSwipeAction(WithDirection("up")) err := swipeAction(driverExt) checkErr(t, err) - action = MobileAction{ - Method: ACTION_Swipe, - Params: []float64{0.5, 0.5, 0.5, 0.9}, - } - swipeAction = driverExt.prepareSwipeAction(action) - + swipeAction = driverExt.prepareSwipeAction(WithCustomDirection(0.5, 0.5, 0.5, 0.9)) err = swipeAction(driverExt) checkErr(t, err) } @@ -31,7 +21,7 @@ func TestAndroidSwipeAction(t *testing.T) { func TestAndroidSwipeToTapApp(t *testing.T) { setupAndroid(t) - err := driverExt.swipeToTapApp("抖音", MobileAction{}) + err := driverExt.swipeToTapApp("抖音") checkErr(t, err) } @@ -41,9 +31,6 @@ func TestAndroidSwipeToTapTexts(t *testing.T) { err := driverExt.Driver.AppLaunch("com.ss.android.ugc.aweme") checkErr(t, err) - action := MobileAction{ - Params: "up", - } - err = driverExt.swipeToTapTexts([]string{"点击进入直播间", "直播中"}, action) + err = driverExt.swipeToTapTexts([]string{"点击进入直播间", "直播中"}, WithDirection("up")) checkErr(t, err) } diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index d5d80cbe..b4707839 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -4,12 +4,12 @@ import ( "fmt" ) -func (dExt *DriverExt) TapAbsXY(x, y float64, options ...DataOption) error { +func (dExt *DriverExt) TapAbsXY(x, y float64, options ...ActionOption) error { // tap on absolute coordinate [x, y] return dExt.Driver.TapFloat(x, y, options...) } -func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error { +func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error { // tap on [x, y] percent of window size if x > 1 || y > 1 { return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) @@ -21,12 +21,12 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error { return dExt.TapAbsXY(x, y, options...) } -func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { - dataOptions := NewDataOptions(options...) +func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error { + actionOptions := NewActionOptions(options...) point, err := dExt.FindScreenTextByOCR(ocrText, options...) if err != nil { - if dataOptions.IgnoreNotFoundError { + if actionOptions.IgnoreNotFoundError { return nil } return err @@ -35,12 +35,12 @@ func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { return dExt.TapAbsXY(point.X, point.Y, options...) } -func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { - dataOptions := NewDataOptions(options...) +func (dExt *DriverExt) TapByCV(imagePath string, options ...ActionOption) error { + actionOptions := NewActionOptions(options...) point, err := dExt.FindImageRectInUIKit(imagePath, options...) if err != nil { - if dataOptions.IgnoreNotFoundError { + if actionOptions.IgnoreNotFoundError { return nil } return err @@ -49,16 +49,16 @@ func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { return dExt.TapAbsXY(point.X, point.Y, options...) } -func (dExt *DriverExt) Tap(param string, options ...DataOption) error { +func (dExt *DriverExt) Tap(param string, options ...ActionOption) error { return dExt.TapOffset(param, 0.5, 0.5, options...) } -func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...DataOption) (err error) { - dataOptions := NewDataOptions(options...) +func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) point, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { - if dataOptions.IgnoreNotFoundError { + if actionOptions.IgnoreNotFoundError { return nil } return err diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 2a4f1f6a..a36afe09 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -69,12 +69,11 @@ func (s *StepMobile) Home() *StepMobile { // TapXY taps the point {X,Y}, X & Y is percentage of coordinates func (s *StepMobile) TapXY(x, y float64, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_TapXY, - Params: []float64{x, y}, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_TapXY, + Params: []float64{x, y}, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } @@ -82,12 +81,11 @@ func (s *StepMobile) TapXY(x, y float64, options ...uixt.ActionOption) *StepMobi // TapAbsXY taps the point {X,Y}, X & Y is absolute coordinates func (s *StepMobile) TapAbsXY(x, y float64, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_TapAbsXY, - Params: []float64{x, y}, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_TapAbsXY, + Params: []float64{x, y}, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } @@ -95,12 +93,11 @@ func (s *StepMobile) TapAbsXY(x, y float64, options ...uixt.ActionOption) *StepM // Tap taps on the target element func (s *StepMobile) Tap(params string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Tap, - Params: params, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Tap, + Params: params, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } @@ -108,12 +105,11 @@ func (s *StepMobile) Tap(params string, options ...uixt.ActionOption) *StepMobil // Tap taps on the target element by OCR recognition func (s *StepMobile) TapByOCR(ocrText string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_TapByOCR, - Params: ocrText, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_TapByOCR, + Params: ocrText, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } @@ -121,153 +117,142 @@ func (s *StepMobile) TapByOCR(ocrText string, options ...uixt.ActionOption) *Ste // Tap taps on the target element by CV recognition func (s *StepMobile) TapByCV(imagePath string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_TapByCV, - Params: imagePath, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_TapByCV, + Params: imagePath, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates -func (s *StepMobile) DoubleTapXY(x, y float64) *StepMobile { +func (s *StepMobile) DoubleTapXY(x, y float64, options ...uixt.ActionOption) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_DoubleTapXY, - Params: []float64{x, y}, + Method: uixt.ACTION_DoubleTapXY, + Params: []float64{x, y}, + Options: uixt.NewActionOptions(options...), }) return &StepMobile{step: s.step} } func (s *StepMobile) DoubleTap(params string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_DoubleTap, - Params: params, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_DoubleTap, + Params: params, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) Back(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Back, - Params: nil, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Back, + Params: nil, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) Swipe(sx, sy, ex, ey float64, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Swipe, - Params: []float64{sx, sy, ex, ey}, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Swipe, + Params: []float64{sx, sy, ex, ey}, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeUp(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Swipe, - Params: "up", - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Swipe, + Params: "up", + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeDown(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Swipe, - Params: "down", - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Swipe, + Params: "down", + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeLeft(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Swipe, - Params: "left", - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Swipe, + Params: "left", + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeRight(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Swipe, - Params: "right", - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Swipe, + Params: "right", + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeToTapApp(appName string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_SwipeToTapApp, - Params: appName, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_SwipeToTapApp, + Params: appName, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeToTapText(text string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_SwipeToTapText, - Params: text, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_SwipeToTapText, + Params: text, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) SwipeToTapTexts(texts interface{}, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_SwipeToTapTexts, - Params: texts, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_SwipeToTapTexts, + Params: texts, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } func (s *StepMobile) Input(text string, options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ - Method: uixt.ACTION_Input, - Params: text, - } - for _, option := range options { - option(&action) + Method: uixt.ACTION_Input, + Params: text, + Options: uixt.NewActionOptions(options...), } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) return &StepMobile{step: s.step} } From 0413ff62d16b14219eadc0904ed93ba09c4bb59b Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 2 May 2023 00:28:00 +0800 Subject: [PATCH 26/67] fix: convert relative scope to absolute scope --- .../uitest/demo_android_video_crawler.json | 34 ++++----- hrp/pkg/uixt/action.go | 71 +++++++++---------- hrp/pkg/uixt/ext.go | 21 ++++-- hrp/pkg/uixt/ocr_vedem.go | 20 +++--- hrp/pkg/uixt/swipe.go | 5 +- hrp/step_mobile_ui.go | 30 ++++---- 6 files changed, 94 insertions(+), 87 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 94f060a2..026ba5cf 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -20,23 +20,23 @@ "params": { "app_package_name": "com.ss.android.ugc.aweme", "feed": { - "sleep_random": [ - 0, - 5, - 0.7, - 5, - 10, - 0.3 -], - "target_count": 5 -}, + "sleep_random": [ + 0, + 5, + 0.7, + 5, + 10, + 0.3 + ], + "target_count": 5 + }, "live": { - "sleep_random": [ - 15, - 20 -], - "target_count": 3 -} + "sleep_random": [ + 15, + 20 + ], + "target_count": 3 + } } } ] @@ -62,4 +62,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 9778b623..dfe27ec5 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -3,7 +3,6 @@ package uixt import ( "encoding/json" "fmt" - "math" "math/rand" "time" @@ -82,10 +81,12 @@ type ActionOptions struct { Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty"` // scope related - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + // (x1, y1) is the top left corner, (x2, y2) is the bottom right corner + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // [x1, y1, x2, y2] in percentage of the screen + AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // [x1, y1, x2, y2] in absolute pixels + + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 // element related Text string `json:"text,omitempty" yaml:"text,omitempty"` @@ -146,11 +147,9 @@ func (o *ActionOptions) Options() []ActionOption { if o.Frequency != 0 { options = append(options, WithFrequency(o.Frequency)) } - if len(o.Scope) != 4 { - options = append(options, WithScope(0, 0, 1, 1)) - } - if len(o.AbsScope) != 4 { - o.AbsScope = []int{0, 0, math.MaxInt64, math.MaxInt64} + if len(o.AbsScope) == 4 { + options = append(options, WithAbsScope( + o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3])) } if len(o.Offset) == 2 { options = append(options, WithOffset(o.Offset[0], o.Offset[1])) @@ -167,9 +166,7 @@ func (o *ActionOptions) Options() []ActionOption { } func NewActionOptions(options ...ActionOption) *ActionOptions { - actionOptions := &ActionOptions{ - Custom: make(map[string]interface{}), - } + actionOptions := &ActionOptions{} for _, option := range options { option(actionOptions) } @@ -179,11 +176,6 @@ func NewActionOptions(options ...ActionOption) *ActionOptions { func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} { actionOptions := NewActionOptions(options...) - // custom options - for k, v := range actionOptions.Custom { - data[k] = v - } - if actionOptions.Identifier != "" { data["log"] = map[string]interface{}{ "enable": true, @@ -228,6 +220,13 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) data["isReplace"] = true // default true } + // custom options + if actionOptions.Custom != nil { + for k, v := range actionOptions.Custom { + data[k] = v + } + } + return data } @@ -235,6 +234,9 @@ type ActionOption func(o *ActionOptions) func WithCustomOption(key string, value interface{}) ActionOption { return func(o *ActionOptions) { + if o.Custom == nil { + o.Custom = make(map[string]interface{}) + } o.Custom[key] = value } } @@ -284,12 +286,21 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { } // WithScope inputs area of [(x1,y1), (x2,y2)] +// x1, y1, x2, y2 are all in [0, 1], which means the relative position of the screen func WithScope(x1, y1, x2, y2 float64) ActionOption { return func(o *ActionOptions) { o.Scope = []float64{x1, y1, x2, y2} } } +// WithAbsScope inputs area of [(x1,y1), (x2,y2)] +// x1, y1, x2, y2 are all absolute position of the screen +func WithAbsScope(x1, y1, x2, y2 int) ActionOption { + return func(o *ActionOptions) { + o.AbsScope = []int{x1, y1, x2, y2} + } +} + func WithOffset(offsetX, offsetY int) ActionOption { return func(o *ActionOptions) { o.Offset = []int{offsetX, offsetY} @@ -346,13 +357,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params, should be app text(string), got %v", ACTION_SwipeToTapText, action.Params) case ACTION_SwipeToTapTexts: - if texts, ok := action.Params.([]interface{}); ok { - var textList []string - for _, t := range texts { - textList = append(textList, t.(string)) - } - action.Params = textList - } if texts, ok := action.Params.([]string); ok { return dExt.swipeToTapTexts(texts, action.Options.Options()...) } @@ -391,9 +395,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - if len(action.Options.Offset) != 2 { - action.Options.Offset = []int{0, 0} - } return dExt.TapAbsXY(x, y, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) @@ -404,13 +405,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - if len(action.Options.Scope) != 4 { - action.Options.Scope = []float64{0, 0, 1, 1} - } - if len(action.Options.Offset) != 2 { - action.Options.Offset = []int{0, 0} - } - return dExt.TapByOCR(ocrText, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) @@ -460,11 +454,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params) case ACTION_SleepRandom: - params, ok := action.Params.([]interface{}) - if !ok { - return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) + if params, ok := action.Params.([]interface{}); ok { + return sleepRandom(params) } - return sleepRandom(params) + return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 40610319..fc82cd83 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -219,13 +219,20 @@ func (dExt *DriverExt) IsImageExist(text string) bool { return err == nil } -// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner -// the value of (x, y) is between 0 and 1, which means the percentage of the screen -func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { - return int(x1 * float64(dExt.windowSize.Width)), - int(y1 * float64(dExt.windowSize.Height)), - int(x2 * float64(dExt.windowSize.Width)), - int(y2 * float64(dExt.windowSize.Height)) +func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption { + actionOptions := NewActionOptions(options...) + + // convert relative scope to absolute scope + if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 { + scope := actionOptions.Scope + x1 := int(scope[0] * float64(dExt.windowSize.Width)) + y1 := int(scope[1] * float64(dExt.windowSize.Height)) + x2 := int(scope[2] * float64(dExt.windowSize.Width)) + y2 := int(scope[3] * float64(dExt.windowSize.Height)) + actionOptions.AbsScope = []int{x1, y1, x2, y2} + } + + return actionOptions.Options() } func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index b88cb85f..b2f0ccc0 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -57,15 +57,19 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( for _, ocrText := range t { rect := ocrText.Rect - // not contains text - if !strings.Contains(ocrText.Text, text) { - continue + // check if text in scope + if len(actionOptions.AbsScope) == 4 { + if rect.Min.X < actionOptions.AbsScope[0] || + rect.Min.Y < actionOptions.AbsScope[1] || + rect.Max.X > actionOptions.AbsScope[2] || + rect.Max.Y > actionOptions.AbsScope[3] { + // not in scope + continue + } } - // check if text in scope - if rect.Min.X < actionOptions.AbsScope[0] || rect.Min.Y < actionOptions.AbsScope[1] || - rect.Max.X > actionOptions.AbsScope[2] || rect.Max.Y > actionOptions.AbsScope[3] { - // not in scope + // not contains text + if !strings.Contains(ocrText.Text, text) { continue } @@ -301,7 +305,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) if err != nil { return } - point, err = ocrTexts.FindText(text, options...) + point, err = ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index a654a6c8..69d5cfe7 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -132,9 +132,6 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) return errors.New("no text to tap") } - // default to retry 10 times - options = append(options, WithMaxRetryTimes(10)) - var point PointF findTexts := func(d *DriverExt) error { var err error @@ -142,7 +139,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) if err != nil { return err } - points, err := ocrTexts.FindTexts(texts, options...) + points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...) if err != nil { return err } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index a36afe09..3e0944e2 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -260,8 +260,9 @@ func (s *StepMobile) Input(text string, options ...uixt.ActionOption) *StepMobil // Sleep specify sleep seconds after last action func (s *StepMobile) Sleep(n float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_Sleep, - Params: n, + Method: uixt.ACTION_Sleep, + Params: n, + Options: nil, }) return &StepMobile{step: s.step} } @@ -272,40 +273,45 @@ func (s *StepMobile) Sleep(n float64) *StepMobile { // 2. [min1, max1, weight1, min2, max2, weight2, ...] : weight is the probability of the time range func (s *StepMobile) SleepRandom(params ...float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_SleepRandom, - Params: params, + Method: uixt.ACTION_SleepRandom, + Params: params, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_VideoCrawler, - Params: params, + Method: uixt.ACTION_VideoCrawler, + Params: params, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) ScreenShot() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_ScreenShot, - Params: nil, + Method: uixt.ACTION_ScreenShot, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) StartCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_StartCamera, - Params: nil, + Method: uixt.ACTION_StartCamera, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) StopCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_StopCamera, - Params: nil, + Method: uixt.ACTION_StopCamera, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} } From 042e3ba8ba29107db103dfe84cdc17278899ecd5 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 2 May 2023 10:55:02 +0800 Subject: [PATCH 27/67] feat: find text with regex --- hrp/pkg/uixt/action.go | 17 +++++++++++------ hrp/pkg/uixt/ocr_vedem.go | 35 +++++++++++++++-------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index dfe27ec5..e17db95f 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -85,13 +85,9 @@ type ActionOptions struct { Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // [x1, y1, x2, y2] in percentage of the screen AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // [x1, y1, x2, y2] in absolute pixels + Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - - // element related - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element // set custiom options such as textview, id, description Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"` @@ -154,6 +150,9 @@ func (o *ActionOptions) Options() []ActionOption { if len(o.Offset) == 2 { options = append(options, WithOffset(o.Offset[0], o.Offset[1])) } + if o.Regex { + options = append(options, WithRegex(true)) + } // custom options if o.Custom != nil { @@ -307,6 +306,12 @@ func WithOffset(offsetX, offsetY int) ActionOption { } } +func WithRegex(regex bool) ActionOption { + return func(o *ActionOptions) { + o.Regex = regex + } +} + func WithFrequency(frequency int) ActionOption { return func(o *ActionOptions) { o.Frequency = frequency diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index b2f0ccc0..92888ef3 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" - "strings" + "regexp" "time" "github.com/pkg/errors" @@ -68,22 +68,19 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( } } - // not contains text - if !strings.Contains(ocrText.Text, text) { - continue + if actionOptions.Regex { + // regex on, check if match regex + if !regexp.MustCompile(text).MatchString(ocrText.Text) { + continue + } + } else { + // regex off, check if match exactly + if ocrText.Text != text { + continue + } } rects = append(rects, rect) - - // contains text while not match exactly - if ocrText.Text != text { - continue - } - - // match exactly, and not specify index, return the first one - if actionOptions.Index == 0 { - return getRectangleCenterPoint(rect), nil - } } if len(rects) == 0 { @@ -93,15 +90,12 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( // get index idx := actionOptions.Index - if idx > 0 { - // NOTICE: index start from 1 - idx = idx - 1 - } else if idx < 0 { + if idx < 0 { idx = len(rects) + idx } // index out of range - if idx >= len(rects) { + if idx >= len(rects) || idx < 0 { return PointF{}, errors.Wrap(code.OCRTextNotFoundError, fmt.Sprintf("text %s found %d, index %d out of range", text, len(rects), idx)) } @@ -109,7 +103,8 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( return getRectangleCenterPoint(rects[idx]), nil } -func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (points []PointF, err error) { +func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) ( + points []PointF, err error) { for _, text := range texts { point, err := t.FindText(text, options...) if err != nil { From b5b84aed0a470bd83185a454a647077d735ac284 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 2 May 2023 11:49:37 +0800 Subject: [PATCH 28/67] refactor: FindText(s) returns OCRText(s) --- hrp/pkg/uixt/demo/main_test.go | 3 ++- hrp/pkg/uixt/ocr_vedem.go | 37 +++++++++++++++++++--------------- hrp/pkg/uixt/swipe.go | 2 +- hrp/pkg/uixt/video_crawler.go | 5 +++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index b2610e0e..c2c8c3dd 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -47,7 +47,8 @@ func TestIOSDemo(t *testing.T) { continue } - err = driverExt.TapAbsXY(points[1].X, points[1].Y) + point := points[1].Center() + err = driverExt.TapAbsXY(point.X, point.Y) if err != nil { t.Fatal(err) } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 92888ef3..9b7dd24a 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -39,6 +39,10 @@ type OCRText struct { Rect image.Rectangle } +func (t OCRText) Center() PointF { + return getRectangleCenterPoint(t.Rect) +} + type OCRTexts []OCRText func (t OCRTexts) texts() (texts []string) { @@ -49,11 +53,11 @@ func (t OCRTexts) texts() (texts []string) { } func (t OCRTexts) FindText(text string, options ...ActionOption) ( - point PointF, err error) { + result OCRText, err error) { actionOptions := NewActionOptions(options...) - var rects []image.Rectangle + var results []OCRText for _, ocrText := range t { rect := ocrText.Rect @@ -80,44 +84,44 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( } } - rects = append(rects, rect) + results = append(results, ocrText) } - if len(rects) == 0 { - return PointF{}, errors.Wrap(code.OCRTextNotFoundError, + if len(results) == 0 { + return OCRText{}, errors.Wrap(code.OCRTextNotFoundError, fmt.Sprintf("text %s not found in %v", text, t.texts())) } // get index idx := actionOptions.Index if idx < 0 { - idx = len(rects) + idx + idx = len(results) + idx } // index out of range - if idx >= len(rects) || idx < 0 { - return PointF{}, errors.Wrap(code.OCRTextNotFoundError, - fmt.Sprintf("text %s found %d, index %d out of range", text, len(rects), idx)) + if idx >= len(results) || idx < 0 { + return OCRText{}, errors.Wrap(code.OCRTextNotFoundError, + fmt.Sprintf("text %s found %d, index %d out of range", text, len(results), idx)) } - return getRectangleCenterPoint(rects[idx]), nil + return results[idx], nil } func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) ( - points []PointF, err error) { + results OCRTexts, err error) { for _, text := range texts { - point, err := t.FindText(text, options...) + ocrText, err := t.FindText(text, options...) if err != nil { continue } - points = append(points, point) + results = append(results, ocrText) } - if len(points) != len(texts) { + if len(results) != len(texts) { return nil, errors.Wrap(code.OCRTextNotFoundError, fmt.Sprintf("texts %s not found in %v", texts, t.texts())) } - return points, nil + return results, nil } func newVEDEMOCRService() (*veDEMOCRService, error) { @@ -300,11 +304,12 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) if err != nil { return } - point, err = ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) + result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return } + point = result.Center() log.Info().Str("text", text). Interface("point", point).Msgf("FindScreenTextByOCR success") diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 69d5cfe7..7173caf6 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -143,7 +143,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) if err != nil { return err } - point = points[0] // FIXME + point = points[0].Center() // FIXME return nil } foundTextAction := func(d *DriverExt) error { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 02e8324e..809314d1 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -79,7 +79,7 @@ func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes boo // 预览流入口 points, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) if err == nil { - return points[0], true + return points[0].Center(), true } // TODO: 头像入口 @@ -271,7 +271,8 @@ func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { points, err := texts.FindTexts([]string{"确定", "取消"}) if err == nil { log.Warn().Msg("text popup found") - if err := dExt.TapAbsXY(points[1].X, points[1].Y); err != nil { + point := points[1].Center() + if err := dExt.TapAbsXY(point.X, point.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") return err } From 1c40f60f4b1ea5e838425d3c66c27e8f80055d3c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 2 May 2023 13:09:51 +0800 Subject: [PATCH 29/67] feat: add GetAbsScope --- hrp/pkg/uixt/action.go | 42 +++++++++++++++++++++++++++++++++++++----- hrp/pkg/uixt/ext.go | 16 ---------------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index e17db95f..5e762e11 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -66,6 +66,17 @@ type MobileAction struct { Options *ActionOptions `json:"options,omitempty" yaml:"options,omitempty"` } +// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner +// [x1, y1, x2, y2] in percentage of the screen +type Scope []float64 + +// [x1, y1, x2, y2] in absolute pixels +type AbsScope []int + +func (s AbsScope) Option() ActionOption { + return WithAbsScope(s[0], s[1], s[2], s[3]) +} + type ActionOptions struct { // log Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log @@ -81,9 +92,8 @@ type ActionOptions struct { Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty"` // scope related - // (x1, y1) is the top left corner, (x2, y2) is the bottom right corner - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // [x1, y1, x2, y2] in percentage of the screen - AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // [x1, y1, x2, y2] in absolute pixels + Scope Scope `json:"scope,omitempty" yaml:"scope,omitempty"` + AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point @@ -288,7 +298,7 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { // x1, y1, x2, y2 are all in [0, 1], which means the relative position of the screen func WithScope(x1, y1, x2, y2 float64) ActionOption { return func(o *ActionOptions) { - o.Scope = []float64{x1, y1, x2, y2} + o.Scope = Scope{x1, y1, x2, y2} } } @@ -296,7 +306,7 @@ func WithScope(x1, y1, x2, y2 float64) ActionOption { // x1, y1, x2, y2 are all absolute position of the screen func WithAbsScope(x1, y1, x2, y2 int) ActionOption { return func(o *ActionOptions) { - o.AbsScope = []int{x1, y1, x2, y2} + o.AbsScope = AbsScope{x1, y1, x2, y2} } } @@ -336,6 +346,28 @@ func WithIgnoreNotFoundError(ignoreError bool) ActionOption { } } +func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption { + actionOptions := NewActionOptions(options...) + + // convert relative scope to absolute scope + if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 { + scope := actionOptions.Scope + actionOptions.AbsScope = dExt.GenAbsScope( + scope[0], scope[1], scope[2], scope[3]) + } + + return actionOptions.Options() +} + +func (dExt *DriverExt) GenAbsScope(x1, y1, x2, y2 float64) AbsScope { + // convert relative scope to absolute scope + absX1 := int(x1 * float64(dExt.windowSize.Width)) + absY1 := int(y1 * float64(dExt.windowSize.Height)) + absX2 := int(x2 * float64(dExt.windowSize.Width)) + absY2 := int(y2 * float64(dExt.windowSize.Height)) + return AbsScope{absX1, absY1, absX2, absY2} +} + func (dExt *DriverExt) DoAction(action MobileAction) error { log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action") diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index fc82cd83..daf059c9 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -219,22 +219,6 @@ func (dExt *DriverExt) IsImageExist(text string) bool { return err == nil } -func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption { - actionOptions := NewActionOptions(options...) - - // convert relative scope to absolute scope - if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 { - scope := actionOptions.Scope - x1 := int(scope[0] * float64(dExt.windowSize.Width)) - y1 := int(scope[1] * float64(dExt.windowSize.Height)) - x2 := int(scope[2] * float64(dExt.windowSize.Width)) - y2 := int(scope[3] * float64(dExt.windowSize.Height)) - actionOptions.AbsScope = []int{x1, y1, x2, y2} - } - - return actionOptions.Options() -} - func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { var exp bool if assert == AssertionExists || assert == AssertionEqual { From 7b9e637d8798a6af149b5dbd27bc9a5b0f0cb8d4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 2 May 2023 23:29:06 +0800 Subject: [PATCH 30/67] feat: check feed type and incr feed count --- .../uitest/demo_android_video_crawler.json | 54 ++++++++++++++- .../uitest/demo_android_video_crawler_test.go | 7 ++ examples/worldcup/main.go | 3 +- hrp/pkg/uixt/video_crawler.go | 69 ++++++++++++++++--- hrp/pkg/uixt/video_crawler_test.go | 7 ++ 5 files changed, 130 insertions(+), 10 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 026ba5cf..697a75f6 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -28,7 +28,59 @@ 10, 0.3 ], - "target_count": 5 + "target_count": 5, + "target_labels": [ + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^广告$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^图文$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^特效\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^模板\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^购物\\|" + } + ] }, "live": { "sleep_random": [ diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index 9655808b..2a64ec94 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -23,6 +23,13 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { "app_package_name": "com.ss.android.ugc.aweme", "feed": map[string]interface{}{ "target_count": 5, + "target_labels": []map[string]interface{}{ + {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": "^图文$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": `^特效\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": `^模板\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": `^购物\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + }, "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, }, "live": map[string]interface{}{ diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index fc84d996..8670c115 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -214,7 +214,8 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { // 青少年弹窗处理 if ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { - _ = wc.driver.TapAbsXY(points[1].X, points[1].Y) + point := points[1].Center() + _ = wc.driver.TapAbsXY(point.X, point.Y) } } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 809314d1..b2d270e9 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -12,13 +12,16 @@ import ( type VideoStat struct { configs *VideoCrawlerConfigs - FeedCount int `json:"feed_count"` - LiveCount int `json:"live_count"` + FeedCount int `json:"feed_count"` + FeedStat map[string]int `json:"feed_stat"` // 分类统计 feed 数量：视频/图文/广告/特效/模板/购物 + LiveCount int `json:"live_count"` + LiveStat map[string]int `json:"live_stat"` // 分类统计 live 数量：秀场/游戏/电商/多人 } func (s *VideoStat) isFeedTargetAchieved() bool { log.Info(). Int("count", s.FeedCount). + Interface("stat", s.FeedStat). Int("target", s.configs.Feed.TargetCount). Msg("current feed count") @@ -28,6 +31,7 @@ func (s *VideoStat) isFeedTargetAchieved() bool { func (s *VideoStat) isLiveTargetAchieved() bool { log.Info(). Int("count", s.LiveCount). + Interface("stat", s.FeedStat). Int("target", s.configs.Live.TargetCount). Msg("current live count") @@ -38,14 +42,55 @@ func (s *VideoStat) isTargetAchieved() bool { return s.isFeedTargetAchieved() && s.isLiveTargetAchieved() } +// incrFeed increases feed count and feed stat +func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { + // feed author + actionOptions := []ActionOption{ + WithRegex(true), + driverExt.GenAbsScope(0, 0.5, 1, 1).Option(), + } + if ocrText, err := texts.FindText("^@", actionOptions...); err == nil { + log.Info().Str("author", ocrText.Text).Msg("found feed author") + } + + for _, targetLabel := range s.configs.Feed.TargetLabels { + scope := targetLabel.Scope + actionOptions := []ActionOption{ + WithRegex(targetLabel.Regex), + driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(), + } + if ocrText, err := texts.FindText(targetLabel.Text, actionOptions...); err == nil { + log.Info().Str("label", targetLabel.Text). + Str("text", ocrText.Text).Msg("found feed success") + + key := targetLabel.Text + if _, ok := s.FeedStat[key]; !ok { + s.FeedStat[key] = 0 + } + s.FeedStat[key]++ + } + } + + s.FeedCount++ + return nil +} + +type TargetLabel struct { + Text string `json:"text"` + Scope Scope `json:"scope"` + Regex bool `json:"regex"` +} + type FeedConfig struct { - TargetCount int `json:"target_count"` - SleepRandom []interface{} `json:"sleep_random"` + TargetCount int `json:"target_count"` + TargetLabels []TargetLabel `json:"target_labels"` + SleepRandom []interface{} `json:"sleep_random"` } type LiveConfig struct { - TargetCount int `json:"target_count"` - SleepRandom []interface{} `json:"sleep_random"` + TargetCount int `json:"target_count"` + TargetLabels []TargetLabel `json:"target_labels"` + SleepRandom []interface{} `json:"sleep_random"` } type VideoCrawlerConfigs struct { @@ -156,6 +201,11 @@ func (l *LiveCrawler) exitLiveRoom() error { func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { currVideoStat := &VideoStat{ configs: configs, + + FeedCount: 0, + FeedStat: make(map[string]int), + LiveCount: 0, + LiveStat: make(map[string]int), } defer func() { dExt.cacheStepData.VideoStat = currVideoStat @@ -205,9 +255,11 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } } - // TODO: check feed type + // check feed type and incr feed count + if err := currVideoStat.incrFeed(texts, dExt); err != nil { + log.Error().Err(err).Msg("incr feed failed") + } - currVideoStat.FeedCount++ // sleep custom random time if err := sleepRandom(configs.Feed.SleepRandom); err != nil { log.Error().Err(err).Msg("sleep random failed") @@ -225,6 +277,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Error().Err(err).Msg("swipe up failed") return err } + time.Sleep(1 * time.Second) } return nil diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index e6f78c18..2c8ca391 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -12,6 +12,13 @@ func TestVideoCrawler(t *testing.T) { Feed: FeedConfig{ TargetCount: 5, + TargetLabels: []TargetLabel{ + {Text: `^广告$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^图文$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^特效\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^模板\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^购物\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + }, SleepRandom: []interface{}{0, 5, 0.7, 5, 10, 0.3}, }, Live: LiveConfig{ From d9638852763b5c0db5b4ed76d89aa05d41d85bb7 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 00:08:56 +0800 Subject: [PATCH 31/67] feat: set default sleep random strategy if not set --- hrp/pkg/uixt/video_crawler.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index b2d270e9..efee17f0 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -199,6 +199,14 @@ func (l *LiveCrawler) exitLiveRoom() error { } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { + // set default sleep random strategy if not set + if configs.Feed.SleepRandom == nil { + configs.Feed.SleepRandom = []interface{}{1, 5} + } + if configs.Live.SleepRandom == nil { + configs.Live.SleepRandom = []interface{}{10, 15} + } + currVideoStat := &VideoStat{ configs: configs, From d27729edc6f31104ca1145aaba82777c3e975585 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 11:40:48 +0800 Subject: [PATCH 32/67] feat: add timeout for video crawler --- .../uitest/demo_android_video_crawler.json | 3 +- .../uitest/demo_android_video_crawler_test.go | 1 + hrp/internal/code/code.go | 2 + hrp/pkg/uixt/video_crawler.go | 164 ++++++++++-------- hrp/pkg/uixt/video_crawler_test.go | 1 + 5 files changed, 96 insertions(+), 75 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 697a75f6..41b00b01 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -88,7 +88,8 @@ 20 ], "target_count": 3 - } + }, + "timeout": 600 } } ] diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index 2a64ec94..897f901e 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -21,6 +21,7 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { Android(). VideoCrawler(map[string]interface{}{ "app_package_name": "com.ss.android.ugc.aweme", + "timeout": 600, "feed": map[string]interface{}{ "target_count": 5, "target_labels": []map[string]interface{}{ diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 23d79bea..301850b7 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -44,6 +44,7 @@ var ( InitPluginFailed = errors.New("init plugin failed") // 31 BuildGoPluginFailed = errors.New("build go plugin failed") // 32 BuildPyPluginFailed = errors.New("build py plugin failed") // 33 + TimeoutError = errors.New("timeout error") // 39 ) // summary: [40, 50) @@ -111,6 +112,7 @@ var errorsMap = map[error]int{ InitPluginFailed: 31, BuildGoPluginFailed: 32, BuildPyPluginFailed: 33, + TimeoutError: 39, // ios related IOSDeviceConnectionError: 50, diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index efee17f0..4608d9f1 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -7,10 +7,13 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/code" ) type VideoStat struct { configs *VideoCrawlerConfigs + timer *time.Timer FeedCount int `json:"feed_count"` FeedStat map[string]int `json:"feed_stat"` // 分类统计 feed 数量：视频/图文/广告/特效/模板/购物 @@ -95,6 +98,7 @@ type LiveConfig struct { type VideoCrawlerConfigs struct { AppPackageName string `json:"app_package_name"` + Timeout int `json:"timeout"` // seconds Feed FeedConfig `json:"feed"` Live LiveConfig `json:"live"` @@ -142,36 +146,41 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { time.Sleep(5 * time.Second) for !l.currentStat.isLiveTargetAchieved() { - // check if live room - if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { - return err + select { + case <-l.currentStat.timer.C: + return errors.Wrap(code.TimeoutError, "timeout in live crawler") + default: + // check if live room + if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { + return err + } + + // take screenshot and get screen texts by OCR + _, err := l.driver.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + continue + } + + // TODO: check live type + + // swipe to next live video + err = l.driver.SwipeUp() + if err != nil { + log.Error().Err(err).Msg("swipe up failed") + // TODO: retry maximum 3 times + continue + } + + // sleep custom random time + if err := sleepRandom(l.configs.Live.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } + + // TODO: check live type + + l.currentStat.LiveCount++ } - - // take screenshot and get screen texts by OCR - _, err := l.driver.GetScreenTextsByOCR() - if err != nil { - log.Error().Err(err).Msg("OCR GetTexts failed") - continue - } - - // TODO: check live type - - // swipe to next live video - err = l.driver.SwipeUp() - if err != nil { - log.Error().Err(err).Msg("swipe up failed") - // TODO: retry maximum 3 times - continue - } - - // sleep custom random time - if err := sleepRandom(l.configs.Live.SleepRandom); err != nil { - log.Error().Err(err).Msg("sleep random failed") - } - - // TODO: check live type - - l.currentStat.LiveCount++ } log.Info().Msg("live count achieved, exit live room") @@ -231,64 +240,71 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { currentStat: currVideoStat, } - // loop until target count achieved + // loop until target count achieved or timeout // the main loop is feed crawler + currVideoStat.timer = time.NewTimer(time.Duration(configs.Timeout) * time.Second) for { - // check if feed page - if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { - return err - } + select { + case <-currVideoStat.timer.C: + return errors.Wrap(code.TimeoutError, "timeout in feed crawler") + default: + // check if feed page + if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { + return err + } - // take screenshot and get screen texts by OCR - texts, err := dExt.GetScreenTextsByOCR() - if err != nil { - log.Error().Err(err).Msg("OCR GetTexts failed") - continue - } + // take screenshot and get screen texts by OCR + texts, err := dExt.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + continue + } - // automatic handling of pop-up windows - if err := dExt.autoPopupHandler(texts); err != nil { - log.Error().Err(err).Msg("auto handle popup failed") - return err - } + // automatic handling of pop-up windows + if err := dExt.autoPopupHandler(texts); err != nil { + log.Error().Err(err).Msg("auto handle popup failed") + return err + } - // check if live video && run live crawler - if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { - log.Info().Msg("live video found") - if !liveCrawler.currentStat.isLiveTargetAchieved() { - if err := liveCrawler.Run(dExt, enterPoint); err != nil { - log.Error().Err(err).Msg("run live crawler failed, continue") - continue + // check if live video && run live crawler + if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { + log.Info().Msg("live video found") + if !liveCrawler.currentStat.isLiveTargetAchieved() { + if err := liveCrawler.Run(dExt, enterPoint); err != nil { + if errors.Is(err, code.TimeoutError) { + return err + } + log.Error().Err(err).Msg("run live crawler failed, continue") + continue + } } } - } - // check feed type and incr feed count - if err := currVideoStat.incrFeed(texts, dExt); err != nil { - log.Error().Err(err).Msg("incr feed failed") - } + // check feed type and incr feed count + if err := currVideoStat.incrFeed(texts, dExt); err != nil { + log.Error().Err(err).Msg("incr feed failed") + } - // sleep custom random time - if err := sleepRandom(configs.Feed.SleepRandom); err != nil { - log.Error().Err(err).Msg("sleep random failed") - } + // sleep custom random time + if err := sleepRandom(configs.Feed.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } - // check if target count achieved - if currVideoStat.isTargetAchieved() { - log.Info().Msg("target count achieved, exit crawler") - break - } + // check if target count achieved + if currVideoStat.isTargetAchieved() { + log.Info().Msg("target count achieved, exit crawler") + return nil + } - // swipe to next feed video - log.Info().Msg("swipe to next feed video") - if err = dExt.SwipeUp(); err != nil { - log.Error().Err(err).Msg("swipe up failed") - return err + // swipe to next feed video + log.Info().Msg("swipe to next feed video") + if err = dExt.SwipeUp(); err != nil { + log.Error().Err(err).Msg("swipe up failed") + return err + } + time.Sleep(1 * time.Second) } - time.Sleep(1 * time.Second) } - - return nil } func (dExt *DriverExt) assertActivity(packageName, activityType string) error { diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index 2c8ca391..8ecd7e88 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -9,6 +9,7 @@ func TestVideoCrawler(t *testing.T) { configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", + Timeout: 600, Feed: FeedConfig{ TargetCount: 5, From 4ee3b43d1f9f439ca036b544a532b42563ebd09c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 12:48:20 +0800 Subject: [PATCH 33/67] feat: set testcase and request timeout in seconds --- hrp/cmd/run.go | 5 +- hrp/config.go | 16 ++-- hrp/runner.go | 158 ++++++++++++++++++++++----------------- hrp/step_request_test.go | 6 +- 4 files changed, 106 insertions(+), 79 deletions(-) diff --git a/hrp/cmd/run.go b/hrp/cmd/run.go index 5a7b4f8e..f46f4efc 100644 --- a/hrp/cmd/run.go +++ b/hrp/cmd/run.go @@ -37,6 +37,7 @@ var ( proxyUrl string saveTests bool genHTMLReport bool + caseTimeout float32 ) func init() { @@ -48,12 +49,14 @@ func init() { runCmd.Flags().StringVarP(&proxyUrl, "proxy-url", "p", "", "set proxy url") runCmd.Flags().BoolVarP(&saveTests, "save-tests", "s", false, "save tests summary") runCmd.Flags().BoolVarP(&genHTMLReport, "gen-html-report", "g", false, "generate html report") + runCmd.Flags().Float32Var(&caseTimeout, "case-timeout", 3600, "set testcase timeout (seconds)") } func makeHRPRunner() *hrp.HRPRunner { runner := hrp.NewRunner(nil). SetFailfast(!continueOnFailure). - SetSaveTests(saveTests) + SetSaveTests(saveTests). + SetCaseTimeout(caseTimeout) if genHTMLReport { runner.GenHTMLReport() } diff --git a/hrp/config.go b/hrp/config.go index 9c03278f..6857aee1 100644 --- a/hrp/config.go +++ b/hrp/config.go @@ -2,7 +2,6 @@ package hrp import ( "reflect" - "time" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" @@ -32,7 +31,8 @@ type TConfig struct { WebSocketSetting *WebSocketConfig `json:"websocket,omitempty" yaml:"websocket,omitempty"` IOS []*uixt.IOSDevice `json:"ios,omitempty" yaml:"ios,omitempty"` Android []*uixt.AndroidDevice `json:"android,omitempty" yaml:"android,omitempty"` - Timeout float64 `json:"timeout,omitempty" yaml:"timeout,omitempty"` // global timeout in seconds + RequestTimeout float32 `json:"request_timeout,omitempty" yaml:"request_timeout,omitempty"` // request timeout in seconds + CaseTimeout float32 `json:"case_timeout,omitempty" yaml:"case_timeout,omitempty"` // testcase timeout in seconds Export []string `json:"export,omitempty" yaml:"export,omitempty"` Weight int `json:"weight,omitempty" yaml:"weight,omitempty"` Path string `json:"path,omitempty" yaml:"path,omitempty"` // testcase file path @@ -75,9 +75,15 @@ func (c *TConfig) SetThinkTime(strategy thinkTimeStrategy, cfg interface{}, limi return c } -// SetTimeout sets testcase timeout in seconds. -func (c *TConfig) SetTimeout(timeout time.Duration) *TConfig { - c.Timeout = timeout.Seconds() +// SetRequestTimeout sets request timeout in seconds. +func (c *TConfig) SetRequestTimeout(seconds float32) *TConfig { + c.RequestTimeout = seconds + return c +} + +// SetCaseTimeout sets testcase timeout in seconds. +func (c *TConfig) SetCaseTimeout(seconds float32) *TConfig { + c.CaseTimeout = seconds return c } diff --git a/hrp/runner.go b/hrp/runner.go index 2c5e3c06..0ac6fe16 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -21,6 +21,7 @@ import ( "golang.org/x/net/http2" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" "github.com/httprunner/httprunner/v4/hrp/internal/version" "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" @@ -60,22 +61,24 @@ func NewRunner(t *testing.T) *HRPRunner { wsDialer: &websocket.Dialer{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, + caseTimeoutTimer: time.NewTimer(time.Hour * 2), // default case timeout to 2 hour } } type HRPRunner struct { - t *testing.T - failfast bool - httpStatOn bool - requestsLogOn bool - pluginLogOn bool - venv string - saveTests bool - genHTMLReport bool - httpClient *http.Client - http2Client *http.Client - wsDialer *websocket.Dialer - uiClients map[string]*uixt.DriverExt // UI automation clients for iOS and Android, key is udid/serial + t *testing.T + failfast bool + httpStatOn bool + requestsLogOn bool + pluginLogOn bool + venv string + saveTests bool + genHTMLReport bool + httpClient *http.Client + http2Client *http.Client + wsDialer *websocket.Dialer + uiClients map[string]*uixt.DriverExt // UI automation clients for iOS and Android, key is udid/serial + caseTimeoutTimer *time.Timer // case timeout timer } // SetClientTransport configures transport of http client for high concurrency load testing @@ -152,10 +155,17 @@ func (r *HRPRunner) SetProxyUrl(proxyUrl string) *HRPRunner { return r } -// SetTimeout configures global timeout in seconds. -func (r *HRPRunner) SetTimeout(timeout time.Duration) *HRPRunner { - log.Info().Float64("timeout(seconds)", timeout.Seconds()).Msg("[init] SetTimeout") - r.httpClient.Timeout = timeout +// SetRequestTimeout configures global request timeout in seconds. +func (r *HRPRunner) SetRequestTimeout(seconds float32) *HRPRunner { + log.Info().Float32("timeout_seconds", seconds).Msg("[init] SetRequestTimeout") + r.httpClient.Timeout = time.Duration(seconds*1000) * time.Millisecond + return r +} + +// SetCaseTimeout configures global testcase timeout in seconds. +func (r *HRPRunner) SetCaseTimeout(seconds float32) *HRPRunner { + log.Info().Float32("timeout_seconds", seconds).Msg("[init] SetCaseTimeout") + r.caseTimeoutTimer = time.NewTimer(time.Duration(seconds*1000) * time.Millisecond) return r } @@ -291,10 +301,13 @@ func (r *HRPRunner) NewCaseRunner(testcase *TestCase) (*CaseRunner, error) { return nil, errors.Wrap(err, "parse testcase config failed") } + // set request timeout in seconds + if testcase.Config.RequestTimeout != 0 { + r.SetRequestTimeout(testcase.Config.RequestTimeout) + } // set testcase timeout in seconds - if testcase.Config.Timeout != 0 { - timeout := time.Duration(testcase.Config.Timeout*1000) * time.Millisecond - r.SetTimeout(timeout) + if testcase.Config.CaseTimeout != 0 { + r.SetCaseTimeout(testcase.Config.CaseTimeout) } // load plugin info to testcase config @@ -505,66 +518,71 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) error { // run step in sequential order for _, step := range r.caseRunner.testCase.TestSteps { - // TODO: parse step struct - // parse step name - parsedName, err := r.caseRunner.parser.ParseString(step.Name(), r.sessionVariables) - if err != nil { - parsedName = step.Name() - } - stepName := convertString(parsedName) - log.Info().Str("step", stepName). - Str("type", string(step.Type())).Msg("run step start") + select { + case <-r.caseRunner.hrpRunner.caseTimeoutTimer.C: + return errors.Wrap(code.TimeoutError, "session runner timeout") + default: + // TODO: parse step struct + // parse step name + parsedName, err := r.caseRunner.parser.ParseString(step.Name(), r.sessionVariables) + if err != nil { + parsedName = step.Name() + } + stepName := convertString(parsedName) + log.Info().Str("step", stepName). + Str("type", string(step.Type())).Msg("run step start") - // run times of step - loopTimes := step.Struct().Loops - if loopTimes < 0 { - log.Warn().Int("loops", loopTimes).Msg("loop times should be positive, set to 1") - loopTimes = 1 - } else if loopTimes == 0 { - loopTimes = 1 - } else if loopTimes > 1 { - log.Info().Int("loops", loopTimes).Msg("run step with specified loop times") - } - - // run step with specified loop times - var stepResult *StepResult - for i := 1; i <= loopTimes; i++ { - var loopIndex string - if loopTimes > 1 { - log.Info().Int("index", i).Msg("start running step in loop") - loopIndex = fmt.Sprintf("_loop_%d", i) + // run times of step + loopTimes := step.Struct().Loops + if loopTimes < 0 { + log.Warn().Int("loops", loopTimes).Msg("loop times should be positive, set to 1") + loopTimes = 1 + } else if loopTimes == 0 { + loopTimes = 1 + } else if loopTimes > 1 { + log.Info().Int("loops", loopTimes).Msg("run step with specified loop times") } - // run step - stepResult, err = step.Run(r) - stepResult.Name = stepName + loopIndex + // run step with specified loop times + var stepResult *StepResult + for i := 1; i <= loopTimes; i++ { + var loopIndex string + if loopTimes > 1 { + log.Info().Int("index", i).Msg("start running step in loop") + loopIndex = fmt.Sprintf("_loop_%d", i) + } - r.updateSummary(stepResult) - } + // run step + stepResult, err = step.Run(r) + stepResult.Name = stepName + loopIndex - // update extracted variables - for k, v := range stepResult.ExportVars { - r.sessionVariables[k] = v - } + r.updateSummary(stepResult) + } - if err == nil { - log.Info().Str("step", stepResult.Name). + // update extracted variables + for k, v := range stepResult.ExportVars { + r.sessionVariables[k] = v + } + + if err == nil { + log.Info().Str("step", stepResult.Name). + Str("type", string(stepResult.StepType)). + Bool("success", true). + Interface("exportVars", stepResult.ExportVars). + Msg("run step end") + continue + } + + // failed + log.Error().Err(err).Str("step", stepResult.Name). Str("type", string(stepResult.StepType)). - Bool("success", true). - Interface("exportVars", stepResult.ExportVars). + Bool("success", false). Msg("run step end") - continue - } - // failed - log.Error().Err(err).Str("step", stepResult.Name). - Str("type", string(stepResult.StepType)). - Bool("success", false). - Msg("run step end") - - // check if failfast - if r.caseRunner.hrpRunner.failfast { - return errors.Wrap(err, "abort running due to failfast setting") + // check if failfast + if r.caseRunner.hrpRunner.failfast { + return errors.Wrap(err, "abort running due to failfast setting") + } } } diff --git a/hrp/step_request_test.go b/hrp/step_request_test.go index 7172bf66..7b476ba9 100644 --- a/hrp/step_request_test.go +++ b/hrp/step_request_test.go @@ -164,7 +164,7 @@ func TestRunCaseWithTimeout(t *testing.T) { // global timeout testcase1 := &TestCase{ Config: NewConfig("TestCase1"). - SetTimeout(10 * time.Second). // set global timeout to 10s + SetRequestTimeout(10). // set global timeout to 10s SetBaseURL("https://httpbin.org"), TestSteps: []IStep{ NewStep("step1"). @@ -180,7 +180,7 @@ func TestRunCaseWithTimeout(t *testing.T) { testcase2 := &TestCase{ Config: NewConfig("TestCase2"). - SetTimeout(10 * time.Second). // set global timeout to 10s + SetRequestTimeout(10). // set global timeout to 10s SetBaseURL("https://httpbin.org"), TestSteps: []IStep{ NewStep("step1"). @@ -197,7 +197,7 @@ func TestRunCaseWithTimeout(t *testing.T) { // step timeout testcase3 := &TestCase{ Config: NewConfig("TestCase3"). - SetTimeout(10 * time.Second). + SetRequestTimeout(10). SetBaseURL("https://httpbin.org"), TestSteps: []IStep{ NewStep("step2"). From 1bb2079c872f117c942f2db0da42ad0771abb251 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 23:00:17 +0800 Subject: [PATCH 34/67] feat: catch interrupt signal --- examples/uitest/demo_android_feed_swipe.json | 15 ++++++---- hrp/internal/code/code.go | 2 ++ hrp/pkg/uixt/ext.go | 5 ++++ hrp/pkg/uixt/video_crawler.go | 14 ++++++++-- hrp/runner.go | 16 +++++++++++ hrp/step_mobile_ui.go | 29 +++++++++++++------- 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/examples/uitest/demo_android_feed_swipe.json b/examples/uitest/demo_android_feed_swipe.json index d74f45ae..c50ba576 100644 --- a/examples/uitest/demo_android_feed_swipe.json +++ b/examples/uitest/demo_android_feed_swipe.json @@ -45,7 +45,9 @@ { "method": "tap_ocr", "params": "我知道了", - "ignore_NotFoundError": true + "options": { + "ignore_NotFoundError": true + } } ] } @@ -56,7 +58,8 @@ "actions": [ { "method": "swipe", - "params": "up" + "params": "up", + "options": {} }, { "method": "sleep_random", @@ -75,7 +78,8 @@ "actions": [ { "method": "swipe", - "params": "up" + "params": "up", + "options": {} }, { "method": "sleep_random", @@ -94,7 +98,8 @@ "actions": [ { "method": "swipe", - "params": "up" + "params": "up", + "options": {} }, { "method": "sleep_random", @@ -131,4 +136,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 301850b7..87c830ea 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -44,6 +44,7 @@ var ( InitPluginFailed = errors.New("init plugin failed") // 31 BuildGoPluginFailed = errors.New("build go plugin failed") // 32 BuildPyPluginFailed = errors.New("build py plugin failed") // 33 + InterruptError = errors.New("interrupt error") // 38 TimeoutError = errors.New("timeout error") // 39 ) @@ -112,6 +113,7 @@ var errorsMap = map[error]int{ InitPluginFailed: 31, BuildGoPluginFailed: 32, BuildPyPluginFailed: 33, + InterruptError: 38, TimeoutError: 39, // ios related diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index daf059c9..cb25faa0 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -12,8 +12,10 @@ import ( "mime/multipart" "net/http" "os" + "os/signal" "path/filepath" "strings" + "syscall" "time" "github.com/pkg/errors" @@ -62,6 +64,7 @@ type DriverExt struct { frame *bytes.Buffer doneMjpegStream chan bool OCRService IOCRService // used to get texts from image + interruptSignal chan os.Signal // cache step data cacheStepData cacheStepData @@ -75,7 +78,9 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) ScreenShots: make([]string, 0), OcrResults: make(map[string]string), }, + interruptSignal: make(chan os.Signal, 1), } + signal.Notify(dExt.interruptSignal, syscall.SIGTERM, syscall.SIGINT) dExt.doneMjpegStream = make(chan bool, 1) // get device window size diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 4608d9f1..169f208a 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -148,7 +148,11 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { for !l.currentStat.isLiveTargetAchieved() { select { case <-l.currentStat.timer.C: - return errors.Wrap(code.TimeoutError, "timeout in live crawler") + log.Warn().Msg("timeout in live crawler") + return errors.Wrap(code.TimeoutError, "live crawler timeout") + case <-l.driver.interruptSignal: + log.Warn().Msg("interrupted in live crawler") + return errors.Wrap(code.InterruptError, "live crawler interrupted") default: // check if live room if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { @@ -246,7 +250,11 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { for { select { case <-currVideoStat.timer.C: - return errors.Wrap(code.TimeoutError, "timeout in feed crawler") + log.Warn().Msg("timeout in feed crawler") + return errors.Wrap(code.TimeoutError, "feed crawler timeout") + case <-dExt.interruptSignal: + log.Warn().Msg("interrupted in feed crawler") + return errors.Wrap(code.InterruptError, "feed crawler interrupted") default: // check if feed page if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { @@ -271,7 +279,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Info().Msg("live video found") if !liveCrawler.currentStat.isLiveTargetAchieved() { if err := liveCrawler.Run(dExt, enterPoint); err != nil { - if errors.Is(err, code.TimeoutError) { + if errors.Is(err, code.TimeoutError) || errors.Is(err, code.InterruptError) { return err } log.Error().Err(err).Msg("run live crawler failed, continue") diff --git a/hrp/runner.go b/hrp/runner.go index 0ac6fe16..818d695c 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -8,8 +8,11 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "os" + "os/signal" "path/filepath" "strings" + "syscall" "testing" "time" @@ -39,6 +42,8 @@ func NewRunner(t *testing.T) *HRPRunner { t = &testing.T{} } jar, _ := cookiejar.New(nil) + interruptSignal := make(chan os.Signal, 1) + signal.Notify(interruptSignal, syscall.SIGTERM, syscall.SIGINT) return &HRPRunner{ t: t, failfast: true, // default to failfast @@ -62,6 +67,7 @@ func NewRunner(t *testing.T) *HRPRunner { TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, caseTimeoutTimer: time.NewTimer(time.Hour * 2), // default case timeout to 2 hour + interruptSignal: interruptSignal, } } @@ -79,6 +85,7 @@ type HRPRunner struct { wsDialer *websocket.Dialer uiClients map[string]*uixt.DriverExt // UI automation clients for iOS and Android, key is udid/serial caseTimeoutTimer *time.Timer // case timeout timer + interruptSignal chan os.Signal // interrupt signal channel } // SetClientTransport configures transport of http client for high concurrency load testing @@ -520,7 +527,11 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) error { for _, step := range r.caseRunner.testCase.TestSteps { select { case <-r.caseRunner.hrpRunner.caseTimeoutTimer.C: + log.Warn().Msg("timeout in session runner") return errors.Wrap(code.TimeoutError, "session runner timeout") + case <-r.caseRunner.hrpRunner.interruptSignal: + log.Warn().Msg("interrupted in session runner") + return errors.Wrap(code.InterruptError, "session runner interrupted") default: // TODO: parse step struct // parse step name @@ -579,6 +590,11 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) error { Bool("success", false). Msg("run step end") + // interrupted or timeout, abort running + if errors.Is(err, code.InterruptError) || errors.Is(err, code.TimeoutError) { + return err + } + // check if failfast if r.caseRunner.hrpRunner.failfast { return errors.Wrap(err, "abort running due to failfast setting") diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 3e0944e2..850330a0 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -629,18 +629,27 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // run actions for _, action := range actions { - if action.Params, err = s.caseRunner.parser.Parse(action.Params, stepVariables); err != nil { - if !code.IsErrorPredefined(err) { - err = errors.Wrap(code.ParseError, - fmt.Sprintf("parse action params failed: %v", err)) + select { + case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C: + log.Warn().Msg("timeout in mobile UI runner") + return stepResult, errors.Wrap(code.TimeoutError, "mobile UI runner timeout") + case <-s.caseRunner.hrpRunner.interruptSignal: + log.Warn().Msg("interrupted in mobile UI runner") + return stepResult, errors.Wrap(code.InterruptError, "mobile UI runner interrupted") + default: + if action.Params, err = s.caseRunner.parser.Parse(action.Params, stepVariables); err != nil { + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.ParseError, + fmt.Sprintf("parse action params failed: %v", err)) + } + return stepResult, err } - return stepResult, err - } - if err := uiDriver.DoAction(action); err != nil { - if !code.IsErrorPredefined(err) { - err = errors.Wrap(code.MobileUIDriverError, err.Error()) + if err := uiDriver.DoAction(action); err != nil { + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.MobileUIDriverError, err.Error()) + } + return stepResult, err } - return stepResult, err } } From 94462c48bb763387ee58eee5ce2aaf616fda05ef Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 23:00:43 +0800 Subject: [PATCH 35/67] bump version v4.3.4-beta-202305032303 --- docs/CHANGELOG.md | 14 ++++++++++++++ hrp/internal/version/VERSION | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8f18b940..93b39efd 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,19 @@ # Release History +## v4.3.4 (2023-05-03) + +**go version** + +- feat: add video crawler for feed and live +- feat: cache screenshot ocr texts +- feat: set testcase and request timeout in seconds +- feat: catch interrupt signal +- feat: add new exit code MobileUILaunchAppError/InterruptError/TimeoutError +- feat: find text with regex +- refactor: simplify OCR APIs +- refactor: FindText(s) returns OCRText(s) +- refactor: merge ActionOption with DataOption + ## v4.3.3 (2023-04-19) **go version** diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index c294f2b5..8ef9558c 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.3 \ No newline at end of file +v4.3.4-beta-202305032303 \ No newline at end of file From 87cba441649ab2f354e1ab189fcddddb528cea8c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 3 May 2023 23:51:03 +0800 Subject: [PATCH 36/67] feat: check target achieving for each feed type --- .../uitest/demo_android_video_crawler.json | 1 + .../uitest/demo_android_video_crawler_test.go | 2 +- hrp/pkg/uixt/video_crawler.go | 67 ++++++++++++++----- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 41b00b01..356d62e6 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -38,6 +38,7 @@ 1, 1 ], + "target": 2, "text": "^广告$" }, { diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index 897f901e..4cda04b3 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -25,7 +25,7 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { "feed": map[string]interface{}{ "target_count": 5, "target_labels": []map[string]interface{}{ - {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true, "target": 2}, {"text": "^图文$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, {"text": `^特效\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, {"text": `^模板\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 169f208a..11237ad6 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -22,23 +22,59 @@ type VideoStat struct { } func (s *VideoStat) isFeedTargetAchieved() bool { - log.Info(). - Int("count", s.FeedCount). - Interface("stat", s.FeedStat). - Int("target", s.configs.Feed.TargetCount). - Msg("current feed count") + targetStat := make(map[string]int) + for _, targetLabel := range s.configs.Feed.TargetLabels { + targetStat[targetLabel.Text] = targetLabel.Target + } - return s.FeedCount >= s.configs.Feed.TargetCount + log.Info(). + Int("current_total", s.FeedCount). + Interface("current_stat", s.FeedStat). + Int("target_total", s.configs.Feed.TargetCount). + Interface("target_stat", targetStat). + Msg("display feed crawler progress") + + // check total feed count + if s.FeedCount < s.configs.Feed.TargetCount { + return false + } + + // check each feed type's count + for _, targetLabel := range s.configs.Feed.TargetLabels { + if s.FeedStat[targetLabel.Text] < targetLabel.Target { + return false + } + } + + return true } func (s *VideoStat) isLiveTargetAchieved() bool { - log.Info(). - Int("count", s.LiveCount). - Interface("stat", s.FeedStat). - Int("target", s.configs.Live.TargetCount). - Msg("current live count") + targetStat := make(map[string]int) + for _, targetLabel := range s.configs.Live.TargetLabels { + targetStat[targetLabel.Text] = targetLabel.Target + } - return s.LiveCount >= s.configs.Live.TargetCount + log.Info(). + Int("current_total", s.LiveCount). + Interface("current_stat", s.LiveStat). + Int("target_total", s.configs.Live.TargetCount). + Interface("target_stat", targetStat). + Msg("display live crawler progress") + + // check total live count + if s.LiveCount < s.configs.Live.TargetCount { + return false + } + + // check each live type's count + for _, targetLabel := range s.configs.Live.TargetLabels { + if s.LiveStat[targetLabel.Text] < targetLabel.Target { + return false + } + } + + return true } func (s *VideoStat) isTargetAchieved() bool { @@ -79,9 +115,10 @@ func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { } type TargetLabel struct { - Text string `json:"text"` - Scope Scope `json:"scope"` - Regex bool `json:"regex"` + Text string `json:"text"` + Scope Scope `json:"scope"` + Regex bool `json:"regex"` + Target int `json:"target"` // target count for current label } type FeedConfig struct { From 247f6b99750dec7f67f9bfcacc4393692e2531f7 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 00:15:02 +0800 Subject: [PATCH 37/67] refactor: exit with AndroidShellExecError code for adb shell failure --- hrp/internal/code/code.go | 2 +- hrp/pkg/gadb/device.go | 10 ++++++--- hrp/pkg/uixt/android_adb_driver.go | 32 +++++++++++++++++++++-------- hrp/pkg/uixt/android_uia2_driver.go | 2 +- hrp/pkg/uixt/ext.go | 2 +- hrp/pkg/uixt/ios_driver.go | 2 +- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 87c830ea..e7e74aa7 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -63,7 +63,7 @@ var ( var ( AndroidDeviceConnectionError = errors.New("android device connection error") // 60 AndroidDeviceUSBDriverError = errors.New("android device USB driver error") // 61 - AndroidShellExecError = errors.New("android shell exec error") // 62 + AndroidShellExecError = errors.New("android adb shell exec error") // 62 AndroidScreenShotError = errors.New("android screenshot error") // 65 AndroidCaptureLogError = errors.New("android capture log error") // 66 ) diff --git a/hrp/pkg/gadb/device.go b/hrp/pkg/gadb/device.go index 2067755a..8bbf7e92 100644 --- a/hrp/pkg/gadb/device.go +++ b/hrp/pkg/gadb/device.go @@ -3,16 +3,17 @@ package gadb import ( "bytes" "encoding/binary" - "errors" "fmt" "io" "os" "strings" "time" + "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" ) type DeviceFileInfo struct { @@ -244,7 +245,10 @@ func (d *Device) ReverseForwardKillAll() error { func (d *Device) RunShellCommand(cmd string, args ...string) (string, error) { raw, err := d.RunShellCommandWithBytes(cmd, args...) - return string(raw), err + if err != nil { + return "", errors.Wrap(code.AndroidShellExecError, err.Error()) + } + return string(raw), nil } func (d *Device) RunShellCommandWithBytes(cmd string, args ...string) ([]byte, error) { @@ -548,7 +552,7 @@ func (d *Device) InstallAPK(apk io.ReadSeeker) (string, error) { res, err := d.RunShellCommand("pm", "install", "-f", remote) if err != nil { - return "", fmt.Errorf("error installing: %v", err) + return "", errors.Wrap(err, "install apk failed") } if haserr(res) { return "", errors.New(res) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 858ad229..89c33859 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -59,7 +59,7 @@ func (ad *adbDriver) WindowSize() (size Size, err error) { // adb shell wm size resp, err := ad.adbClient.RunShellCommand("wm", "size") if err != nil { - return + return size, errors.Wrap(err, "get window size failed") } // Physical size: 1080x2340 @@ -84,12 +84,15 @@ func (ad *adbDriver) Scale() (scale float64, err error) { func (ad *adbDriver) PressBack(options ...ActionOption) (err error) { // adb shell input keyevent 4 _, err = ad.adbClient.RunShellCommand("input", "keyevent", fmt.Sprintf("%d", KCBack)) - return + if err != nil { + return errors.Wrap(err, "press back failed") + } + return nil } func (ad *adbDriver) StartCamera() (err error) { if _, err = ad.adbClient.RunShellCommand("rm", "-r", "/sdcard/DCIM/Camera"); err != nil { - return err + return errors.Wrap(err, "remove /sdcard/DCIM/Camera failed") } time.Sleep(5 * time.Second) var version string @@ -176,7 +179,7 @@ func (ad *adbDriver) AppTerminate(packageName string) (successful bool, err erro // adb shell am force-stop _, err = ad.adbClient.RunShellCommand("am", "force-stop", packageName) if err != nil { - return false, err + return false, errors.Wrap(err, "force-stop app failed") } if ad.lastLaunchedPackageName == packageName { @@ -198,9 +201,14 @@ func (ad *adbDriver) TapFloat(x, y float64, options ...ActionOption) (err error) } // adb shell input tap x y + xStr := fmt.Sprintf("%.1f", x) + yStr := fmt.Sprintf("%.1f", y) _, err = ad.adbClient.RunShellCommand( - "input", "tap", fmt.Sprintf("%.1f", x), fmt.Sprintf("%.1f", y)) - return + "input", "tap", xStr, yStr) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) + } + return nil } func (ad *adbDriver) DoubleTap(x, y int) error { @@ -241,7 +249,10 @@ func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio fmt.Sprintf("%.1f", fromX), fmt.Sprintf("%.1f", fromY), fmt.Sprintf("%.1f", toX), fmt.Sprintf("%.1f", toY), ) - return err + if err != nil { + return errors.Wrap(err, "swipe failed") + } + return nil } func (ad *adbDriver) ForceTouch(x, y int, pressure float64, second ...float64) error { @@ -266,7 +277,10 @@ func (ad *adbDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error) { // adb shell input text _, err = ad.adbClient.RunShellCommand("input", "text", text) - return + if err != nil { + return errors.Wrap(err, "send keys failed") + } + return nil } func (ad *adbDriver) Input(text string, options ...ActionOption) (err error) { @@ -382,7 +396,7 @@ func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") - return AppInfo{}, errors.Wrap(code.AndroidShellExecError, err.Error()) + return AppInfo{}, errors.Wrap(err, "dumpsys activities failed") } lines := strings.Split(string(output), "\n") diff --git a/hrp/pkg/uixt/android_uia2_driver.go b/hrp/pkg/uixt/android_uia2_driver.go index 5f426e04..a6e92334 100644 --- a/hrp/pkg/uixt/android_uia2_driver.go +++ b/hrp/pkg/uixt/android_uia2_driver.go @@ -163,7 +163,7 @@ func (ud *uiaDriver) WindowSize() (size Size, err error) { // register(getHandler, new GetDeviceSize("/wd/hub/session/:sessionId/window/:windowHandle/size")) var rawResp rawResponse if rawResp, err = ud.httpGET("/session", ud.sessionId, "window/:windowHandle/size"); err != nil { - return Size{}, err + return Size{}, errors.Wrap(err, "get window size failed with uiautomator2") } reply := new(struct{ Value struct{ Size } }) if err = json.Unmarshal(rawResp, reply); err != nil { diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index cb25faa0..0b2bd70b 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -86,7 +86,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) // get device window size dExt.windowSize, err = dExt.Driver.WindowSize() if err != nil { - return nil, errors.Wrap(err, "failed to get windows size") + return nil, err } if dExt.OCRService, err = newVEDEMOCRService(); err != nil { diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 5507de6e..004e7a62 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -137,7 +137,7 @@ func (wd *wdaDriver) WindowSize() (size Size, err error) { // [[FBRoute GET:@"/window/size"] respondWithTarget:self action:@selector(handleGetWindowSize:)] var rawResp rawResponse if rawResp, err = wd.httpGET("/session", wd.sessionId, "/window/size"); err != nil { - return Size{}, err + return Size{}, errors.Wrap(err, "get window size failed with wda") } reply := new(struct{ Value struct{ Size } }) if err = json.Unmarshal(rawResp, reply); err != nil { From 0d8d53d43178cd5d107404abb2b19b3bb61e7448 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 00:23:14 +0800 Subject: [PATCH 38/67] feat: add MobileUIPopupError code --- hrp/internal/code/code.go | 2 ++ hrp/pkg/uixt/video_crawler.go | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index e7e74aa7..9c2ebfa9 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -74,6 +74,7 @@ var ( MobileUILaunchAppError = errors.New("mobile UI launch app error") // 71 MobileUIValidationError = errors.New("mobile UI validation error") // 75 MobileUIAppNotInForegroundError = errors.New("mobile UI app not in foreground error") // 76 + MobileUIPopupError = errors.New("mobile UI popup error") // 77 ) // OCR related: [80, 90) @@ -135,6 +136,7 @@ var errorsMap = map[error]int{ MobileUILaunchAppError: 71, MobileUIValidationError: 75, MobileUIAppNotInForegroundError: 76, + MobileUIPopupError: 77, // OCR related OCREnvMissedError: 80, diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 11237ad6..3f47b313 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -362,7 +362,8 @@ func (dExt *DriverExt) assertActivity(packageName, activityType string) error { } if app.PackageName != packageName { - return fmt.Errorf("app %s is not in foreground", packageName) + return errors.Wrap(code.MobileUIAppNotInForegroundError, + fmt.Sprintf("app %s is not in foreground", packageName)) } if activities, ok := androidActivities[app.PackageName]; ok { @@ -374,7 +375,8 @@ func (dExt *DriverExt) assertActivity(packageName, activityType string) error { } log.Error().Interface("app", app.AppBaseInfo).Msg("app activity not match") - return fmt.Errorf("%s activity is not in foreground", activityType) + return errors.Wrap(code.MobileUIAppNotInForegroundError, + fmt.Sprintf("%s activity is not in foreground", activityType)) } // TODO: add more popup texts @@ -396,7 +398,7 @@ func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { point := points[1].Center() if err := dExt.TapAbsXY(point.X, point.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") - return err + return errors.Wrap(code.MobileUIPopupError, err.Error()) } // tap popup success return nil From 8330e6ccf6741c55ce3b621efbc2fe17e7713835 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 00:39:43 +0800 Subject: [PATCH 39/67] refactor: GetScreenTextsByOCR returns image path and ocr texts --- examples/worldcup/main.go | 4 ++-- hrp/pkg/uixt/demo/main_test.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 10 +++++----- hrp/pkg/uixt/swipe.go | 2 +- hrp/pkg/uixt/video_crawler.go | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index 8670c115..eea2f47e 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { utcTimeStr := utcTime.Format("15:04:05") - ocrTexts, err := wc.driver.GetScreenTextsByOCR() + _, ocrTexts, err := wc.driver.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("get ocr texts failed") return err @@ -212,7 +212,7 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { time.Sleep(5 * time.Second) // 青少年弹窗处理 - if ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { + if _, ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { point := points[1].Center() _ = wc.driver.TapAbsXY(point.X, point.Y) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index c2c8c3dd..8a042a0d 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -35,7 +35,7 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { // take screenshot and get screen texts by OCR - texts, err := driverExt.GetScreenTextsByOCR() + _, texts, err := driverExt.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") t.Fatal(err) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 9b7dd24a..ccac3f15 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -280,15 +280,15 @@ type IOCRService interface { GetTexts(imageBuf *bytes.Buffer) (texts OCRTexts, err error) } -func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { +// GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts. +func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRTexts, err error) { var bufSource *bytes.Buffer - var imagePath string if bufSource, imagePath, err = dExt.TakeScreenShot( builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } - ocrTexts, err := dExt.OCRService.GetTexts(bufSource) + ocrTexts, err = dExt.OCRService.GetTexts(bufSource) if err != nil { log.Error().Err(err).Msg("GetScreenTextsByOCR failed") return @@ -296,11 +296,11 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (texts OCRTexts, err error) { o, _ := json.Marshal(ocrTexts) dExt.cacheStepData.OcrResults[imagePath] = string(o) - return ocrTexts, nil + return imagePath, ocrTexts, nil } func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) { - ocrTexts, err := dExt.GetScreenTextsByOCR() + _, ocrTexts, err := dExt.GetScreenTextsByOCR() if err != nil { return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 7173caf6..06488ef2 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -135,7 +135,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) var point PointF findTexts := func(d *DriverExt) error { var err error - ocrTexts, err := d.GetScreenTextsByOCR() + _, ocrTexts, err := d.GetScreenTextsByOCR() if err != nil { return err } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 3f47b313..3df26c2b 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -89,7 +89,7 @@ func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { driverExt.GenAbsScope(0, 0.5, 1, 1).Option(), } if ocrText, err := texts.FindText("^@", actionOptions...); err == nil { - log.Info().Str("author", ocrText.Text).Msg("found feed author") + log.Debug().Str("author", ocrText.Text).Msg("found feed author") } for _, targetLabel := range s.configs.Feed.TargetLabels { @@ -197,7 +197,7 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } // take screenshot and get screen texts by OCR - _, err := l.driver.GetScreenTextsByOCR() + _, _, err := l.driver.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue @@ -299,7 +299,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // take screenshot and get screen texts by OCR - texts, err := dExt.GetScreenTextsByOCR() + _, texts, err := dExt.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue From 70d0f666ea7607f8db3cd938acd813094196aa9a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 01:21:00 +0800 Subject: [PATCH 40/67] feat: add ocr tags to summary --- hrp/pkg/uixt/ext.go | 13 +++++++++---- hrp/pkg/uixt/ocr_vedem.go | 6 ++++-- hrp/pkg/uixt/video_crawler.go | 25 +++++++++++++++---------- hrp/pkg/uixt/video_crawler_test.go | 2 +- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 0b2bd70b..fec1690e 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -47,11 +47,16 @@ func WithThreshold(threshold float64) CVOption { } } +type OcrResult struct { + Texts OCRTexts `json:"texts"` // dumped OCRTexts + Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] +} + type cacheStepData struct { // cache step screenshot paths ScreenShots []string - // cache step screenshot ocr results, key is image path, value is dumped OCRTexts - OcrResults map[string]string + // cache step screenshot ocr results, key is image path, value is OcrResult + OcrResults map[string]*OcrResult // cache feed/live video stat VideoStat *VideoStat } @@ -76,7 +81,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) Driver: driver, cacheStepData: cacheStepData{ ScreenShots: make([]string, 0), - OcrResults: make(map[string]string), + OcrResults: make(map[string]*OcrResult), }, interruptSignal: make(chan os.Signal, 1), } @@ -188,7 +193,7 @@ func (dExt *DriverExt) GetStepCacheData() cacheStepData { // clear cache dExt.cacheStepData = cacheStepData{ ScreenShots: []string{}, - OcrResults: make(map[string]string), + OcrResults: make(map[string]*OcrResult), } return copied } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index ccac3f15..fda92a98 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -294,8 +294,10 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText return } - o, _ := json.Marshal(ocrTexts) - dExt.cacheStepData.OcrResults[imagePath] = string(o) + dExt.cacheStepData.OcrResults[imagePath] = &OcrResult{ + Texts: ocrTexts, + } + return imagePath, ocrTexts, nil } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 3df26c2b..32a7b450 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -82,14 +82,15 @@ func (s *VideoStat) isTargetAchieved() bool { } // incrFeed increases feed count and feed stat -func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { +func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { // feed author actionOptions := []ActionOption{ WithRegex(true), driverExt.GenAbsScope(0, 0.5, 1, 1).Option(), } - if ocrText, err := texts.FindText("^@", actionOptions...); err == nil { + if ocrText, err := ocrResult.Texts.FindText("^@", actionOptions...); err == nil { log.Debug().Str("author", ocrText.Text).Msg("found feed author") + ocrResult.Tags = append(ocrResult.Tags, ocrText.Text) } for _, targetLabel := range s.configs.Feed.TargetLabels { @@ -98,7 +99,7 @@ func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { WithRegex(targetLabel.Regex), driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(), } - if ocrText, err := texts.FindText(targetLabel.Text, actionOptions...); err == nil { + if ocrText, err := ocrResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil { log.Info().Str("label", targetLabel.Text). Str("text", ocrText.Text).Msg("found feed success") @@ -107,6 +108,7 @@ func (s *VideoStat) incrFeed(texts OCRTexts, driverExt *DriverExt) error { s.FeedStat[key] = 0 } s.FeedStat[key]++ + ocrResult.Tags = append(ocrResult.Tags, key) } } @@ -163,7 +165,7 @@ type LiveCrawler struct { func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) { // 预览流入口 - points, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) + points, err := texts.FindTexts([]string{".?点击进入直播间", "直播中"}, WithRegex(true)) if err == nil { return points[0].Center(), true } @@ -197,11 +199,12 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } // take screenshot and get screen texts by OCR - _, _, err := l.driver.GetScreenTextsByOCR() + imagePath, _, err := l.driver.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } + l.driver.cacheStepData.OcrResults[imagePath].Tags = []string{"live"} // TODO: check live type @@ -299,14 +302,16 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // take screenshot and get screen texts by OCR - _, texts, err := dExt.GetScreenTextsByOCR() + imagePath, texts, err := dExt.GetScreenTextsByOCR() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } + ocrResult := dExt.cacheStepData.OcrResults[imagePath] + ocrResult.Tags = []string{"feed"} // automatic handling of pop-up windows - if err := dExt.autoPopupHandler(texts); err != nil { + if err := dExt.autoPopupHandler(ocrResult); err != nil { log.Error().Err(err).Msg("auto handle popup failed") return err } @@ -326,7 +331,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // check feed type and incr feed count - if err := currVideoStat.incrFeed(texts, dExt); err != nil { + if err := currVideoStat.incrFeed(ocrResult, dExt); err != nil { log.Error().Err(err).Msg("incr feed failed") } @@ -386,13 +391,13 @@ var popups = [][]string{ {"确定", "取消"}, } -func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { +func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { for _, popup := range popups { if len(popup) != 2 { continue } - points, err := texts.FindTexts([]string{"确定", "取消"}) + points, err := ocrResult.Texts.FindTexts([]string{"确定", "取消"}) if err == nil { log.Warn().Msg("text popup found") point := points[1].Center() diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index 8ecd7e88..d74b9016 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -14,7 +14,7 @@ func TestVideoCrawler(t *testing.T) { Feed: FeedConfig{ TargetCount: 5, TargetLabels: []TargetLabel{ - {Text: `^广告$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^广告$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true, Target: 2}, {Text: `^图文$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, {Text: `^特效\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, {Text: `^模板\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, From ab8b5b133fa844a9d09dcd92f11269141f63ea70 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 13:06:57 +0800 Subject: [PATCH 41/67] fix: GetStepCacheData --- .../uitest/demo_android_video_crawler.json | 3 +- .../uitest/demo_android_video_crawler_test.go | 4 +-- hrp/pkg/uixt/ext.go | 32 +++++++++++++++---- hrp/step_mobile_ui.go | 14 ++++---- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 356d62e6..6ad87edd 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -38,7 +38,7 @@ 1, 1 ], - "target": 2, + "target": 1, "text": "^广告$" }, { @@ -49,6 +49,7 @@ 1, 1 ], + "target": 1, "text": "^图文$" }, { diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index 4cda04b3..2d6aaa45 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -25,8 +25,8 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { "feed": map[string]interface{}{ "target_count": 5, "target_labels": []map[string]interface{}{ - {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true, "target": 2}, - {"text": "^图文$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true, "target": 1}, + {"text": "^图文$", "scope": []float64{0, 0.5, 1, 1}, "regex": true, "target": 1}, {"text": `^特效\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, {"text": `^模板\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, {"text": `^购物\|`, "scope": []float64{0, 0.5, 1, 1}, "regex": true}, diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index fec1690e..d7dd00d5 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -2,6 +2,7 @@ package uixt import ( "bytes" + "encoding/json" "fmt" "image" "image/gif" @@ -61,6 +62,12 @@ type cacheStepData struct { VideoStat *VideoStat } +func (d *cacheStepData) reset() { + d.ScreenShots = make([]string, 0) + d.OcrResults = make(map[string]*OcrResult) + d.VideoStat = nil +} + type DriverExt struct { CVArgs Device Device @@ -188,14 +195,25 @@ func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (strin return screenshotPath, nil } -func (dExt *DriverExt) GetStepCacheData() cacheStepData { - copied := dExt.cacheStepData - // clear cache - dExt.cacheStepData = cacheStepData{ - ScreenShots: []string{}, - OcrResults: make(map[string]*OcrResult), +func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { + cacheData := make(map[string]interface{}) + cacheData["video_stat"] = dExt.cacheStepData.VideoStat + cacheData["screenshots"] = dExt.cacheStepData.ScreenShots + + ocrResults := make(map[string]interface{}) + for imagePath, ocrResult := range dExt.cacheStepData.OcrResults { + o, _ := json.Marshal(ocrResult.Texts) + data := map[string]interface{}{ + "tags": ocrResult.Tags, + "texts": string(o), + } + ocrResults[imagePath] = data } - return copied + cacheData["ocr_results"] = ocrResults + + // clear cache + dExt.cacheStepData.reset() + return cacheData } // isPathExists returns true if path exists, whether path is file or dir diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 850330a0..d947baca 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -592,10 +592,10 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // check if app is in foreground packageName := uiDriver.Driver.GetLastLaunchedApp() - err := uiDriver.Driver.AssertAppForeground(packageName) - if packageName != "" && err != nil { - log.Error().Err(err).Str("packageName", packageName).Msg("app is not in foreground") - err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) + err2 := uiDriver.Driver.AssertAppForeground(packageName) + if packageName != "" && err2 != nil { + log.Error().Err(err2).Str("packageName", packageName).Msg("app is not in foreground") + err = errors.Wrap(code.MobileUIAppNotInForegroundError, err2.Error()) } } @@ -608,9 +608,9 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // save attachments cacheData := uiDriver.GetStepCacheData() - attachments["screenshots"] = cacheData.ScreenShots - attachments["ocr_results"] = cacheData.OcrResults - attachments["video_stat"] = cacheData.VideoStat + for key, value := range cacheData { + attachments[key] = value + } stepResult.Attachments = attachments }() From 038f4171ba531ddd5abe55a605b2ed33117efe12 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 17:39:59 +0800 Subject: [PATCH 42/67] feat: add popularity data for feed --- hrp/pkg/uixt/ext.go | 12 ++++++++++-- hrp/pkg/uixt/ocr_vedem.go | 35 +++++++++++++++++++++-------------- hrp/pkg/uixt/video_crawler.go | 27 ++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index d7dd00d5..8384709d 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -48,9 +48,17 @@ func WithThreshold(threshold float64) CVOption { } } +type Popularity struct { + Stars string `json:"stars"` // 点赞数 + Comments string `json:"comments"` // 评论数 + Favorites string `json:"favorites"` // 收藏数 + Shares string `json:"shares"` // 分享数 +} + type OcrResult struct { - Texts OCRTexts `json:"texts"` // dumped OCRTexts - Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] + Texts OCRTexts `json:"texts"` // dumped OCRTexts + Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] + Popularity Popularity `json:"popularity"` // video popularity data } type cacheStepData struct { diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index fda92a98..152b7f40 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -52,26 +52,33 @@ func (t OCRTexts) texts() (texts []string) { return texts } +func (t OCRTexts) FilterScope(scope AbsScope) (results OCRTexts) { + for _, ocrText := range t { + rect := ocrText.Rect + + // check if text in scope + if len(scope) == 4 { + if rect.Min.X < scope[0] || + rect.Min.Y < scope[1] || + rect.Max.X > scope[2] || + rect.Max.Y > scope[3] { + // not in scope + continue + } + } + + results = append(results, ocrText) + } + return +} + func (t OCRTexts) FindText(text string, options ...ActionOption) ( result OCRText, err error) { actionOptions := NewActionOptions(options...) var results []OCRText - for _, ocrText := range t { - rect := ocrText.Rect - - // check if text in scope - if len(actionOptions.AbsScope) == 4 { - if rect.Min.X < actionOptions.AbsScope[0] || - rect.Min.Y < actionOptions.AbsScope[1] || - rect.Max.X > actionOptions.AbsScope[2] || - rect.Max.Y > actionOptions.AbsScope[3] { - // not in scope - continue - } - } - + for _, ocrText := range t.FilterScope(actionOptions.AbsScope) { if actionOptions.Regex { // regex on, check if match regex if !regexp.MustCompile(text).MatchString(ocrText.Text) { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 32a7b450..2b566a17 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -112,6 +112,21 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { } } + // add popularity data for feed + popularityData := ocrResult.Texts.FilterScope(driverExt.GenAbsScope(0.8, 0.5, 1, 0.8)) + if len(popularityData) != 4 { + log.Warn().Interface("popularity", popularityData).Msg("get popularity data failed") + } else { + ocrResult.Popularity = Popularity{ + Stars: popularityData[0].Text, + Comments: popularityData[1].Text, + Favorites: popularityData[2].Text, + Shares: popularityData[3].Text, + } + log.Info().Interface("popularity", ocrResult.Popularity). + Msg("found feed popularity success") + } + s.FeedCount++ return nil } @@ -308,7 +323,6 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { continue } ocrResult := dExt.cacheStepData.OcrResults[imagePath] - ocrResult.Tags = []string{"feed"} // automatic handling of pop-up windows if err := dExt.autoPopupHandler(ocrResult); err != nil { @@ -328,11 +342,14 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { continue } } - } + ocrResult.Tags = []string{"live-preview"} + } else { + ocrResult.Tags = []string{"feed"} - // check feed type and incr feed count - if err := currVideoStat.incrFeed(ocrResult, dExt); err != nil { - log.Error().Err(err).Msg("incr feed failed") + // check feed type and incr feed count + if err := currVideoStat.incrFeed(ocrResult, dExt); err != nil { + log.Error().Err(err).Msg("incr feed failed") + } } // sleep custom random time From 5d9fa3c91ec16f497f2fc6b670e7161c6109b730 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 19:34:42 +0800 Subject: [PATCH 43/67] feat: add popularity data for live --- hrp/pkg/uixt/ext.go | 9 ++++--- hrp/pkg/uixt/video_crawler.go | 48 +++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 8384709d..13d99aac 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -49,10 +49,11 @@ func WithThreshold(threshold float64) CVOption { } type Popularity struct { - Stars string `json:"stars"` // 点赞数 - Comments string `json:"comments"` // 评论数 - Favorites string `json:"favorites"` // 收藏数 - Shares string `json:"shares"` // 分享数 + Stars string `json:"stars,omitempty"` // 点赞数 + Comments string `json:"comments,omitempty"` // 评论数 + Favorites string `json:"favorites,omitempty"` // 收藏数 + Shares string `json:"shares,omitempty"` // 分享数 + LiveUsers string `json:"live_users,omitempty"` // 直播间人数 } type OcrResult struct { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 2b566a17..01225fbf 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -115,7 +115,7 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { // add popularity data for feed popularityData := ocrResult.Texts.FilterScope(driverExt.GenAbsScope(0.8, 0.5, 1, 0.8)) if len(popularityData) != 4 { - log.Warn().Interface("popularity", popularityData).Msg("get popularity data failed") + log.Warn().Interface("popularity", popularityData).Msg("get feed popularity data failed") } else { ocrResult.Popularity = Popularity{ Stars: popularityData[0].Text, @@ -131,6 +131,26 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { return nil } +// incrLive increases live count and live stat +func (s *VideoStat) incrLive(ocrResult *OcrResult, driverExt *DriverExt) error { + // TODO: check live type + + // add popularity data for live + popularityData := ocrResult.Texts.FilterScope(driverExt.GenAbsScope(0.7, 0.05, 1, 0.15)) + if len(popularityData) != 1 { + log.Warn().Interface("popularity", popularityData).Msg("get live popularity data failed") + } else { + ocrResult.Popularity = Popularity{ + LiveUsers: popularityData[0].Text, + } + log.Info().Interface("popularity", ocrResult.Popularity). + Msg("found live popularity success") + } + + s.LiveCount++ + return nil +} + type TargetLabel struct { Text string `json:"text"` Scope Scope `json:"scope"` @@ -213,18 +233,8 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { return err } - // take screenshot and get screen texts by OCR - imagePath, _, err := l.driver.GetScreenTextsByOCR() - if err != nil { - log.Error().Err(err).Msg("OCR GetTexts failed") - continue - } - l.driver.cacheStepData.OcrResults[imagePath].Tags = []string{"live"} - - // TODO: check live type - // swipe to next live video - err = l.driver.SwipeUp() + err := l.driver.SwipeUp() if err != nil { log.Error().Err(err).Msg("swipe up failed") // TODO: retry maximum 3 times @@ -236,9 +246,19 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { log.Error().Err(err).Msg("sleep random failed") } - // TODO: check live type + // take screenshot and get screen texts by OCR + imagePath, _, err := l.driver.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + continue + } + ocrResult := l.driver.cacheStepData.OcrResults[imagePath] + ocrResult.Tags = []string{"live"} - l.currentStat.LiveCount++ + // check live type and incr live count + if err := l.currentStat.incrLive(ocrResult, l.driver); err != nil { + log.Error().Err(err).Msg("incr live failed") + } } } From f720eaacfb9c78854179c9a6d6d30b3c7827fd41 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 20:32:52 +0800 Subject: [PATCH 44/67] change: autoPopupHandler --- hrp/pkg/uixt/video_crawler.go | 10 +++++----- hrp/pkg/uixt/video_crawler_test.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 01225fbf..5e9968c4 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -423,9 +423,8 @@ func (dExt *DriverExt) assertActivity(packageName, activityType string) error { // TODO: add more popup texts var popups = [][]string{ - {"青少年", "我知道了"}, // 青少年弹窗 - {"允许", "拒绝"}, - {"确定", "取消"}, + {"青少年模式", "我知道了"}, // 青少年弹窗 + {"个人信息保护指引", "同意"}, } func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { @@ -434,9 +433,10 @@ func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { continue } - points, err := ocrResult.Texts.FindTexts([]string{"确定", "取消"}) + points, err := ocrResult.Texts.FindTexts([]string{popup[0], popup[1]}) if err == nil { - log.Warn().Msg("text popup found") + log.Warn().Interface("popup", popup). + Interface("texts", ocrResult.Texts).Msg("text popup found") point := points[1].Center() if err := dExt.TapAbsXY(point.X, point.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index d74b9016..5044df5d 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -14,8 +14,8 @@ func TestVideoCrawler(t *testing.T) { Feed: FeedConfig{ TargetCount: 5, TargetLabels: []TargetLabel{ - {Text: `^广告$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true, Target: 2}, - {Text: `^图文$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^广告$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, + {Text: `^图文$`, Scope: Scope{0, 0.5, 1, 1}, Regex: true, Target: 2}, {Text: `^特效\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, {Text: `^模板\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, {Text: `^购物\|`, Scope: Scope{0, 0.5, 1, 1}, Regex: true}, From b480c6058b850bce69498306896607f6580fcdc9 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 4 May 2023 22:37:38 +0800 Subject: [PATCH 45/67] bump version v4.3.4-beta-202305042235 --- docs/CHANGELOG.md | 6 +- go.mod | 36 +- go.sum | 345 +++--------------- .../templates/plugin/debugtalk_gen.go | 2 +- hrp/internal/version/VERSION | 2 +- 5 files changed, 72 insertions(+), 319 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 93b39efd..86ea9958 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## v4.3.4 (2023-05-03) +## v4.3.4 (2023-05-04) **go version** @@ -8,11 +8,13 @@ - feat: cache screenshot ocr texts - feat: set testcase and request timeout in seconds - feat: catch interrupt signal -- feat: add new exit code MobileUILaunchAppError/InterruptError/TimeoutError +- feat: add new exit code MobileUILaunchAppError/InterruptError/TimeoutError/MobileUIPopupError - feat: find text with regex +- feat: add UI ocr tags to summary - refactor: simplify OCR APIs - refactor: FindText(s) returns OCRText(s) - refactor: merge ActionOption with DataOption +- change: exit with AndroidShellExecError code for adb shell failure ## v4.3.3 (2023-04-19) diff --git a/go.mod b/go.mod index 39dc3ec5..e4ed83df 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.18 require ( github.com/andybalholm/brotli v1.0.4 github.com/denisbrodbeck/machineid v1.0.1 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.15.0 github.com/getsentry/sentry-go v0.13.0 github.com/go-openapi/spec v0.20.7 github.com/go-ping/ping v1.1.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/gorilla/websocket v1.5.0 - github.com/httprunner/funplugin v0.5.0 + github.com/httprunner/funplugin v0.5.1 github.com/jinzhu/copier v0.3.5 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 @@ -20,26 +20,28 @@ require ( github.com/miekg/dns v1.1.50 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/otiai10/gosseract/v2 v2.4.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.13.0 github.com/rs/zerolog v1.29.1 github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.2 gocv.io/x/gocv v0.32.1 - golang.org/x/net v0.7.0 - golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 - google.golang.org/grpc v1.49.0 - google.golang.org/protobuf v1.28.1 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.6.0 + google.golang.org/grpc v1.54.0 + google.golang.org/protobuf v1.30.0 gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 ) require ( - cloud.google.com/go/compute v1.7.0 // indirect + cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -47,10 +49,10 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.22.3 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/go-hclog v1.3.0 // indirect - github.com/hashicorp/go-plugin v1.4.5 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-plugin v1.4.9 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -73,13 +75,13 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.5.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 58a172df..17115b67 100644 --- a/go.sum +++ b/go.sum @@ -13,37 +13,18 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -53,11 +34,9 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,30 +44,20 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -101,19 +70,12 @@ github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbj github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -156,8 +118,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -173,10 +133,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -186,19 +145,12 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -206,55 +158,33 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.3.0 h1:G0ACM8Z2WilWgPv3Vdzwm3V0BQu/kSmrkVtpe1fy9do= -github.com/hashicorp/go-hclog v1.3.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= -github.com/hashicorp/go-plugin v1.4.5 h1:oTE/oQR4eghggRg8VY7PAz3dr++VwDNBGCcOfIvHpBo= -github.com/hashicorp/go-plugin v1.4.5/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.4.9 h1:ESiK220/qE0aGxWdzKIvRH69iLiuN/PjoLTm69RoWtU= +github.com/hashicorp/go-plugin v1.4.9/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= -github.com/httprunner/funplugin v0.5.0 h1:Laoe8URu71qeyST9wvRtGSkDWc8Y3T1IrnvFSTHmO84= -github.com/httprunner/funplugin v0.5.0/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= +github.com/httprunner/funplugin v0.5.1 h1:7CbdN0jfSn8GOWgdxiHqqModIZ6pintqVZwPRw9Koww= +github.com/httprunner/funplugin v0.5.1/go.mod h1:o6l442jWROJgQytrEa9E/PgmL2uKA8c2AWeaYH4wSkg= github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= -github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -277,8 +207,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -292,13 +222,10 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maja42/goval v1.2.1 h1:fyEgzddqPgCZsKcFLk4C6SdCHyEaAHYvtZG4mGzQOHU= github.com/maja42/goval v1.2.1/go.mod h1:42LU+BQXL/veE9jnTTUOSj38GRmOTSThYSXRVodI5J4= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -311,7 +238,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -326,11 +252,17 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/gosseract/v2 v2.4.0 h1:gYd3mx6FuMtIlxL4sYb9JLCFEDzg09VgNSZRNbqpiGM= +github.com/otiai10/gosseract/v2 v2.4.0/go.mod h1:fhbIDRh29bj13vni6RT3gtWKjKCAeqDYI4C1dxeJuek= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -366,11 +298,9 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -382,7 +312,6 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -390,16 +319,17 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= @@ -408,9 +338,7 @@ github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKo github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -418,9 +346,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= gocv.io/x/gocv v0.32.1 h1:BC9hHs5+47nVgySUFVKntc6RsF3SULFzqk6OV9xz+C0= gocv.io/x/gocv v0.32.1/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -429,7 +354,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -452,8 +376,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -462,12 +384,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,48 +415,23 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -546,17 +440,14 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,7 +458,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -587,45 +477,20 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= @@ -637,12 +502,10 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -686,28 +549,13 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -724,29 +572,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -755,7 +580,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -779,65 +603,14 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220314164441-57ef72a4c106/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51 h1:ucpgjuzWqWrj0NEwjUpsGTf2IGxyLtmuSk0oGgifjec= -google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -850,28 +623,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -884,10 +637,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -898,7 +649,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -907,7 +657,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index 1590229f..ca8bcde3 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.3.3, DO NOT EDIT! +// NOTE: Generated By hrp v4.3.4-beta-202305032303, DO NOT EDIT! package main import ( diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 8ef9558c..720b54ba 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305032303 \ No newline at end of file +v4.3.4-beta-202305042235 \ No newline at end of file From 41e4d89c95a5ae18481030ac19b8a8bc3018a5f4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 9 May 2023 20:12:14 +0800 Subject: [PATCH 46/67] feat: request vedem ocr with uploading image --- examples/uitest/demo_android_live_swipe.json | 23 +++++--- hrp/pkg/uixt/ext.go | 15 +++--- hrp/pkg/uixt/ocr_vedem.go | 57 ++++++++++++++------ hrp/pkg/uixt/ocr_vedem_test.go | 11 ++-- 4 files changed, 71 insertions(+), 35 deletions(-) diff --git a/examples/uitest/demo_android_live_swipe.json b/examples/uitest/demo_android_live_swipe.json index bd79ad18..4c5edadc 100644 --- a/examples/uitest/demo_android_live_swipe.json +++ b/examples/uitest/demo_android_live_swipe.json @@ -45,7 +45,9 @@ { "method": "tap_ocr", "params": "我知道了", - "ignore_NotFoundError": true + "options": { + "ignore_NotFoundError": true + } } ] } @@ -57,8 +59,10 @@ { "method": "swipe_to_tap_text", "params": "点击进入直播间", - "identifier": "进入直播间", - "max_retry_times": 10 + "options": { + "identifier": "进入直播间", + "max_retry_times": 10 + } } ] } @@ -69,7 +73,10 @@ "actions": [ { "method": "swipe", - "params": "up" + "params": "up", + "options": { + + } }, { "method": "sleep_random", @@ -93,7 +100,9 @@ { "method": "swipe", "params": "up", - "identifier": "第一次上划" + "options": { + "identifier": "第一次上划" + } }, { "method": "sleep", @@ -105,7 +114,9 @@ { "method": "swipe", "params": "up", - "identifier": "第二次上划" + "options": { + "identifier": "第二次上划" + } }, { "method": "sleep", diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 13d99aac..b774ffb7 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -64,7 +64,8 @@ type OcrResult struct { type cacheStepData struct { // cache step screenshot paths - ScreenShots []string + ScreenShots []string + screenShotsUrls map[string]string // map screenshot file path to uploaded url // cache step screenshot ocr results, key is image path, value is OcrResult OcrResults map[string]*OcrResult // cache feed/live video stat @@ -73,6 +74,7 @@ type cacheStepData struct { func (d *cacheStepData) reset() { d.ScreenShots = make([]string, 0) + d.screenShotsUrls = make(map[string]string) d.OcrResults = make(map[string]*OcrResult) d.VideoStat = nil } @@ -93,14 +95,12 @@ type DriverExt struct { func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) { dExt = &DriverExt{ - Device: device, - Driver: driver, - cacheStepData: cacheStepData{ - ScreenShots: make([]string, 0), - OcrResults: make(map[string]*OcrResult), - }, + Device: device, + Driver: driver, + cacheStepData: cacheStepData{}, interruptSignal: make(chan os.Signal, 1), } + dExt.cacheStepData.reset() signal.Notify(dExt.interruptSignal, syscall.SIGTERM, syscall.SIGINT) dExt.doneMjpegStream = make(chan bool, 1) @@ -208,6 +208,7 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { cacheData := make(map[string]interface{}) cacheData["video_stat"] = dExt.cacheStepData.VideoStat cacheData["screenshots"] = dExt.cacheStepData.ScreenShots + cacheData["screenshots_urls"] = dExt.cacheStepData.screenShotsUrls ocrResults := make(map[string]interface{}) for imagePath, ocrResult := range dExt.cacheStepData.OcrResults { diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 152b7f40..d2b1111f 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -31,6 +31,7 @@ type OCRResult struct { type ResponseOCR struct { Code int `json:"code"` Message string `json:"message"` + URL string `json:"url"` // image uploaded url OCRResult []OCRResult `json:"ocrResult"` } @@ -141,33 +142,40 @@ func newVEDEMOCRService() (*veDEMOCRService, error) { // veDEMOCRService implements IOCRService interface type veDEMOCRService struct{} -func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, error) { +func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( + ocrResutls []OCRResult, url string, err error) { + bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) bodyWriter.WriteField("withDet", "true") + bodyWriter.WriteField("upload", "true") // get image uploaded url // bodyWriter.WriteField("timestampOnly", "true") formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") if err != nil { - return nil, errors.Wrap(code.OCRRequestError, + err = errors.Wrap(code.OCRRequestError, fmt.Sprintf("create form file error: %v", err)) + return } size, err := formWriter.Write(imageBuf.Bytes()) if err != nil { - return nil, errors.Wrap(code.OCRRequestError, + err = errors.Wrap(code.OCRRequestError, fmt.Sprintf("write form error: %v", err)) + return } err = bodyWriter.Close() if err != nil { - return nil, errors.Wrap(code.OCRRequestError, + err = errors.Wrap(code.OCRRequestError, fmt.Sprintf("close body writer error: %v", err)) + return } req, err := http.NewRequest("POST", env.VEDEM_OCR_URL, bodyBuf) if err != nil { - return nil, errors.Wrap(code.OCRRequestError, + err = errors.Wrap(code.OCRRequestError, fmt.Sprintf("construct request error: %v", err)) + return } token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, bodyBuf.Bytes()) @@ -195,41 +203,52 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err log.Error().Err(err). Str("X-TT-LOGID", logID). Int("imageBufSize", size). - Msgf("request OCR service failed, retry %d", i) + Msgf("request veDEM OCR service failed, retry %d", i) time.Sleep(1 * time.Second) } if resp == nil { - return nil, code.OCRServiceConnectionError + err = code.OCRServiceConnectionError + return } defer resp.Body.Close() results, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrap(code.OCRResponseError, + err = errors.Wrap(code.OCRResponseError, fmt.Sprintf("read response body error: %v", err)) + return } if resp.StatusCode != http.StatusOK { - return nil, errors.Wrap(code.OCRResponseError, + err = errors.Wrap(code.OCRResponseError, fmt.Sprintf("unexpected response status code: %d, results: %v", resp.StatusCode, string(results))) + return } var ocrResult ResponseOCR err = json.Unmarshal(results, &ocrResult) if err != nil { - return nil, errors.Wrap(code.OCRResponseError, + err = errors.Wrap(code.OCRResponseError, fmt.Sprintf("json unmarshal response body error: %v", err)) + return } - return ocrResult.OCRResult, nil + if ocrResult.Code != 0 { + log.Error(). + Int("code", ocrResult.Code). + Str("message", ocrResult.Message). + Msg("request veDEM OCR service failed") + } + + return ocrResult.OCRResult, ocrResult.URL, nil } func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer) ( - ocrTexts OCRTexts, err error) { + ocrTexts OCRTexts, url string, err error) { - ocrResults, err := s.getOCRResult(imageBuf) + ocrResults, url, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return @@ -262,6 +281,7 @@ func checkEnv() error { if env.VEDEM_OCR_URL == "" { return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_URL missed") } + log.Info().Str("VEDEM_OCR_URL", env.VEDEM_OCR_URL).Msg("get env") if env.VEDEM_OCR_AK == "" { return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_AK missed") } @@ -284,7 +304,8 @@ func getLogID(header http.Header) string { } type IOCRService interface { - GetTexts(imageBuf *bytes.Buffer) (texts OCRTexts, err error) + // GetTexts returns ocr texts and uploaded image url + GetTexts(imageBuf *bytes.Buffer) (texts OCRTexts, url string, err error) } // GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts. @@ -295,12 +316,18 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText return } - ocrTexts, err = dExt.OCRService.GetTexts(bufSource) + var imageUrl string + ocrTexts, imageUrl, err = dExt.OCRService.GetTexts(bufSource) if err != nil { log.Error().Err(err).Msg("GetScreenTextsByOCR failed") return } + if imageUrl != "" { + dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl + log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot") + } + dExt.cacheStepData.OcrResults[imagePath] = &OcrResult{ Texts: ocrTexts, } diff --git a/hrp/pkg/uixt/ocr_vedem_test.go b/hrp/pkg/uixt/ocr_vedem_test.go index fa409fc9..afa2262f 100644 --- a/hrp/pkg/uixt/ocr_vedem_test.go +++ b/hrp/pkg/uixt/ocr_vedem_test.go @@ -14,22 +14,19 @@ func checkOCR(buff *bytes.Buffer) error { if err != nil { return err } - ocrResults, err := service.getOCRResult(buff) + ocrResults, url, err := service.getOCRResult(buff) if err != nil { return err } fmt.Println(ocrResults) + fmt.Println(url) return nil } func TestOCRWithScreenshot(t *testing.T) { - device, _ := NewAndroidDevice() - driver, err := device.NewDriver(nil) - if err != nil { - t.Fatal(err) - } + setupAndroid(t) - raw, err := driver.Driver.Screenshot() + raw, err := driverExt.Driver.Screenshot() if err != nil { t.Fatal(err) } From 56e065c10ba2e1e9ecf371e2ca24a4dbccdf293a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 10 May 2023 16:23:47 +0800 Subject: [PATCH 47/67] change: add ks --- docs/CHANGELOG.md | 3 +- .../uitest/demo_android_video_crawler.json | 8 +- .../uitest/demo_android_video_crawler_test.go | 46 +++++++++ .../uitest/demo_android_video_ks_crawler.json | 98 +++++++++++++++++++ hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/video_crawler.go | 15 ++- 6 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 examples/uitest/demo_android_video_ks_crawler.json diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 86ea9958..bf8c500b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## v4.3.4 (2023-05-04) +## v4.3.4 (2023-05-10) **go version** @@ -15,6 +15,7 @@ - refactor: FindText(s) returns OCRText(s) - refactor: merge ActionOption with DataOption - change: exit with AndroidShellExecError code for adb shell failure +- change: request vedem ocr with uploading image ## v4.3.3 (2023-04-19) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 6ad87edd..ce65f138 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -18,7 +18,7 @@ { "method": "video_crawler", "params": { - "app_package_name": "com.ss.android.ugc.aweme", + "app_package_name": "com.smile.gifmaker", "feed": { "sleep_random": [ 0, @@ -38,7 +38,7 @@ 1, 1 ], - "target": 1, + "target": 0, "text": "^广告$" }, { @@ -49,7 +49,7 @@ 1, 1 ], - "target": 1, + "target": 0, "text": "^图文$" }, { @@ -89,7 +89,7 @@ 15, 20 ], - "target_count": 3 + "target_count": 0 }, "timeout": 600 } diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index 2d6aaa45..3ce48f88 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -56,3 +56,49 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { t.Fatal(err) } } + +func TestAndroidVideoCrawlerKSTest(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("抓取 KS 视频信息"). + WithVariables(map[string]interface{}{ + "device": "${ENV(SerialNumber)}", + }). + SetAndroid(uixt.WithSerialNumber("$device")), + TestSteps: []hrp.IStep{ + hrp.NewStep("滑动消费 feed 至少 100 个；滑动过程中，70% 随机间隔 0-5s，30% 随机间隔 5-10s"). + Android(). + VideoCrawler(map[string]interface{}{ + "app_package_name": "com.smile.gifmaker", + "timeout": 3600, + "feed": map[string]interface{}{ + "target_count": 100, + "target_labels": []map[string]interface{}{ + {"text": "^广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": "^推广$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + {"text": "^磁力广告$", "scope": []float64{0, 0.5, 1, 1}, "regex": true}, + }, + "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, + }, + "live": map[string]interface{}{ + "target_count": 0, + "sleep_random": []float64{15, 20}, + }, + }), + hrp.NewStep("exit"). + Android(). + AppTerminate("com.smile.gifmaker"). + Validate(). + AssertAppNotInForeground("com.smile.gifmaker"), + }, + } + + if err := testCase.Dump2JSON("demo_android_video_ks_crawler.json"); err != nil { + t.Fatal(err) + } + + runner := hrp.NewRunner(t).SetSaveTests(true) + err := runner.Run(testCase) + if err != nil { + t.Fatal(err) + } +} diff --git a/examples/uitest/demo_android_video_ks_crawler.json b/examples/uitest/demo_android_video_ks_crawler.json new file mode 100644 index 00000000..0ca26d1d --- /dev/null +++ b/examples/uitest/demo_android_video_ks_crawler.json @@ -0,0 +1,98 @@ +{ + "config": { + "name": "抓取 KS 视频信息", + "variables": { + "device": "${ENV(SerialNumber)}" + }, + "android": [ + { + "serial": "$device" + } + ] + }, + "teststeps": [ + { + "name": "滑动消费 feed 至少 100 个；滑动过程中，70% 随机间隔 0-5s，30% 随机间隔 5-10s", + "android": { + "actions": [ + { + "method": "video_crawler", + "params": { + "app_package_name": "com.smile.gifmaker", + "feed": { + "sleep_random": [ + 0, + 5, + 0.7, + 5, + 10, + 0.3 + ], + "target_count": 100, + "target_labels": [ + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^广告$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^推广$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^磁力广告$" + } + ] + }, + "live": { + "sleep_random": [ + 15, + 20 + ], + "target_count": 0 + }, + "timeout": 3600 + } + } + ] + } + }, + { + "name": "exit", + "android": { + "actions": [ + { + "method": "app_terminate", + "params": "com.smile.gifmaker" + } + ] + }, + "validate": [ + { + "check": "ui_foreground_app", + "assert": "not_equal", + "expect": "com.smile.gifmaker", + "msg": "app [com.smile.gifmaker] should not be in foreground" + } + ] + } + ] +} \ No newline at end of file diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 720b54ba..ccaca37a 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305042235 \ No newline at end of file +v4.3.4-beta-202305101622 \ No newline at end of file diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 5e9968c4..8cf86f6e 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -199,12 +199,23 @@ type LiveCrawler struct { } func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) { - // 预览流入口 - points, err := texts.FindTexts([]string{".?点击进入直播间", "直播中"}, WithRegex(true)) + // 预览流入口：DY/KS + points, err := texts.FindTexts([]string{".?点击进入直播间"}, WithRegex(true)) if err == nil { return points[0].Center(), true } + // 预览流入口：KS + points, err = texts.FindTexts([]string{"和主播聊聊天.*"}, WithRegex(true)) + if err == nil { + point := points[0].Center() + enterPoint = PointF{ + X: point.X, + Y: point.Y - 100, + } + return enterPoint, true + } + // TODO: 头像入口 return PointF{}, false From b6373fd1a169b67841f76309ec669a76e6554052 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 11 May 2023 16:41:15 +0800 Subject: [PATCH 48/67] change: remove AGW body verification --- docs/CHANGELOG.md | 2 +- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/ocr_vedem.go | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bf8c500b..a7b54304 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## v4.3.4 (2023-05-10) +## v4.3.4 (2023-05-11) **go version** diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index ccaca37a..0891f9ad 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305101622 \ No newline at end of file +v4.3.4-beta-202305111640 \ No newline at end of file diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index d2b1111f..d9aad9ed 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -178,8 +178,11 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( return } - token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, bodyBuf.Bytes()) + signToken := "UNSIGNED-PAYLOAD" + token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, []byte(signToken)) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Agw-Auth-Content", signToken) req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) var resp *http.Response From 1ba13cf6b247a39a1efb64a4841144398e758d73 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 11 May 2023 16:47:53 +0800 Subject: [PATCH 49/67] change: handle popup with regex --- hrp/pkg/uixt/video_crawler.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 8cf86f6e..0a642f61 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -200,7 +200,7 @@ type LiveCrawler struct { func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) { // 预览流入口：DY/KS - points, err := texts.FindTexts([]string{".?点击进入直播间"}, WithRegex(true)) + points, err := texts.FindTexts([]string{".*点击进入直播间"}, WithRegex(true)) if err == nil { return points[0].Center(), true } @@ -434,8 +434,9 @@ func (dExt *DriverExt) assertActivity(packageName, activityType string) error { // TODO: add more popup texts var popups = [][]string{ - {"青少年模式", "我知道了"}, // 青少年弹窗 - {"个人信息保护指引", "同意"}, + {".*青少年.*", "我知道了"}, // 青少年弹窗 + {".*个人信息保护.*", "同意"}, + {".*更新.*", "以后再说"}, } func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { @@ -444,7 +445,7 @@ func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { continue } - points, err := ocrResult.Texts.FindTexts([]string{popup[0], popup[1]}) + points, err := ocrResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true)) if err == nil { log.Warn().Interface("popup", popup). Interface("texts", ocrResult.Texts).Msg("text popup found") From 786fdfe405acdd4e02f2238d1a43986be243baad Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 11 May 2023 19:33:11 +0800 Subject: [PATCH 50/67] change: update feed/live found log --- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/video_crawler.go | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 0891f9ad..e88d1a3c 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305111640 \ No newline at end of file +v4.3.4-beta-202305111932 \ No newline at end of file diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 0a642f61..31171079 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -99,10 +99,7 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { WithRegex(targetLabel.Regex), driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(), } - if ocrText, err := ocrResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil { - log.Info().Str("label", targetLabel.Text). - Str("text", ocrText.Text).Msg("found feed success") - + if _, err := ocrResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil { key := targetLabel.Text if _, ok := s.FeedStat[key]; !ok { s.FeedStat[key] = 0 @@ -123,10 +120,11 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { Favorites: popularityData[2].Text, Shares: popularityData[3].Text, } - log.Info().Interface("popularity", ocrResult.Popularity). - Msg("found feed popularity success") } + log.Info().Strs("tags", ocrResult.Tags). + Interface("popularity", ocrResult.Popularity). + Msg("found feed success") s.FeedCount++ return nil } @@ -143,10 +141,11 @@ func (s *VideoStat) incrLive(ocrResult *OcrResult, driverExt *DriverExt) error { ocrResult.Popularity = Popularity{ LiveUsers: popularityData[0].Text, } - log.Info().Interface("popularity", ocrResult.Popularity). - Msg("found live popularity success") } + log.Info().Strs("tags", ocrResult.Tags). + Interface("popularity", ocrResult.Popularity). + Msg("found live success") s.LiveCount++ return nil } From da9fde63bf5ba317caaf4a202b903e56adcc6f60 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Thu, 11 May 2023 20:20:14 +0800 Subject: [PATCH 51/67] fix: tap X button on upper-right corner to exit live room --- hrp/pkg/uixt/video_crawler.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 31171079..94048787 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -293,6 +293,17 @@ func (l *LiveCrawler) exitLiveRoom() error { return nil } + // click X button on upper-right corner + if err := l.driver.TapXY(0.95, 0.05); err == nil { + log.Info().Msg("tap X button on upper-right corner to exit live room") + time.Sleep(2 * time.Second) + + // check if back to feed page + if err := l.driver.assertActivity(l.configs.AppPackageName, "feed"); err == nil { + return nil + } + } + return errors.New("exit live room failed") } From a36c4b7a9a3719b3b08f4102961b0fbdf101f7d4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 14:48:04 +0800 Subject: [PATCH 52/67] fix: avoid panic when ActionOptions is nil --- hrp/pkg/uixt/action.go | 4 ++++ hrp/pkg/uixt/video_crawler.go | 1 + 2 files changed, 5 insertions(+) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 5e762e11..be503b62 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -106,6 +106,10 @@ type ActionOptions struct { func (o *ActionOptions) Options() []ActionOption { options := make([]ActionOption, 0) + if o == nil { + return options + } + if o.Identifier != "" { options = append(options, WithIdentifier(o.Identifier)) } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 94048787..3f7da689 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -447,6 +447,7 @@ var popups = [][]string{ {".*青少年.*", "我知道了"}, // 青少年弹窗 {".*个人信息保护.*", "同意"}, {".*更新.*", "以后再说"}, + {".*定位.*", ".*允许.*"}, } func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { From 788d2aed9c84526516c842281d1bb2d2a05cb6cc Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 15:50:09 +0800 Subject: [PATCH 53/67] change: set alias WithWaitTime for compatibility --- hrp/pkg/uixt/action.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index be503b62..138474eb 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -266,6 +266,9 @@ func WithIndex(index int) ActionOption { } } +// set alias for compatibility +var WithWaitTime = WithInterval + func WithInterval(sec float64) ActionOption { return func(o *ActionOptions) { o.Interval = sec From 4c6cce1d178c0566ecc9397af234b7d30ac173ad Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 16:42:08 +0800 Subject: [PATCH 54/67] fix: get foreground app --- hrp/pkg/uixt/android_adb_driver.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 89c33859..8017139c 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -392,7 +392,7 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { } func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { - // adb shell dumpsys activity activities | grep mResumedActivity + // adb shell dumpsys activity activities output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") @@ -402,8 +402,10 @@ func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { lines := strings.Split(string(output), "\n") for _, line := range lines { trimmedLine := strings.TrimSpace(line) - if strings.HasPrefix(trimmedLine, "mResumedActivity:") { + // grep mResumedActivity|ResumedActivity + if strings.HasPrefix(trimmedLine, "mResumedActivity:") || strings.HasPrefix(trimmedLine, "ResumedActivity:") { // mResumedActivity: ActivityRecord{9656d74 u0 com.android.settings/.Settings t407} + // ResumedActivity: ActivityRecord{8265c25 u0 com.android.settings/.Settings t73} strs := strings.Split(trimmedLine, " ") for _, str := range strs { if strings.Contains(str, "/") { From 21a2f9fd226d4dd65f532ca52fc18f348acc32ad Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 17:40:41 +0800 Subject: [PATCH 55/67] refactor: check feed activity --- hrp/pkg/uixt/android_adb_driver.go | 3 ++- hrp/pkg/uixt/video_crawler.go | 31 ++++++++++++++++++++++-------- hrp/step_mobile_ui.go | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 8017139c..650aa9b8 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -386,7 +386,8 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { return err } if app.PackageName != packageName { - return errors.New("app is not in foreground") + return fmt.Errorf("%v is not in foreground, current is %v", + packageName, app.PackageName) } return nil } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 3f7da689..440e5ded 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -329,10 +329,22 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { }() // launch app - if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { - return err + if configs.AppPackageName != "" { + if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { + return err + } + time.Sleep(5 * time.Second) + } else { + app, err := dExt.Driver.GetForegroundApp() + if err != nil { + return err + } + log.Info(). + Str("packageName", app.PackageName). + Str("activity", app.Activity). + Msg("start to video crawler for current foreground app") + configs.AppPackageName = app.PackageName } - time.Sleep(5 * time.Second) liveCrawler := LiveCrawler{ driver: dExt, @@ -352,11 +364,6 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Warn().Msg("interrupted in feed crawler") return errors.Wrap(code.InterruptError, "feed crawler interrupted") default: - // check if feed page - if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { - return err - } - // take screenshot and get screen texts by OCR imagePath, texts, err := dExt.GetScreenTextsByOCR() if err != nil { @@ -411,6 +418,11 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return err } time.Sleep(1 * time.Second) + + // check if feed page + if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { + return err + } } } } @@ -448,6 +460,8 @@ var popups = [][]string{ {".*个人信息保护.*", "同意"}, {".*更新.*", "以后再说"}, {".*定位.*", ".*允许.*"}, + {".*拍照.*", "仅.*允许"}, + {".*录音.*", "仅.*允许"}, } func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { @@ -461,6 +475,7 @@ func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { log.Warn().Interface("popup", popup). Interface("texts", ocrResult.Texts).Msg("text popup found") point := points[1].Center() + log.Info().Str("text", points[1].Text).Msg("close popup") if err := dExt.TapAbsXY(point.X, point.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") return errors.Wrap(code.MobileUIPopupError, err.Error()) diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index d947baca..87a1792a 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -594,7 +594,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err packageName := uiDriver.Driver.GetLastLaunchedApp() err2 := uiDriver.Driver.AssertAppForeground(packageName) if packageName != "" && err2 != nil { - log.Error().Err(err2).Str("packageName", packageName).Msg("app is not in foreground") + log.Error().Err(err2).Msg("app is not in foreground") err = errors.Wrap(code.MobileUIAppNotInForegroundError, err2.Error()) } } From b0e7a6609f7aa63da4a7416a136fd54ad3c63a57 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 21:31:51 +0800 Subject: [PATCH 56/67] change: check android device offline --- hrp/pkg/gadb/device.go | 3 +++ hrp/pkg/gadb/transport.go | 19 +++++++++++++++---- hrp/pkg/uixt/video_crawler.go | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/hrp/pkg/gadb/device.go b/hrp/pkg/gadb/device.go index 8bbf7e92..2245c5af 100644 --- a/hrp/pkg/gadb/device.go +++ b/hrp/pkg/gadb/device.go @@ -246,6 +246,9 @@ func (d *Device) ReverseForwardKillAll() error { func (d *Device) RunShellCommand(cmd string, args ...string) (string, error) { raw, err := d.RunShellCommandWithBytes(cmd, args...) if err != nil { + if errors.Is(err, code.AndroidDeviceConnectionError) { + return "", err + } return "", errors.Wrap(code.AndroidShellExecError, err.Error()) } return string(raw), nil diff --git a/hrp/pkg/gadb/transport.go b/hrp/pkg/gadb/transport.go index c450d053..b59ed42e 100644 --- a/hrp/pkg/gadb/transport.go +++ b/hrp/pkg/gadb/transport.go @@ -1,15 +1,18 @@ package gadb import ( - "errors" "fmt" "io" "io/ioutil" "net" + "regexp" "strconv" "time" + "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/code" ) var ErrConnBroken = errors.New("socket connection broken") @@ -45,6 +48,8 @@ func (t transport) Conn() net.Conn { return t.sock } +var regexDeviceOffline = regexp.MustCompile("device .* not found") + func (t transport) VerifyResponse() (err error) { var status string if status, err = t.ReadStringN(4); err != nil { @@ -58,9 +63,15 @@ func (t transport) VerifyResponse() (err error) { if sError, err = t.UnpackString(); err != nil { return err } - err = fmt.Errorf("command failed: %s", sError) - log.Error().Str("status", status).Str("err", sError).Msg("verify adb response failed") - return + + if regexDeviceOffline.MatchString(sError) { + // device offline + return errors.Wrap(code.AndroidDeviceConnectionError, sError) + } + + log.Warn().Str("status", status).Str("err", sError). + Msg("verify adb response failed") + return errors.New(sError) } func (t transport) ReadStringAll() (s string, err error) { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 440e5ded..40ee16e6 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -462,6 +462,7 @@ var popups = [][]string{ {".*定位.*", ".*允许.*"}, {".*拍照.*", "仅.*允许"}, {".*录音.*", "仅.*允许"}, + {"管理使用时间", ".*忽略.*"}, } func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { From 809a01148880d2c7df607e156ed4fb62209a65e4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 30 May 2023 22:59:58 +0800 Subject: [PATCH 57/67] refactor: use vedem image api --- hrp/internal/env/env.go | 6 +-- hrp/pkg/uixt/README.md | 6 +-- hrp/pkg/uixt/ocr_vedem.go | 73 ++++++++++++++++++++++------------ hrp/pkg/uixt/ocr_vedem_test.go | 5 +-- 4 files changed, 55 insertions(+), 35 deletions(-) diff --git a/hrp/internal/env/env.go b/hrp/internal/env/env.go index 03b0c3c9..8bb808bf 100644 --- a/hrp/internal/env/env.go +++ b/hrp/internal/env/env.go @@ -10,9 +10,9 @@ var ( WDA_USB_DRIVER = os.Getenv("WDA_USB_DRIVER") WDA_LOCAL_PORT = os.Getenv("WDA_LOCAL_PORT") WDA_LOCAL_MJPEG_PORT = os.Getenv("WDA_LOCAL_MJPEG_PORT") - VEDEM_OCR_URL = os.Getenv("VEDEM_OCR_URL") - VEDEM_OCR_AK = os.Getenv("VEDEM_OCR_AK") - VEDEM_OCR_SK = os.Getenv("VEDEM_OCR_SK") + VEDEM_IMAGE_URL = os.Getenv("VEDEM_IMAGE_URL") + VEDEM_IMAGE_AK = os.Getenv("VEDEM_IMAGE_AK") + VEDEM_IMAGE_SK = os.Getenv("VEDEM_IMAGE_SK") DISABLE_GA = os.Getenv("DISABLE_GA") DISABLE_SENTRY = os.Getenv("DISABLE_SENTRY") PYPI_INDEX_URL = os.Getenv("PYPI_INDEX_URL") diff --git a/hrp/pkg/uixt/README.md b/hrp/pkg/uixt/README.md index c422d5d6..e5d4dd3a 100644 --- a/hrp/pkg/uixt/README.md +++ b/hrp/pkg/uixt/README.md @@ -28,9 +28,9 @@ You can get more installation introduction on [hybridgroup/gocv]. OCR API is a paid service, you need to pre-purchase and configure the environment variables. -- VEDEM_OCR_URL -- VEDEM_OCR_AK -- VEDEM_OCR_SK +- VEDEM_IMAGE_URL +- VEDEM_IMAGE_AK +- VEDEM_IMAGE_SK ## Thanks diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index d9aad9ed..e915ef76 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -28,11 +28,16 @@ type OCRResult struct { Points []PointF `json:"points"` } -type ResponseOCR struct { - Code int `json:"code"` - Message string `json:"message"` - URL string `json:"url"` // image uploaded url - OCRResult []OCRResult `json:"ocrResult"` +type ImageResult struct { + URL string `json:"url"` // image uploaded url + OCRResult []OCRResult `json:"ocrResult"` // OCR texts + LiveType string `json:"liveType"` // 直播间类型 +} + +type ImageResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result ImageResult `json:"result"` } type OCRText struct { @@ -142,14 +147,22 @@ func newVEDEMOCRService() (*veDEMOCRService, error) { // veDEMOCRService implements IOCRService interface type veDEMOCRService struct{} -func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( - ocrResutls []OCRResult, url string, err error) { +var actions = []string{ + "ocr", // get ocr texts + "upload", // get image uploaded url + "liveType", // get live type + // "popup", + // "close", +} + +func (s *veDEMOCRService) getImageResult(imageBuf *bytes.Buffer) ( + imageResult ImageResult, err error) { bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) - bodyWriter.WriteField("withDet", "true") - bodyWriter.WriteField("upload", "true") // get image uploaded url - // bodyWriter.WriteField("timestampOnly", "true") + for _, action := range actions { + bodyWriter.WriteField("actions", action) + } formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") if err != nil { @@ -171,7 +184,7 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( return } - req, err := http.NewRequest("POST", env.VEDEM_OCR_URL, bodyBuf) + req, err := http.NewRequest("POST", env.VEDEM_IMAGE_URL, bodyBuf) if err != nil { err = errors.Wrap(code.OCRRequestError, fmt.Sprintf("construct request error: %v", err)) @@ -179,12 +192,16 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( } signToken := "UNSIGNED-PAYLOAD" - token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, []byte(signToken)) + token := builtin.Sign("auth-v2", env.VEDEM_IMAGE_AK, env.VEDEM_IMAGE_SK, []byte(signToken)) req.Header.Add("Agw-Auth", token) req.Header.Add("Agw-Auth-Content", signToken) req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + // ppe + // req.Header.Add("x-use-ppe", "1") + // req.Header.Add("x-tt-env", "ppe_vedem_algorithm") + var resp *http.Response // retry 3 times for i := 1; i <= 3; i++ { @@ -230,34 +247,37 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ( return } - var ocrResult ResponseOCR - err = json.Unmarshal(results, &ocrResult) + var imageResponse ImageResponse + err = json.Unmarshal(results, &imageResponse) if err != nil { + log.Error().Err(err). + Str("response", string(results)). + Msg("json unmarshal veDEM image response body failed") err = errors.Wrap(code.OCRResponseError, - fmt.Sprintf("json unmarshal response body error: %v", err)) + "json unmarshal veDEM image response body error") return } - if ocrResult.Code != 0 { + if imageResponse.Code != 0 { log.Error(). - Int("code", ocrResult.Code). - Str("message", ocrResult.Message). + Int("code", imageResponse.Code). + Str("message", imageResponse.Message). Msg("request veDEM OCR service failed") } - return ocrResult.OCRResult, ocrResult.URL, nil + return imageResponse.Result, nil } func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer) ( ocrTexts OCRTexts, url string, err error) { - ocrResults, url, err := s.getOCRResult(imageBuf) + imageResult, err := s.getImageResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return } - for _, ocrResult := range ocrResults { + for _, ocrResult := range imageResult.OCRResult { rect := image.Rectangle{ // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 Min: image.Point{ @@ -275,20 +295,21 @@ func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer) ( Rect: rect, }) } + url = imageResult.URL log.Debug().Interface("texts", ocrTexts).Msg("get screen texts by veDEM OCR") return } func checkEnv() error { - if env.VEDEM_OCR_URL == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_URL missed") + if env.VEDEM_IMAGE_URL == "" { + return errors.Wrap(code.OCREnvMissedError, "VEDEM_IMAGE_URL missed") } - log.Info().Str("VEDEM_OCR_URL", env.VEDEM_OCR_URL).Msg("get env") - if env.VEDEM_OCR_AK == "" { + log.Info().Str("VEDEM_IMAGE_URL", env.VEDEM_IMAGE_URL).Msg("get env") + if env.VEDEM_IMAGE_AK == "" { return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_AK missed") } - if env.VEDEM_OCR_SK == "" { + if env.VEDEM_IMAGE_SK == "" { return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_SK missed") } return nil diff --git a/hrp/pkg/uixt/ocr_vedem_test.go b/hrp/pkg/uixt/ocr_vedem_test.go index afa2262f..ba0a4e74 100644 --- a/hrp/pkg/uixt/ocr_vedem_test.go +++ b/hrp/pkg/uixt/ocr_vedem_test.go @@ -14,12 +14,11 @@ func checkOCR(buff *bytes.Buffer) error { if err != nil { return err } - ocrResults, url, err := service.getOCRResult(buff) + imageResult, err := service.getImageResult(buff) if err != nil { return err } - fmt.Println(ocrResults) - fmt.Println(url) + fmt.Println(fmt.Sprintf("imageResult: %v", imageResult)) return nil } From 9d78c527130857dfaacba5cfd9fcc0aa064bd847 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 11:44:24 +0800 Subject: [PATCH 58/67] bump version --- docs/CHANGELOG.md | 2 +- hrp/internal/version/VERSION | 2 +- hrp/pkg/uixt/ocr_vedem.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a7b54304..df8823ae 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## v4.3.4 (2023-05-11) +## v4.3.4 (2023-05-31) **go version** diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index e88d1a3c..686073dd 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305111932 \ No newline at end of file +v4.3.4-beta-202305311142 \ No newline at end of file diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index e915ef76..afc97a76 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -307,10 +307,10 @@ func checkEnv() error { } log.Info().Str("VEDEM_IMAGE_URL", env.VEDEM_IMAGE_URL).Msg("get env") if env.VEDEM_IMAGE_AK == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_AK missed") + return errors.Wrap(code.OCREnvMissedError, "VEDEM_IMAGE_AK missed") } if env.VEDEM_IMAGE_SK == "" { - return errors.Wrap(code.OCREnvMissedError, "VEDEM_OCR_SK missed") + return errors.Wrap(code.OCREnvMissedError, "VEDEM_IMAGE_SK missed") } return nil } From aafff8de9d964f4a8141a14835d7bfc9ba27e51f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 12:08:54 +0800 Subject: [PATCH 59/67] refactor: move IOCRService to IImageService --- hrp/pkg/uixt/ext.go | 4 +- hrp/pkg/uixt/ocr_vedem.go | 80 +++++++++++++++------------------- hrp/pkg/uixt/ocr_vedem_test.go | 4 +- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index b774ffb7..70de478c 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -86,7 +86,7 @@ type DriverExt struct { windowSize Size frame *bytes.Buffer doneMjpegStream chan bool - OCRService IOCRService // used to get texts from image + ImageService IImageService // used to extract image data interruptSignal chan os.Signal // cache step data @@ -110,7 +110,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, err } - if dExt.OCRService, err = newVEDEMOCRService(); err != nil { + if dExt.ImageService, err = newVEDEMImageService(); err != nil { return nil, err } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index afc97a76..e9b81816 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -28,6 +28,25 @@ type OCRResult struct { Points []PointF `json:"points"` } +func (o OCRResult) ToOCRText() OCRText { + rect := image.Rectangle{ + // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(o.Points[0].X), + Y: int(o.Points[0].Y), + }, + Max: image.Point{ + X: int(o.Points[2].X), + Y: int(o.Points[2].Y), + }, + } + + return OCRText{ + Text: o.Text, + Rect: rect, + } +} + type ImageResult struct { URL string `json:"url"` // image uploaded url OCRResult []OCRResult `json:"ocrResult"` // OCR texts @@ -137,15 +156,15 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) ( return results, nil } -func newVEDEMOCRService() (*veDEMOCRService, error) { +func newVEDEMImageService() (*veDEMImageService, error) { if err := checkEnv(); err != nil { return nil, err } - return &veDEMOCRService{}, nil + return &veDEMImageService{}, nil } -// veDEMOCRService implements IOCRService interface -type veDEMOCRService struct{} +// veDEMImageService implements IImageService interface +type veDEMImageService struct{} var actions = []string{ "ocr", // get ocr texts @@ -155,7 +174,7 @@ var actions = []string{ // "close", } -func (s *veDEMOCRService) getImageResult(imageBuf *bytes.Buffer) ( +func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( imageResult ImageResult, err error) { bodyBuf := &bytes.Buffer{} @@ -265,40 +284,9 @@ func (s *veDEMOCRService) getImageResult(imageBuf *bytes.Buffer) ( Msg("request veDEM OCR service failed") } - return imageResponse.Result, nil -} - -func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer) ( - ocrTexts OCRTexts, url string, err error) { - - imageResult, err := s.getImageResult(imageBuf) - if err != nil { - log.Error().Err(err).Msg("getOCRResult failed") - return - } - - for _, ocrResult := range imageResult.OCRResult { - rect := image.Rectangle{ - // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 - Min: image.Point{ - X: int(ocrResult.Points[0].X), - Y: int(ocrResult.Points[0].Y), - }, - Max: image.Point{ - X: int(ocrResult.Points[2].X), - Y: int(ocrResult.Points[2].Y), - }, - } - - ocrTexts = append(ocrTexts, OCRText{ - Text: ocrResult.Text, - Rect: rect, - }) - } - url = imageResult.URL - - log.Debug().Interface("texts", ocrTexts).Msg("get screen texts by veDEM OCR") - return + imageResult = imageResponse.Result + log.Debug().Interface("imageResult", imageResult).Msg("get image data by veDEM") + return imageResult, nil } func checkEnv() error { @@ -327,9 +315,9 @@ func getLogID(header http.Header) string { return logID[0] } -type IOCRService interface { - // GetTexts returns ocr texts and uploaded image url - GetTexts(imageBuf *bytes.Buffer) (texts OCRTexts, url string, err error) +type IImageService interface { + // GetImage returns image result including ocr texts, uploaded image url, etc + GetImage(imageBuf *bytes.Buffer) (imageResult ImageResult, err error) } // GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts. @@ -340,13 +328,17 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText return } - var imageUrl string - ocrTexts, imageUrl, err = dExt.OCRService.GetTexts(bufSource) + imageResult, err := dExt.ImageService.GetImage(bufSource) if err != nil { log.Error().Err(err).Msg("GetScreenTextsByOCR failed") return } + for _, ocrResult := range imageResult.OCRResult { + ocrTexts = append(ocrTexts, ocrResult.ToOCRText()) + } + + imageUrl := imageResult.URL if imageUrl != "" { dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot") diff --git a/hrp/pkg/uixt/ocr_vedem_test.go b/hrp/pkg/uixt/ocr_vedem_test.go index ba0a4e74..ec7b5a4e 100644 --- a/hrp/pkg/uixt/ocr_vedem_test.go +++ b/hrp/pkg/uixt/ocr_vedem_test.go @@ -10,11 +10,11 @@ import ( ) func checkOCR(buff *bytes.Buffer) error { - service, err := newVEDEMOCRService() + service, err := newVEDEMImageService() if err != nil { return err } - imageResult, err := service.getImageResult(buff) + imageResult, err := service.GetImage(buff) if err != nil { return err } From 55e6a42b0ff471464540636c392a8b3aaeed6e95 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 14:15:43 +0800 Subject: [PATCH 60/67] refactor: rename variables --- hrp/pkg/uixt/ext.go | 34 +++++++++++------------ hrp/pkg/uixt/ocr_vedem.go | 2 +- hrp/pkg/uixt/video_crawler.go | 52 +++++++++++++++++------------------ 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 70de478c..f414e414 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -56,7 +56,7 @@ type Popularity struct { LiveUsers string `json:"live_users,omitempty"` // 直播间人数 } -type OcrResult struct { +type ScreenResult struct { Texts OCRTexts `json:"texts"` // dumped OCRTexts Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] Popularity Popularity `json:"popularity"` // video popularity data @@ -64,19 +64,19 @@ type OcrResult struct { type cacheStepData struct { // cache step screenshot paths - ScreenShots []string + screenShots []string screenShotsUrls map[string]string // map screenshot file path to uploaded url - // cache step screenshot ocr results, key is image path, value is OcrResult - OcrResults map[string]*OcrResult + // cache step screenshot ocr results, key is image path, value is ScreenResult + screenResults map[string]*ScreenResult // cache feed/live video stat - VideoStat *VideoStat + videoStat *VideoStat } func (d *cacheStepData) reset() { - d.ScreenShots = make([]string, 0) + d.screenShots = make([]string, 0) d.screenShotsUrls = make(map[string]string) - d.OcrResults = make(map[string]*OcrResult) - d.VideoStat = nil + d.screenResults = make(map[string]*ScreenResult) + d.videoStat = nil } type DriverExt struct { @@ -199,27 +199,27 @@ func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (strin return "", errors.Wrap(err, "encode screenshot image failed") } - dExt.cacheStepData.ScreenShots = append(dExt.cacheStepData.ScreenShots, screenshotPath) + dExt.cacheStepData.screenShots = append(dExt.cacheStepData.screenShots, screenshotPath) log.Info().Str("path", screenshotPath).Msg("save screenshot file success") return screenshotPath, nil } func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { cacheData := make(map[string]interface{}) - cacheData["video_stat"] = dExt.cacheStepData.VideoStat - cacheData["screenshots"] = dExt.cacheStepData.ScreenShots + cacheData["video_stat"] = dExt.cacheStepData.videoStat + cacheData["screenshots"] = dExt.cacheStepData.screenShots cacheData["screenshots_urls"] = dExt.cacheStepData.screenShotsUrls - ocrResults := make(map[string]interface{}) - for imagePath, ocrResult := range dExt.cacheStepData.OcrResults { - o, _ := json.Marshal(ocrResult.Texts) + screenResults := make(map[string]interface{}) + for imagePath, screenResult := range dExt.cacheStepData.screenResults { + o, _ := json.Marshal(screenResult.Texts) data := map[string]interface{}{ - "tags": ocrResult.Tags, + "tags": screenResult.Tags, "texts": string(o), } - ocrResults[imagePath] = data + screenResults[imagePath] = data } - cacheData["ocr_results"] = ocrResults + cacheData["screen_results"] = screenResults // clear cache dExt.cacheStepData.reset() diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index e9b81816..eae5a924 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -344,7 +344,7 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot") } - dExt.cacheStepData.OcrResults[imagePath] = &OcrResult{ + dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{ Texts: ocrTexts, } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 40ee16e6..2e4294cc 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -82,15 +82,15 @@ func (s *VideoStat) isTargetAchieved() bool { } // incrFeed increases feed count and feed stat -func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { +func (s *VideoStat) incrFeed(screenResult *ScreenResult, driverExt *DriverExt) error { // feed author actionOptions := []ActionOption{ WithRegex(true), driverExt.GenAbsScope(0, 0.5, 1, 1).Option(), } - if ocrText, err := ocrResult.Texts.FindText("^@", actionOptions...); err == nil { + if ocrText, err := screenResult.Texts.FindText("^@", actionOptions...); err == nil { log.Debug().Str("author", ocrText.Text).Msg("found feed author") - ocrResult.Tags = append(ocrResult.Tags, ocrText.Text) + screenResult.Tags = append(screenResult.Tags, ocrText.Text) } for _, targetLabel := range s.configs.Feed.TargetLabels { @@ -99,22 +99,22 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { WithRegex(targetLabel.Regex), driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(), } - if _, err := ocrResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil { + if _, err := screenResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil { key := targetLabel.Text if _, ok := s.FeedStat[key]; !ok { s.FeedStat[key] = 0 } s.FeedStat[key]++ - ocrResult.Tags = append(ocrResult.Tags, key) + screenResult.Tags = append(screenResult.Tags, key) } } // add popularity data for feed - popularityData := ocrResult.Texts.FilterScope(driverExt.GenAbsScope(0.8, 0.5, 1, 0.8)) + popularityData := screenResult.Texts.FilterScope(driverExt.GenAbsScope(0.8, 0.5, 1, 0.8)) if len(popularityData) != 4 { log.Warn().Interface("popularity", popularityData).Msg("get feed popularity data failed") } else { - ocrResult.Popularity = Popularity{ + screenResult.Popularity = Popularity{ Stars: popularityData[0].Text, Comments: popularityData[1].Text, Favorites: popularityData[2].Text, @@ -122,29 +122,29 @@ func (s *VideoStat) incrFeed(ocrResult *OcrResult, driverExt *DriverExt) error { } } - log.Info().Strs("tags", ocrResult.Tags). - Interface("popularity", ocrResult.Popularity). + log.Info().Strs("tags", screenResult.Tags). + Interface("popularity", screenResult.Popularity). Msg("found feed success") s.FeedCount++ return nil } // incrLive increases live count and live stat -func (s *VideoStat) incrLive(ocrResult *OcrResult, driverExt *DriverExt) error { +func (s *VideoStat) incrLive(screenResult *ScreenResult, driverExt *DriverExt) error { // TODO: check live type // add popularity data for live - popularityData := ocrResult.Texts.FilterScope(driverExt.GenAbsScope(0.7, 0.05, 1, 0.15)) + popularityData := screenResult.Texts.FilterScope(driverExt.GenAbsScope(0.7, 0.05, 1, 0.15)) if len(popularityData) != 1 { log.Warn().Interface("popularity", popularityData).Msg("get live popularity data failed") } else { - ocrResult.Popularity = Popularity{ + screenResult.Popularity = Popularity{ LiveUsers: popularityData[0].Text, } } - log.Info().Strs("tags", ocrResult.Tags). - Interface("popularity", ocrResult.Popularity). + log.Info().Strs("tags", screenResult.Tags). + Interface("popularity", screenResult.Popularity). Msg("found live success") s.LiveCount++ return nil @@ -262,11 +262,11 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - ocrResult := l.driver.cacheStepData.OcrResults[imagePath] - ocrResult.Tags = []string{"live"} + screenResult := l.driver.cacheStepData.screenResults[imagePath] + screenResult.Tags = []string{"live"} // check live type and incr live count - if err := l.currentStat.incrLive(ocrResult, l.driver); err != nil { + if err := l.currentStat.incrLive(screenResult, l.driver); err != nil { log.Error().Err(err).Msg("incr live failed") } } @@ -325,7 +325,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { LiveStat: make(map[string]int), } defer func() { - dExt.cacheStepData.VideoStat = currVideoStat + dExt.cacheStepData.videoStat = currVideoStat }() // launch app @@ -370,10 +370,10 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - ocrResult := dExt.cacheStepData.OcrResults[imagePath] + screenResult := dExt.cacheStepData.screenResults[imagePath] // automatic handling of pop-up windows - if err := dExt.autoPopupHandler(ocrResult); err != nil { + if err := dExt.autoPopupHandler(screenResult); err != nil { log.Error().Err(err).Msg("auto handle popup failed") return err } @@ -390,12 +390,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { continue } } - ocrResult.Tags = []string{"live-preview"} + screenResult.Tags = []string{"live-preview"} } else { - ocrResult.Tags = []string{"feed"} + screenResult.Tags = []string{"feed"} // check feed type and incr feed count - if err := currVideoStat.incrFeed(ocrResult, dExt); err != nil { + if err := currVideoStat.incrFeed(screenResult, dExt); err != nil { log.Error().Err(err).Msg("incr feed failed") } } @@ -465,16 +465,16 @@ var popups = [][]string{ {"管理使用时间", ".*忽略.*"}, } -func (dExt *DriverExt) autoPopupHandler(ocrResult *OcrResult) error { +func (dExt *DriverExt) autoPopupHandler(screenResult *ScreenResult) error { for _, popup := range popups { if len(popup) != 2 { continue } - points, err := ocrResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true)) + points, err := screenResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true)) if err == nil { log.Warn().Interface("popup", popup). - Interface("texts", ocrResult.Texts).Msg("text popup found") + Interface("texts", screenResult.Texts).Msg("text popup found") point := points[1].Center() log.Info().Str("text", points[1].Text).Msg("close popup") if err := dExt.TapAbsXY(point.X, point.Y); err != nil { From 7d7044f332f5a1fb38b8e03e1b1c9ccd000631b6 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 16:10:24 +0800 Subject: [PATCH 61/67] refactor: GetScreenResult --- examples/worldcup/main.go | 9 ++-- hrp/pkg/uixt/demo/main_test.go | 4 +- hrp/pkg/uixt/ext.go | 6 +-- hrp/pkg/uixt/ocr_test.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 95 +++++++++++++++++++--------------- hrp/pkg/uixt/swipe.go | 5 +- hrp/pkg/uixt/tap.go | 2 +- hrp/pkg/uixt/video_crawler.go | 9 ++-- 8 files changed, 72 insertions(+), 60 deletions(-) diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index eea2f47e..dc47a33e 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { utcTimeStr := utcTime.Format("15:04:05") - _, ocrTexts, err := wc.driver.GetScreenTextsByOCR() + imageResult, err := wc.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("get ocr texts failed") return err @@ -159,7 +159,7 @@ func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { // filter ocr texts with time format secondsMap := map[string]int{} var secondsTexts []string - for _, ocrText := range ocrTexts { + for _, ocrText := range imageResult.OCRResult.ToOCRTexts() { seconds, err := convertTimeToSeconds(ocrText.Text) if err == nil { secondsTexts = append(secondsTexts, ocrText.Text) @@ -212,8 +212,9 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { time.Sleep(5 * time.Second) // 青少年弹窗处理 - if _, ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { - if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { + if imageResult, err := wc.driver.GetScreenResult(); err == nil { + if points, err := imageResult.OCRResult.ToOCRTexts(). + FindTexts([]string{"青少年模式", "我知道了"}); err == nil { point := points[1].Center() _ = wc.driver.TapAbsXY(point.X, point.Y) } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 8a042a0d..34f001d8 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -35,13 +35,13 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { // take screenshot and get screen texts by OCR - _, texts, err := driverExt.GetScreenTextsByOCR() + imageResult, err := driverExt.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") t.Fatal(err) } - points, err := texts.FindTexts([]string{"青少年模式", "我知道了"}) + points, err := imageResult.OCRResult.ToOCRTexts().FindTexts([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index f414e414..8afe2233 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -110,7 +110,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, err } - if dExt.ImageService, err = newVEDEMImageService(); err != nil { + if dExt.ImageService, err = newVEDEMImageService("ocr", "upload", "liveType"); err != nil { return nil, err } @@ -241,14 +241,14 @@ func init() { func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindScreenTextByOCR(search, options...) + return dExt.FindScreenText(search, options...) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, options...) } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, err := dExt.FindScreenTextByOCR(text) + _, err := dExt.FindScreenText(text) return err == nil } diff --git a/hrp/pkg/uixt/ocr_test.go b/hrp/pkg/uixt/ocr_test.go index 053ccaee..da8ed745 100644 --- a/hrp/pkg/uixt/ocr_test.go +++ b/hrp/pkg/uixt/ocr_test.go @@ -10,7 +10,7 @@ func TestDriverExtOCR(t *testing.T) { driverExt, err := iosDevice.NewDriver(nil) checkErr(t, err) - point, err := driverExt.FindScreenTextByOCR("抖音") + point, err := driverExt.FindScreenText("抖音") checkErr(t, err) t.Logf("point.X: %v, point.Y: %v", point.X, point.Y) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index eae5a924..17fa7747 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -28,29 +28,35 @@ type OCRResult struct { Points []PointF `json:"points"` } -func (o OCRResult) ToOCRText() OCRText { - rect := image.Rectangle{ - // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 - Min: image.Point{ - X: int(o.Points[0].X), - Y: int(o.Points[0].Y), - }, - Max: image.Point{ - X: int(o.Points[2].X), - Y: int(o.Points[2].Y), - }, - } +type OCRResults []OCRResult - return OCRText{ - Text: o.Text, - Rect: rect, +func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) { + for _, ocrResult := range o { + rect := image.Rectangle{ + // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(ocrResult.Points[0].X), + Y: int(ocrResult.Points[0].Y), + }, + Max: image.Point{ + X: int(ocrResult.Points[2].X), + Y: int(ocrResult.Points[2].Y), + }, + } + ocrText := OCRText{ + Text: ocrResult.Text, + Rect: rect, + } + ocrTexts = append(ocrTexts, ocrText) } + return } type ImageResult struct { - URL string `json:"url"` // image uploaded url - OCRResult []OCRResult `json:"ocrResult"` // OCR texts - LiveType string `json:"liveType"` // 直播间类型 + imagePath string + URL string `json:"url"` // image uploaded url + OCRResult OCRResults `json:"ocrResult"` // OCR texts + LiveType string `json:"liveType"` // 直播间类型 } type ImageResponse struct { @@ -156,22 +162,27 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) ( return results, nil } -func newVEDEMImageService() (*veDEMImageService, error) { +func newVEDEMImageService(actions ...string) (*veDEMImageService, error) { if err := checkEnv(); err != nil { return nil, err } - return &veDEMImageService{}, nil + if len(actions) == 0 { + actions = []string{"ocr"} + } + return &veDEMImageService{ + actions: actions, + }, nil } // veDEMImageService implements IImageService interface -type veDEMImageService struct{} - -var actions = []string{ - "ocr", // get ocr texts - "upload", // get image uploaded url - "liveType", // get live type - // "popup", - // "close", +// actions: +// ocr - get ocr texts +// upload - get image uploaded url +// liveType - get live type +// popup - get popup windows +// close - get close popup +type veDEMImageService struct { + actions []string } func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( @@ -179,7 +190,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) - for _, action := range actions { + for _, action := range s.actions { bodyWriter.WriteField("actions", action) } @@ -320,24 +331,21 @@ type IImageService interface { GetImage(imageBuf *bytes.Buffer) (imageResult ImageResult, err error) } -// GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts. -func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRTexts, err error) { +// GetScreenResult takes a screenshot, returns the image recognization result +func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) { var bufSource *bytes.Buffer + var imagePath string if bufSource, imagePath, err = dExt.TakeScreenShot( builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } - imageResult, err := dExt.ImageService.GetImage(bufSource) + imageResult, err = dExt.ImageService.GetImage(bufSource) if err != nil { - log.Error().Err(err).Msg("GetScreenTextsByOCR failed") + log.Error().Err(err).Msg("GetScreenResult failed") return } - for _, ocrResult := range imageResult.OCRResult { - ocrTexts = append(ocrTexts, ocrResult.ToOCRText()) - } - imageUrl := imageResult.URL if imageUrl != "" { dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl @@ -345,18 +353,19 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText } dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{ - Texts: ocrTexts, + Texts: imageResult.OCRResult.ToOCRTexts(), } - return imagePath, ocrTexts, nil + imageResult.imagePath = imagePath + return imageResult, nil } -func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) { - _, ocrTexts, err := dExt.GetScreenTextsByOCR() +func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) { + imageResult, err := dExt.GetScreenResult() if err != nil { return } - result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) + result, err := imageResult.OCRResult.ToOCRTexts().FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return @@ -364,7 +373,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) point = result.Center() log.Info().Str("text", text). - Interface("point", point).Msgf("FindScreenTextByOCR success") + Interface("point", point).Msgf("FindScreenText success") return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 06488ef2..b445babb 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -135,11 +135,12 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) var point PointF findTexts := func(d *DriverExt) error { var err error - _, ocrTexts, err := d.GetScreenTextsByOCR() + imageResult, err := d.GetScreenResult() if err != nil { return err } - points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...) + points, err := imageResult.OCRResult.ToOCRTexts(). + FindTexts(texts, dExt.ParseActionOptions(options...)...) if err != nil { return err } diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index b4707839..84b3541b 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -24,7 +24,7 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error { func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error { actionOptions := NewActionOptions(options...) - point, err := dExt.FindScreenTextByOCR(ocrText, options...) + point, err := dExt.FindScreenText(ocrText, options...) if err != nil { if actionOptions.IgnoreNotFoundError { return nil diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 2e4294cc..001734df 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -257,12 +257,12 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } // take screenshot and get screen texts by OCR - imagePath, _, err := l.driver.GetScreenTextsByOCR() + imageResult, err := l.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - screenResult := l.driver.cacheStepData.screenResults[imagePath] + screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath] screenResult.Tags = []string{"live"} // check live type and incr live count @@ -365,12 +365,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return errors.Wrap(code.InterruptError, "feed crawler interrupted") default: // take screenshot and get screen texts by OCR - imagePath, texts, err := dExt.GetScreenTextsByOCR() + imageResult, err := dExt.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - screenResult := dExt.cacheStepData.screenResults[imagePath] + screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath] // automatic handling of pop-up windows if err := dExt.autoPopupHandler(screenResult); err != nil { @@ -379,6 +379,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // check if live video && run live crawler + texts := imageResult.OCRResult.ToOCRTexts() if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") if !liveCrawler.currentStat.isLiveTargetAchieved() { From 78f63014cc4ca2a07a045660738e660cd973389f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 16:11:04 +0800 Subject: [PATCH 62/67] change: remove checking if app is in foreground when step failed --- hrp/step_mobile_ui.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 87a1792a..3838cfdd 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -589,14 +589,6 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err attachments := make(map[string]interface{}) if err != nil { attachments["error"] = err.Error() - - // check if app is in foreground - packageName := uiDriver.Driver.GetLastLaunchedApp() - err2 := uiDriver.Driver.AssertAppForeground(packageName) - if packageName != "" && err2 != nil { - log.Error().Err(err2).Msg("app is not in foreground") - err = errors.Wrap(code.MobileUIAppNotInForegroundError, err2.Error()) - } } // take screenshot after each step From 70039f327cfd81b4069ef4c84ab885dd2ec169cf Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 17:55:24 +0800 Subject: [PATCH 63/67] fix: ignore error when get foreground app failed --- hrp/internal/code/code.go | 6 ++++-- hrp/pkg/uixt/ext.go | 6 ++++-- hrp/pkg/uixt/video_crawler.go | 18 +++++++++++++----- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 9c2ebfa9..7c633649 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -74,7 +74,8 @@ var ( MobileUILaunchAppError = errors.New("mobile UI launch app error") // 71 MobileUIValidationError = errors.New("mobile UI validation error") // 75 MobileUIAppNotInForegroundError = errors.New("mobile UI app not in foreground error") // 76 - MobileUIPopupError = errors.New("mobile UI popup error") // 77 + MobileUIActivityNotMatchError = errors.New("mobile UI activity not match error") // 77 + MobileUIPopupError = errors.New("mobile UI popup error") // 78 ) // OCR related: [80, 90) @@ -136,7 +137,8 @@ var errorsMap = map[error]int{ MobileUILaunchAppError: 71, MobileUIValidationError: 75, MobileUIAppNotInForegroundError: 76, - MobileUIPopupError: 77, + MobileUIActivityNotMatchError: 77, + MobileUIPopupError: 78, // OCR related OCREnvMissedError: 80, diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 8afe2233..4b38d701 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -214,9 +214,11 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { for imagePath, screenResult := range dExt.cacheStepData.screenResults { o, _ := json.Marshal(screenResult.Texts) data := map[string]interface{}{ - "tags": screenResult.Tags, - "texts": string(o), + "tags": screenResult.Tags, + "texts": string(o), + "popularity": screenResult.Popularity, } + screenResults[imagePath] = data } cacheData["screen_results"] = screenResults diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 001734df..1da4803f 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -260,10 +260,14 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { imageResult, err := l.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") + time.Sleep(3 * time.Second) continue } screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath] screenResult.Tags = []string{"live"} + if imageResult.LiveType != "" { + screenResult.Tags = append(screenResult.Tags, imageResult.LiveType) + } // check live type and incr live count if err := l.currentStat.incrLive(screenResult, l.driver); err != nil { @@ -368,6 +372,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { imageResult, err := dExt.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") + time.Sleep(3 * time.Second) continue } screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath] @@ -433,26 +438,29 @@ func (dExt *DriverExt) assertActivity(packageName, activityType string) error { Str("activity_type", activityType).Msg("assert activity") app, err := dExt.Driver.GetForegroundApp() if err != nil { - log.Error().Err(err).Msg("get foreground app failed") - return err + log.Warn().Err(err).Msg("get foreground app failed, skip app/activity assertion") + return nil // Notice: ignore error when get foreground app failed } if app.PackageName != packageName { return errors.Wrap(code.MobileUIAppNotInForegroundError, - fmt.Sprintf("app %s is not in foreground", packageName)) + fmt.Sprintf("foreground app %s, expect %s", app.PackageName, packageName)) } + var expectActivity string if activities, ok := androidActivities[app.PackageName]; ok { if activity, ok := activities[activityType]; ok { if strings.HasSuffix(app.Activity, activity) { return nil } + expectActivity = activity } } log.Error().Interface("app", app.AppBaseInfo).Msg("app activity not match") - return errors.Wrap(code.MobileUIAppNotInForegroundError, - fmt.Sprintf("%s activity is not in foreground", activityType)) + return errors.Wrap(code.MobileUIActivityNotMatchError, + fmt.Sprintf("foreground activity %s, expect %s %s", + app.Activity, activityType, expectActivity)) } // TODO: add more popup texts From 946aa1a7df2daea3f063f6a9809b1c6bfa996fb9 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 18:05:05 +0800 Subject: [PATCH 64/67] bump version --- hrp/internal/version/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index 686073dd..cce148ba 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305311142 \ No newline at end of file +v4.3.4-beta-202305311805 \ No newline at end of file From 42256bc20bc0bb0f7d8cc7614f678f508d0e3bd1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 20:08:02 +0800 Subject: [PATCH 65/67] fix: handle offset --- hrp/pkg/uixt/drag.go | 3 +-- hrp/pkg/uixt/tap.go | 6 ++---- hrp/pkg/uixt/tap_test.go | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hrp/pkg/uixt/drag.go b/hrp/pkg/uixt/drag.go index 3f3dd1f6..f694a815 100644 --- a/hrp/pkg/uixt/drag.go +++ b/hrp/pkg/uixt/drag.go @@ -5,7 +5,7 @@ func (dExt *DriverExt) Drag(pathname string, toX, toY int, pressForDuration ...f } func (dExt *DriverExt) DragFloat(pathname string, toX, toY float64, pressForDuration ...float64) (err error) { - return dExt.DragOffsetFloat(pathname, toX, toY, 0.5, 0.5, pressForDuration...) + return dExt.DragOffsetFloat(pathname, toX, toY, 0, 0, pressForDuration...) } func (dExt *DriverExt) DragOffset(pathname string, toX, toY int, xOffset, yOffset float64, pressForDuration ...float64) (err error) { @@ -22,7 +22,6 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs return err } - // FIXME: handle offset return dExt.Driver.DragFloat(point.X+xOffset, point.Y+yOffset, toX, toY, WithPressDuration(pressForDuration[0])) } diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 84b3541b..5d0ea39b 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -50,7 +50,7 @@ func (dExt *DriverExt) TapByCV(imagePath string, options ...ActionOption) error } func (dExt *DriverExt) Tap(param string, options ...ActionOption) error { - return dExt.TapOffset(param, 0.5, 0.5, options...) + return dExt.TapOffset(param, 0, 0, options...) } func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...ActionOption) (err error) { @@ -64,7 +64,6 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options return err } - // FIXME: handle offset return dExt.TapAbsXY(point.X+xOffset, point.Y+yOffset, options...) } @@ -80,7 +79,7 @@ func (dExt *DriverExt) DoubleTapXY(x, y float64) error { } func (dExt *DriverExt) DoubleTap(param string) (err error) { - return dExt.DoubleTapOffset(param, 0.5, 0.5) + return dExt.DoubleTapOffset(param, 0, 0) } func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) { @@ -89,6 +88,5 @@ func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) ( return err } - // FIXME: handle offset return dExt.Driver.DoubleTapFloat(point.X+xOffset, point.Y+yOffset) } diff --git a/hrp/pkg/uixt/tap_test.go b/hrp/pkg/uixt/tap_test.go index f0cfd287..950b7049 100644 --- a/hrp/pkg/uixt/tap_test.go +++ b/hrp/pkg/uixt/tap_test.go @@ -43,6 +43,6 @@ func TestDriverExt_TapWithOCR(t *testing.T) { checkErr(t, err) // 需要点击文字上方的图标 - err = driverExt.TapOffset("抖音", 0.5, -1) + err = driverExt.TapOffset("抖音", 0, -20) checkErr(t, err) } From e1169ec628269c6c1556f0285ba9012c525a9778 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 20:17:43 +0800 Subject: [PATCH 66/67] feat: add GetScreenTexts --- examples/worldcup/main.go | 9 ++++----- hrp/pkg/uixt/demo/main_test.go | 4 ++-- hrp/pkg/uixt/ocr_vedem.go | 13 +++++++++++-- hrp/pkg/uixt/swipe.go | 5 ++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index dc47a33e..8b830b41 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { utcTimeStr := utcTime.Format("15:04:05") - imageResult, err := wc.driver.GetScreenResult() + ocrTexts, err := wc.driver.GetScreenTexts() if err != nil { log.Error().Err(err).Msg("get ocr texts failed") return err @@ -159,7 +159,7 @@ func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { // filter ocr texts with time format secondsMap := map[string]int{} var secondsTexts []string - for _, ocrText := range imageResult.OCRResult.ToOCRTexts() { + for _, ocrText := range ocrTexts { seconds, err := convertTimeToSeconds(ocrText.Text) if err == nil { secondsTexts = append(secondsTexts, ocrText.Text) @@ -212,9 +212,8 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { time.Sleep(5 * time.Second) // 青少年弹窗处理 - if imageResult, err := wc.driver.GetScreenResult(); err == nil { - if points, err := imageResult.OCRResult.ToOCRTexts(). - FindTexts([]string{"青少年模式", "我知道了"}); err == nil { + if ocrTexts, err := wc.driver.GetScreenTexts(); err == nil { + if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { point := points[1].Center() _ = wc.driver.TapAbsXY(point.X, point.Y) } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 34f001d8..a36b3ae4 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -35,13 +35,13 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { // take screenshot and get screen texts by OCR - imageResult, err := driverExt.GetScreenResult() + ocrTexts, err := driverExt.GetScreenTexts() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") t.Fatal(err) } - points, err := imageResult.OCRResult.ToOCRTexts().FindTexts([]string{"青少年模式", "我知道了"}) + points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 17fa7747..7ff0e982 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -360,12 +360,21 @@ func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) { return imageResult, nil } -func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) { +func (dExt *DriverExt) GetScreenTexts() (ocrTexts OCRTexts, err error) { imageResult, err := dExt.GetScreenResult() if err != nil { return } - result, err := imageResult.OCRResult.ToOCRTexts().FindText(text, dExt.ParseActionOptions(options...)...) + return imageResult.OCRResult.ToOCRTexts(), nil +} + +func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) { + ocrTexts, err := dExt.GetScreenTexts() + if err != nil { + return + } + + result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index b445babb..cf880ecf 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -135,12 +135,11 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) var point PointF findTexts := func(d *DriverExt) error { var err error - imageResult, err := d.GetScreenResult() + ocrTexts, err := d.GetScreenTexts() if err != nil { return err } - points, err := imageResult.OCRResult.ToOCRTexts(). - FindTexts(texts, dExt.ParseActionOptions(options...)...) + points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...) if err != nil { return err } From aa65962ffe4140e05d581633571f861af6da5671 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 20:24:00 +0800 Subject: [PATCH 67/67] bump version --- docs/CHANGELOG.md | 5 +++-- docs/cmd/hrp.md | 6 ++---- docs/cmd/hrp_boom.md | 3 +-- docs/cmd/hrp_build.md | 2 +- docs/cmd/hrp_convert.md | 17 +++++++++------- docs/cmd/hrp_dns.md | 2 +- docs/cmd/hrp_ping.md | 2 +- docs/cmd/hrp_pytest.md | 2 +- docs/cmd/hrp_run.md | 20 +++++++++---------- docs/cmd/hrp_startproject.md | 2 +- docs/cmd/hrp_wiki.md | 2 +- examples/demo-empty-project/proj.json | 4 ++-- examples/demo-with-go-plugin/proj.json | 4 ++-- examples/demo-with-py-plugin/proj.json | 4 ++-- examples/demo-without-plugin/proj.json | 4 ++-- .../templates/plugin/.debugtalk_gen.py | 2 +- .../templates/plugin/debugtalk_gen.go | 2 +- hrp/internal/version/VERSION | 2 +- hrp/runner.go | 2 +- 19 files changed, 44 insertions(+), 43 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index df8823ae..d240eac2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,10 +8,11 @@ - feat: cache screenshot ocr texts - feat: set testcase and request timeout in seconds - feat: catch interrupt signal -- feat: add new exit code MobileUILaunchAppError/InterruptError/TimeoutError/MobileUIPopupError +- feat: add new exit code MobileUILaunchAppError/InterruptError/TimeoutError/MobileUIActivityNotMatchError/MobileUIPopupError - feat: find text with regex - feat: add UI ocr tags to summary -- refactor: simplify OCR APIs +- feat: check android device offline when running shell failed +- refactor: replace OCR APIs with image APIs - refactor: FindText(s) returns OCRText(s) - refactor: merge ActionOption with DataOption - change: exit with AndroidShellExecError code for adb shell failure diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index e60bfc4c..157e6a35 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -31,14 +31,12 @@ Copyright 2017 debugtalk * [hrp boom](hrp_boom.md) - run load test with boomer * [hrp build](hrp_build.md) - build plugin for testing -* [hrp convert](hrp_convert.md) - convert to JSON/YAML/gotest/pytest testcases -* [hrp curl](hrp_curl.md) - run integrated curl command +* [hrp convert](hrp_convert.md) - convert multiple source format to HttpRunner JSON/YAML/gotest/pytest cases * [hrp dns](hrp_dns.md) - DNS resolution for different source and record types * [hrp ping](hrp_ping.md) - run integrated ping command * [hrp pytest](hrp_pytest.md) - run API test with pytest * [hrp run](hrp_run.md) - run API test with go engine * [hrp startproject](hrp_startproject.md) - create a scaffold project -* [hrp traceroute](hrp_traceroute.md) - run integrated traceroute command * [hrp wiki](hrp_wiki.md) - visit https://httprunner.com -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index c5c92782..997687e1 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -53,6 +53,5 @@ hrp boom [flags] ### SEE ALSO * [hrp](hrp.md) - Next-Generation API Testing Solution. -* [hrp boom curl](hrp_boom_curl.md) - run load test with curl command -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_build.md b/docs/cmd/hrp_build.md index 69f024ff..baaf0dc0 100644 --- a/docs/cmd/hrp_build.md +++ b/docs/cmd/hrp_build.md @@ -28,4 +28,4 @@ hrp build $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_convert.md b/docs/cmd/hrp_convert.md index 5f47069a..879dc938 100644 --- a/docs/cmd/hrp_convert.md +++ b/docs/cmd/hrp_convert.md @@ -1,6 +1,6 @@ ## hrp convert -convert to JSON/YAML/gotest/pytest testcases +convert multiple source format to HttpRunner JSON/YAML/gotest/pytest cases ``` hrp convert $path... [flags] @@ -9,18 +9,21 @@ hrp convert $path... [flags] ### Options ``` + --from-curl load from curl format + --from-har load from HAR format + --from-json load from json case format (default true) + --from-postman load from postman format + --from-yaml load from yaml case format -h, --help help for convert - -d, --output-dir string specify output directory, default to the same dir with har file + -d, --output-dir string specify output directory -p, --profile string specify profile path to override headers and cookies - --to-gotest convert to gotest scripts (TODO) - --to-json convert to JSON scripts (default) + --to-json convert to JSON case scripts (default true) --to-pytest convert to pytest scripts - --to-yaml convert to YAML scripts + --to-yaml convert to YAML case scripts ``` ### SEE ALSO * [hrp](hrp.md) - Next-Generation API Testing Solution. -* [hrp convert curl](hrp_convert_curl.md) - convert curl command to httprunner testcase -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_dns.md b/docs/cmd/hrp_dns.md index d2aaef60..353c3fda 100644 --- a/docs/cmd/hrp_dns.md +++ b/docs/cmd/hrp_dns.md @@ -26,4 +26,4 @@ hrp dns $url [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_ping.md b/docs/cmd/hrp_ping.md index 0475d93b..6ef40122 100644 --- a/docs/cmd/hrp_ping.md +++ b/docs/cmd/hrp_ping.md @@ -20,4 +20,4 @@ hrp ping $url [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index 3512a8ea..9dddefc7 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -16,4 +16,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index a0f70751..a80639f3 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -21,19 +21,19 @@ hrp run $path... [flags] ### Options ``` - -c, --continue-on-failure continue running next step when failure occurs - -g, --gen-html-report generate html report - -h, --help help for run - --http-stat turn on HTTP latency stat (DNSLookup, TCP Connection, etc.) - --log-plugin turn on plugin logging - --log-requests-off turn off request & response details logging - -p, --proxy-url string set proxy url - -s, --save-tests save tests summary + --case-timeout float32 set testcase timeout (seconds) (default 3600) + -c, --continue-on-failure continue running next step when failure occurs + -g, --gen-html-report generate html report + -h, --help help for run + --http-stat turn on HTTP latency stat (DNSLookup, TCP Connection, etc.) + --log-plugin turn on plugin logging + --log-requests-off turn off request & response details logging + -p, --proxy-url string set proxy url + -s, --save-tests save tests summary ``` ### SEE ALSO * [hrp](hrp.md) - Next-Generation API Testing Solution. -* [hrp run curl](hrp_run_curl.md) - run API test with curl command -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 888c7457..56d1562a 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -21,4 +21,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/docs/cmd/hrp_wiki.md b/docs/cmd/hrp_wiki.md index 97986ea5..32fcc471 100644 --- a/docs/cmd/hrp_wiki.md +++ b/docs/cmd/hrp_wiki.md @@ -16,4 +16,4 @@ hrp wiki [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 21-Oct-2022 +###### Auto generated by spf13/cobra on 31-May-2023 diff --git a/examples/demo-empty-project/proj.json b/examples/demo-empty-project/proj.json index edf45464..dd7a3b46 100644 --- a/examples/demo-empty-project/proj.json +++ b/examples/demo-empty-project/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-empty-project", - "create_time": "2022-10-21T21:54:56.252853+08:00", - "hrp_version": "v4.3.0" + "create_time": "2023-05-31T20:46:10.736189+08:00", + "hrp_version": "v4.3.4" } diff --git a/examples/demo-with-go-plugin/proj.json b/examples/demo-with-go-plugin/proj.json index 867531eb..8f68209a 100644 --- a/examples/demo-with-go-plugin/proj.json +++ b/examples/demo-with-go-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-go-plugin", - "create_time": "2022-10-21T21:52:38.979867+08:00", - "hrp_version": "v4.3.0" + "create_time": "2023-05-31T20:44:56.120736+08:00", + "hrp_version": "v4.3.4" } diff --git a/examples/demo-with-py-plugin/proj.json b/examples/demo-with-py-plugin/proj.json index c0aeb776..c90653d1 100644 --- a/examples/demo-with-py-plugin/proj.json +++ b/examples/demo-with-py-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-py-plugin", - "create_time": "2022-10-21T21:52:39.555851+08:00", - "hrp_version": "v4.3.0" + "create_time": "2023-05-31T20:45:00.44921+08:00", + "hrp_version": "v4.3.4" } diff --git a/examples/demo-without-plugin/proj.json b/examples/demo-without-plugin/proj.json index 593129a3..d7797860 100644 --- a/examples/demo-without-plugin/proj.json +++ b/examples/demo-without-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-without-plugin", - "create_time": "2022-10-21T21:54:56.136458+08:00", - "hrp_version": "v4.3.0" + "create_time": "2023-05-31T20:46:10.621009+08:00", + "hrp_version": "v4.3.4" } diff --git a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py index d5c51015..75cdd6ed 100644 --- a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py +++ b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py @@ -1,4 +1,4 @@ -# NOTE: Generated By hrp v4.3.0, DO NOT EDIT! +# NOTE: Generated By hrp v4.3.4, DO NOT EDIT! import sys import os diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index ca8bcde3..cec9b779 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.3.4-beta-202305032303, DO NOT EDIT! +// NOTE: Generated By hrp v4.3.4, DO NOT EDIT! package main import ( diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index cce148ba..bd4c8f53 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.3.4-beta-202305311805 \ No newline at end of file +v4.3.4 \ No newline at end of file diff --git a/hrp/runner.go b/hrp/runner.go index 818d695c..b30ca69e 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -452,7 +452,7 @@ func (r *CaseRunner) parseConfig() error { } client, err := device.NewDriver(nil) if err != nil { - return errors.Wrap(err, "init Android UIAutomator client failed") + return errors.Wrap(err, "init Android client failed") } r.hrpRunner.uiClients[device.SerialNumber] = client }