diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 94f060a2..026ba5cf 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -20,23 +20,23 @@ "params": { "app_package_name": "com.ss.android.ugc.aweme", "feed": { - "sleep_random": [ - 0, - 5, - 0.7, - 5, - 10, - 0.3 -], - "target_count": 5 -}, + "sleep_random": [ + 0, + 5, + 0.7, + 5, + 10, + 0.3 + ], + "target_count": 5 + }, "live": { - "sleep_random": [ - 15, - 20 -], - "target_count": 3 -} + "sleep_random": [ + 15, + 20 + ], + "target_count": 3 + } } } ] @@ -62,4 +62,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 9778b623..dfe27ec5 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -3,7 +3,6 @@ package uixt import ( "encoding/json" "fmt" - "math" "math/rand" "time" @@ -82,10 +81,12 @@ type ActionOptions struct { Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty"` // scope related - Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope - AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // used by ocr to get text position in the scope - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + // (x1, y1) is the top left corner, (x2, y2) is the bottom right corner + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // [x1, y1, x2, y2] in percentage of the screen + AbsScope []int `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // [x1, y1, x2, y2] in absolute pixels + + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 // element related Text string `json:"text,omitempty" yaml:"text,omitempty"` @@ -146,11 +147,9 @@ func (o *ActionOptions) Options() []ActionOption { if o.Frequency != 0 { options = append(options, WithFrequency(o.Frequency)) } - if len(o.Scope) != 4 { - options = append(options, WithScope(0, 0, 1, 1)) - } - if len(o.AbsScope) != 4 { - o.AbsScope = []int{0, 0, math.MaxInt64, math.MaxInt64} + if len(o.AbsScope) == 4 { + options = append(options, WithAbsScope( + o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3])) } if len(o.Offset) == 2 { options = append(options, WithOffset(o.Offset[0], o.Offset[1])) @@ -167,9 +166,7 @@ func (o *ActionOptions) Options() []ActionOption { } func NewActionOptions(options ...ActionOption) *ActionOptions { - actionOptions := &ActionOptions{ - Custom: make(map[string]interface{}), - } + actionOptions := &ActionOptions{} for _, option := range options { option(actionOptions) } @@ -179,11 +176,6 @@ func NewActionOptions(options ...ActionOption) *ActionOptions { func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} { actionOptions := NewActionOptions(options...) - // custom options - for k, v := range actionOptions.Custom { - data[k] = v - } - if actionOptions.Identifier != "" { data["log"] = map[string]interface{}{ "enable": true, @@ -228,6 +220,13 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) data["isReplace"] = true // default true } + // custom options + if actionOptions.Custom != nil { + for k, v := range actionOptions.Custom { + data[k] = v + } + } + return data } @@ -235,6 +234,9 @@ type ActionOption func(o *ActionOptions) func WithCustomOption(key string, value interface{}) ActionOption { return func(o *ActionOptions) { + if o.Custom == nil { + o.Custom = make(map[string]interface{}) + } o.Custom[key] = value } } @@ -284,12 +286,21 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { } // WithScope inputs area of [(x1,y1), (x2,y2)] +// x1, y1, x2, y2 are all in [0, 1], which means the relative position of the screen func WithScope(x1, y1, x2, y2 float64) ActionOption { return func(o *ActionOptions) { o.Scope = []float64{x1, y1, x2, y2} } } +// WithAbsScope inputs area of [(x1,y1), (x2,y2)] +// x1, y1, x2, y2 are all absolute position of the screen +func WithAbsScope(x1, y1, x2, y2 int) ActionOption { + return func(o *ActionOptions) { + o.AbsScope = []int{x1, y1, x2, y2} + } +} + func WithOffset(offsetX, offsetY int) ActionOption { return func(o *ActionOptions) { o.Offset = []int{offsetX, offsetY} @@ -346,13 +357,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params, should be app text(string), got %v", ACTION_SwipeToTapText, action.Params) case ACTION_SwipeToTapTexts: - if texts, ok := action.Params.([]interface{}); ok { - var textList []string - for _, t := range texts { - textList = append(textList, t.(string)) - } - action.Params = textList - } if texts, ok := action.Params.([]string); ok { return dExt.swipeToTapTexts(texts, action.Options.Options()...) } @@ -391,9 +395,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - if len(action.Options.Offset) != 2 { - action.Options.Offset = []int{0, 0} - } return dExt.TapAbsXY(x, y, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) @@ -404,13 +405,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - if len(action.Options.Scope) != 4 { - action.Options.Scope = []float64{0, 0, 1, 1} - } - if len(action.Options.Offset) != 2 { - action.Options.Offset = []int{0, 0} - } - return dExt.TapByOCR(ocrText, action.Options.Options()...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) @@ -460,11 +454,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params) case ACTION_SleepRandom: - params, ok := action.Params.([]interface{}) - if !ok { - return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) + if params, ok := action.Params.([]interface{}); ok { + return sleepRandom(params) } - return sleepRandom(params) + return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params) case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 40610319..fc82cd83 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -219,13 +219,20 @@ func (dExt *DriverExt) IsImageExist(text string) bool { return err == nil } -// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner -// the value of (x, y) is between 0 and 1, which means the percentage of the screen -func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { - return int(x1 * float64(dExt.windowSize.Width)), - int(y1 * float64(dExt.windowSize.Height)), - int(x2 * float64(dExt.windowSize.Width)), - int(y2 * float64(dExt.windowSize.Height)) +func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption { + actionOptions := NewActionOptions(options...) + + // convert relative scope to absolute scope + if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 { + scope := actionOptions.Scope + x1 := int(scope[0] * float64(dExt.windowSize.Width)) + y1 := int(scope[1] * float64(dExt.windowSize.Height)) + x2 := int(scope[2] * float64(dExt.windowSize.Width)) + y2 := int(scope[3] * float64(dExt.windowSize.Height)) + actionOptions.AbsScope = []int{x1, y1, x2, y2} + } + + return actionOptions.Options() } func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index b88cb85f..b2f0ccc0 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -57,15 +57,19 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) ( for _, ocrText := range t { rect := ocrText.Rect - // not contains text - if !strings.Contains(ocrText.Text, text) { - continue + // check if text in scope + if len(actionOptions.AbsScope) == 4 { + if rect.Min.X < actionOptions.AbsScope[0] || + rect.Min.Y < actionOptions.AbsScope[1] || + rect.Max.X > actionOptions.AbsScope[2] || + rect.Max.Y > actionOptions.AbsScope[3] { + // not in scope + continue + } } - // check if text in scope - if rect.Min.X < actionOptions.AbsScope[0] || rect.Min.Y < actionOptions.AbsScope[1] || - rect.Max.X > actionOptions.AbsScope[2] || rect.Max.Y > actionOptions.AbsScope[3] { - // not in scope + // not contains text + if !strings.Contains(ocrText.Text, text) { continue } @@ -301,7 +305,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) if err != nil { return } - point, err = ocrTexts.FindText(text, options...) + point, err = ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index a654a6c8..69d5cfe7 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -132,9 +132,6 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) return errors.New("no text to tap") } - // default to retry 10 times - options = append(options, WithMaxRetryTimes(10)) - var point PointF findTexts := func(d *DriverExt) error { var err error @@ -142,7 +139,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) if err != nil { return err } - points, err := ocrTexts.FindTexts(texts, options...) + points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...) if err != nil { return err } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index a36afe09..3e0944e2 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -260,8 +260,9 @@ func (s *StepMobile) Input(text string, options ...uixt.ActionOption) *StepMobil // Sleep specify sleep seconds after last action func (s *StepMobile) Sleep(n float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_Sleep, - Params: n, + Method: uixt.ACTION_Sleep, + Params: n, + Options: nil, }) return &StepMobile{step: s.step} } @@ -272,40 +273,45 @@ func (s *StepMobile) Sleep(n float64) *StepMobile { // 2. [min1, max1, weight1, min2, max2, weight2, ...] : weight is the probability of the time range func (s *StepMobile) SleepRandom(params ...float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_SleepRandom, - Params: params, + Method: uixt.ACTION_SleepRandom, + Params: params, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_VideoCrawler, - Params: params, + Method: uixt.ACTION_VideoCrawler, + Params: params, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) ScreenShot() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_ScreenShot, - Params: nil, + Method: uixt.ACTION_ScreenShot, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) StartCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_StartCamera, - Params: nil, + Method: uixt.ACTION_StartCamera, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} } func (s *StepMobile) StopCamera() *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{ - Method: uixt.ACTION_StopCamera, - Params: nil, + Method: uixt.ACTION_StopCamera, + Params: nil, + Options: nil, }) return &StepMobile{step: s.step} }