fix: convert relative scope to absolute scope

examples/uitest/demo_android_video_crawler.json

@@ -20,23 +20,23 @@
                         "params": {
                             "app_package_name": "com.ss.android.ugc.aweme",
                             "feed": {
-    "sleep_random": [
+                                "sleep_random": [
-    0,
+                                    0,
-    5,
+                                    5,
-    0.7,
+                                    0.7,
-    5,
+                                    5,
-    10,
+                                    10,
-    0.3
+                                    0.3
-],
+                                ],
-    "target_count": 5
+                                "target_count": 5
-},
+                            },
                             "live": {
-    "sleep_random": [
+                                "sleep_random": [
-    15,
+                                    15,
-    20
+                                    20
-],
+                                ],
-    "target_count": 3
+                                "target_count": 3
-}
+                            }
                         }
                     }
                 ]

hrp/pkg/uixt/action.go

@@ -3,7 +3,6 @@ package uixt
 import (
 	"encoding/json"
 	"fmt"
-	"math"
 	"math/rand"
 	"time"
 
@@ -82,10 +81,12 @@ type ActionOptions struct {
 	Frequency           int         `json:"frequency,omitempty" yaml:"frequency,omitempty"`
 
 	// scope related
-	Scope    []float64 `json:"scope,omitempty" yaml:"scope,omitempty"`         // used by ocr to get text position in the scope
+	// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner
-	AbsScope []int     `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // used by ocr to get text position in the scope
+	Scope    []float64 `json:"scope,omitempty" yaml:"scope,omitempty"`         // [x1, y1, x2, y2] in percentage of the screen
-	Offset   []int     `json:"offset,omitempty" yaml:"offset,omitempty"`       // used to tap offset of point
+	AbsScope []int     `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` // [x1, y1, x2, y2] in absolute pixels
-	Index    int       `json:"index,omitempty" yaml:"index,omitempty"`         // index of the target element, should start from 1
+
+	Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point
+	Index  int   `json:"index,omitempty" yaml:"index,omitempty"`   // index of the target element, should start from 1
 
 	// element related
 	Text        string `json:"text,omitempty" yaml:"text,omitempty"`
@@ -146,11 +147,9 @@ func (o *ActionOptions) Options() []ActionOption {
 	if o.Frequency != 0 {
 		options = append(options, WithFrequency(o.Frequency))
 	}
-	if len(o.Scope) != 4 {
+	if len(o.AbsScope) == 4 {
-		options = append(options, WithScope(0, 0, 1, 1))
+		options = append(options, WithAbsScope(
-	}
+			o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3]))
-	if len(o.AbsScope) != 4 {
-		o.AbsScope = []int{0, 0, math.MaxInt64, math.MaxInt64}
 	}
 	if len(o.Offset) == 2 {
 		options = append(options, WithOffset(o.Offset[0], o.Offset[1]))
@@ -167,9 +166,7 @@ func (o *ActionOptions) Options() []ActionOption {
 }
 
 func NewActionOptions(options ...ActionOption) *ActionOptions {
-	actionOptions := &ActionOptions{
+	actionOptions := &ActionOptions{}
-		Custom: make(map[string]interface{}),
-	}
 	for _, option := range options {
 		option(actionOptions)
 	}
@@ -179,11 +176,6 @@ func NewActionOptions(options ...ActionOption) *ActionOptions {
 func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} {
 	actionOptions := NewActionOptions(options...)
 
-	// custom options
-	for k, v := range actionOptions.Custom {
-		data[k] = v
-	}
-
 	if actionOptions.Identifier != "" {
 		data["log"] = map[string]interface{}{
 			"enable": true,
@@ -228,6 +220,13 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption)
 		data["isReplace"] = true // default true
 	}
 
+	// custom options
+	if actionOptions.Custom != nil {
+		for k, v := range actionOptions.Custom {
+			data[k] = v
+		}
+	}
+
 	return data
 }
 
@@ -235,6 +234,9 @@ type ActionOption func(o *ActionOptions)
 
 func WithCustomOption(key string, value interface{}) ActionOption {
 	return func(o *ActionOptions) {
+		if o.Custom == nil {
+			o.Custom = make(map[string]interface{})
+		}
 		o.Custom[key] = value
 	}
 }
@@ -284,12 +286,21 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
 }
 
 // WithScope inputs area of [(x1,y1), (x2,y2)]
+// x1, y1, x2, y2 are all in [0, 1], which means the relative position of the screen
 func WithScope(x1, y1, x2, y2 float64) ActionOption {
 	return func(o *ActionOptions) {
 		o.Scope = []float64{x1, y1, x2, y2}
 	}
 }
 
+// WithAbsScope inputs area of [(x1,y1), (x2,y2)]
+// x1, y1, x2, y2 are all absolute position of the screen
+func WithAbsScope(x1, y1, x2, y2 int) ActionOption {
+	return func(o *ActionOptions) {
+		o.AbsScope = []int{x1, y1, x2, y2}
+	}
+}
+
 func WithOffset(offsetX, offsetY int) ActionOption {
 	return func(o *ActionOptions) {
 		o.Offset = []int{offsetX, offsetY}
@@ -346,13 +357,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid %s params, should be app text(string), got %v",
 			ACTION_SwipeToTapText, action.Params)
 	case ACTION_SwipeToTapTexts:
-		if texts, ok := action.Params.([]interface{}); ok {
-			var textList []string
-			for _, t := range texts {
-				textList = append(textList, t.(string))
-			}
-			action.Params = textList
-		}
 		if texts, ok := action.Params.([]string); ok {
 			return dExt.swipeToTapTexts(texts, action.Options.Options()...)
 		}
@@ -391,9 +395,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			}
 			x, _ := location[0].(float64)
 			y, _ := location[1].(float64)
-			if len(action.Options.Offset) != 2 {
-				action.Options.Offset = []int{0, 0}
-			}
 			return dExt.TapAbsXY(x, y, action.Options.Options()...)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
@@ -404,13 +405,6 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
 	case ACTION_TapByOCR:
 		if ocrText, ok := action.Params.(string); ok {
-			if len(action.Options.Scope) != 4 {
-				action.Options.Scope = []float64{0, 0, 1, 1}
-			}
-			if len(action.Options.Offset) != 2 {
-				action.Options.Offset = []int{0, 0}
-			}
-
 			return dExt.TapByOCR(ocrText, action.Options.Options()...)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
@@ -460,11 +454,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		}
 		return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
 	case ACTION_SleepRandom:
-		params, ok := action.Params.([]interface{})
+		if params, ok := action.Params.([]interface{}); ok {
-		if !ok {
+			return sleepRandom(params)
-			return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
 		}
-		return sleepRandom(params)
+		return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
 	case ACTION_ScreenShot:
 		// take screenshot
 		log.Info().Msg("take screenshot for current screen")

hrp/pkg/uixt/ext.go

@@ -219,13 +219,20 @@ func (dExt *DriverExt) IsImageExist(text string) bool {
 	return err == nil
 }
 
-// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner
+func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption {
-// the value of (x, y) is between 0 and 1, which means the percentage of the screen
+	actionOptions := NewActionOptions(options...)
-func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) {
+
-	return int(x1 * float64(dExt.windowSize.Width)),
+	// convert relative scope to absolute scope
-		int(y1 * float64(dExt.windowSize.Height)),
+	if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 {
-		int(x2 * float64(dExt.windowSize.Width)),
+		scope := actionOptions.Scope
-		int(y2 * float64(dExt.windowSize.Height))
+		x1 := int(scope[0] * float64(dExt.windowSize.Width))
+		y1 := int(scope[1] * float64(dExt.windowSize.Height))
+		x2 := int(scope[2] * float64(dExt.windowSize.Width))
+		y2 := int(scope[3] * float64(dExt.windowSize.Height))
+		actionOptions.AbsScope = []int{x1, y1, x2, y2}
+	}
+
+	return actionOptions.Options()
 }
 
 func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool {

hrp/pkg/uixt/ocr_vedem.go

@@ -57,15 +57,19 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) (
 	for _, ocrText := range t {
 		rect := ocrText.Rect
 
-		// not contains text
+		// check if text in scope
-		if !strings.Contains(ocrText.Text, text) {
+		if len(actionOptions.AbsScope) == 4 {
-			continue
+			if rect.Min.X < actionOptions.AbsScope[0] ||
+				rect.Min.Y < actionOptions.AbsScope[1] ||
+				rect.Max.X > actionOptions.AbsScope[2] ||
+				rect.Max.Y > actionOptions.AbsScope[3] {
+				// not in scope
+				continue
+			}
 		}
 
-		// check if text in scope
+		// not contains text
-		if rect.Min.X < actionOptions.AbsScope[0] || rect.Min.Y < actionOptions.AbsScope[1] ||
+		if !strings.Contains(ocrText.Text, text) {
-			rect.Max.X > actionOptions.AbsScope[2] || rect.Max.Y > actionOptions.AbsScope[3] {
-			// not in scope
 			continue
 		}
 
@@ -301,7 +305,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption)
 	if err != nil {
 		return
 	}
-	point, err = ocrTexts.FindText(text, options...)
+	point, err = ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...)
 	if err != nil {
 		log.Warn().Msgf("FindText failed: %s", err.Error())
 		return

hrp/pkg/uixt/swipe.go

@@ -132,9 +132,6 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
 		return errors.New("no text to tap")
 	}
 
-	// default to retry 10 times
-	options = append(options, WithMaxRetryTimes(10))
-
 	var point PointF
 	findTexts := func(d *DriverExt) error {
 		var err error
@@ -142,7 +139,7 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
 		if err != nil {
 			return err
 		}
-		points, err := ocrTexts.FindTexts(texts, options...)
+		points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...)
 		if err != nil {
 			return err
 		}

hrp/step_mobile_ui.go

@@ -260,8 +260,9 @@ func (s *StepMobile) Input(text string, options ...uixt.ActionOption) *StepMobil
 // Sleep specify sleep seconds after last action
 func (s *StepMobile) Sleep(n float64) *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_Sleep,
+		Method:  uixt.ACTION_Sleep,
-		Params: n,
+		Params:  n,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }
@@ -272,40 +273,45 @@ func (s *StepMobile) Sleep(n float64) *StepMobile {
 // 2. [min1, max1, weight1, min2, max2, weight2, ...] : weight is the probability of the time range
 func (s *StepMobile) SleepRandom(params ...float64) *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_SleepRandom,
+		Method:  uixt.ACTION_SleepRandom,
-		Params: params,
+		Params:  params,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }
 
 func (s *StepMobile) VideoCrawler(params map[string]interface{}) *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_VideoCrawler,
+		Method:  uixt.ACTION_VideoCrawler,
-		Params: params,
+		Params:  params,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }
 
 func (s *StepMobile) ScreenShot() *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_ScreenShot,
+		Method:  uixt.ACTION_ScreenShot,
-		Params: nil,
+		Params:  nil,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }
 
 func (s *StepMobile) StartCamera() *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_StartCamera,
+		Method:  uixt.ACTION_StartCamera,
-		Params: nil,
+		Params:  nil,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }
 
 func (s *StepMobile) StopCamera() *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
-		Method: uixt.ACTION_StopCamera,
+		Method:  uixt.ACTION_StopCamera,
-		Params: nil,
+		Params:  nil,
+		Options: nil,
 	})
 	return &StepMobile{step: s.step}
 }