change: log ocr service request elapsed seconds

hrp/pkg/uixt/action.go

@@ -64,20 +64,19 @@ type MobileAction struct {
 	Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"`
 	Params interface{}  `json:"params,omitempty" yaml:"params,omitempty"`
 
-	Identifier          string      `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
+	Identifier          string    `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
-	MaxRetryTimes       int         `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
+	MaxRetryTimes       int       `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
-	WaitTime            float64     `json:"wait_time,omitempty" yaml:"wait_time,omitempty"`                       // wait time between swipe and ocr, unit: second
+	WaitTime            float64   `json:"wait_time,omitempty" yaml:"wait_time,omitempty"`                       // wait time between swipe and ocr, unit: second
-	Duration            float64     `json:"duration,omitempty" yaml:"duration,omitempty"`                         // used to set duration of ios swipe action
+	Duration            float64   `json:"duration,omitempty" yaml:"duration,omitempty"`                         // used to set duration of ios swipe action
-	Steps               int         `json:"steps,omitempty" yaml:"steps,omitempty"`                               // used to set steps of android swipe action
+	Steps               int       `json:"steps,omitempty" yaml:"steps,omitempty"`                               // used to set steps of android swipe action
-	Direction           interface{} `json:"direction,omitempty" yaml:"direction,omitempty"`                       // used by swipe to tap text or app
+	Scope               []float64 `json:"scope,omitempty" yaml:"scope,omitempty"`                               // used by ocr to get text position in the scope
-	Scope               []float64   `json:"scope,omitempty" yaml:"scope,omitempty"`                               // used by ocr to get text position in the scope
+	Offset              []int     `json:"offset,omitempty" yaml:"offset,omitempty"`                             // used to tap offset of point
-	Offset              []int       `json:"offset,omitempty" yaml:"offset,omitempty"`                             // used to tap offset of point
+	Index               int       `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
-	Index               int         `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
+	Timeout             int       `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
-	Timeout             int         `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
+	IgnoreNotFoundError bool      `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
-	IgnoreNotFoundError bool        `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
+	Text                string    `json:"text,omitempty" yaml:"text,omitempty"`
-	Text                string      `json:"text,omitempty" yaml:"text,omitempty"`
+	ID                  string    `json:"id,omitempty" yaml:"id,omitempty"`
-	ID                  string      `json:"id,omitempty" yaml:"id,omitempty"`
+	Description         string    `json:"description,omitempty" yaml:"description,omitempty"`
-	Description         string      `json:"description,omitempty" yaml:"description,omitempty"`
 }
 
 type ActionOption func(o *MobileAction)
@@ -115,14 +114,14 @@ func WithSteps(steps int) ActionOption {
 // WithDirection inputs direction (up, down, left, right)
 func WithDirection(direction string) ActionOption {
 	return func(o *MobileAction) {
-		o.Direction = direction
+		o.Params = direction
 	}
 }
 
 // WithCustomDirection inputs sx, sy, ex, ey
 func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
 	return func(o *MobileAction) {
-		o.Direction = []float64{sx, sy, ex, ey}
+		o.Params = []float64{sx, sy, ex, ey}
 	}
 }
 
@@ -195,51 +194,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid %s params, should be app name(string), got %v",
 			ACTION_SwipeToTapApp, action.Params)
 	case ACTION_SwipeToTapText:
-		// TODO: merge to LoopUntil
 		if text, ok := action.Params.(string); ok {
-			if len(action.Scope) != 4 {
+			return dExt.swipeToTapTexts([]string{text}, action)
-				action.Scope = []float64{0, 0, 1, 1}
-			}
-			if len(action.Offset) != 2 {
-				action.Offset = []int{0, 0}
-			}
-
-			identifierOption := WithDataIdentifier(action.Identifier)
-			offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
-			indexOption := WithDataIndex(action.Index)
-			scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
-
-			// default to retry 10 times
-			if action.MaxRetryTimes == 0 {
-				action.MaxRetryTimes = 10
-			}
-			maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
-			waitTimeOption := WithDataWaitTime(action.WaitTime)
-
-			var point PointF
-			// findTextAction := func(d *DriverExt) error {
-			// 	return nil
-			// }
-			findTextCondition := func(d *DriverExt) error {
-				var err error
-				point, err = d.FindScreenTextByOCR(text, indexOption, scopeOption)
-				return err
-			}
-			foundTextAction := func(d *DriverExt) error {
-				// tap text
-				return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
-			}
-
-			if action.Direction != nil {
-				return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
-			}
-			// swipe until found
-			return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
 		}
 		return fmt.Errorf("invalid %s params, should be app text(string), got %v",
 			ACTION_SwipeToTapText, action.Params)
 	case ACTION_SwipeToTapTexts:
-		// TODO: merge to LoopUntil
 		if texts, ok := action.Params.([]interface{}); ok {
 			var textList []string
 			for _, t := range texts {
@@ -248,56 +208,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			action.Params = textList
 		}
 		if texts, ok := action.Params.([]string); ok {
-			if len(action.Scope) != 4 {
+			return dExt.swipeToTapTexts(texts, action)
-				action.Scope = []float64{0, 0, 1, 1}
-			}
-			if len(action.Offset) != 2 {
-				action.Offset = []int{0, 0}
-			}
-
-			identifierOption := WithDataIdentifier(action.Identifier)
-			offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
-			scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
-			// default to retry 10 times
-			if action.MaxRetryTimes == 0 {
-				action.MaxRetryTimes = 10
-			}
-			maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
-			waitTimeOption := WithDataWaitTime(action.WaitTime)
-
-			var point PointF
-			findTexts := func(d *DriverExt) error {
-				var err error
-				ocrTexts, err := d.GetScreenTextsByOCR()
-				if err != nil {
-					return err
-				}
-				points, err := ocrTexts.FindTexts(texts, scopeOption)
-				if err != nil {
-					return err
-				}
-				for _, point = range points {
-					if point != (PointF{X: 0, Y: 0}) {
-						return nil
-					}
-				}
-				return errors.New("failed to find text position")
-			}
-			foundTextAction := func(d *DriverExt) error {
-				// tap text
-				return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
-			}
-
-			// default to retry 10 times
-			if action.MaxRetryTimes == 0 {
-				action.MaxRetryTimes = 10
-			}
-
-			if action.Direction != nil {
-				return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption)
-			}
-			// swipe until found
-			return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption)
 		}
 		return fmt.Errorf("invalid %s params, should be app text([]string), got %v",
 			ACTION_SwipeToTapText, action.Params)
@@ -384,27 +295,8 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
 	case ACTION_Swipe:
-		identifierOption := WithDataIdentifier(action.Identifier)
+		swipeAction := dExt.prepareSwipeAction(action)
-		durationOption := WithDataPressDuration(action.Duration)
+		return swipeAction(dExt)
-		if action.Steps == 0 {
-			action.Steps = 10
-		}
-		stepsOption := WithDataSteps(action.Steps)
-		if positions, ok := action.Params.([]interface{}); ok {
-			// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
-			if len(positions) != 4 {
-				return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions)
-			}
-			fromX, _ := positions[0].(float64)
-			fromY, _ := positions[1].(float64)
-			toX, _ := positions[2].(float64)
-			toY, _ := positions[3].(float64)
-			return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption)
-		}
-		if direction, ok := action.Params.(string); ok {
-			return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption)
-		}
-		return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
 	case ACTION_Input:
 		// input text on current active element
 		// append \n to send text with enter

hrp/pkg/uixt/ocr_vedem.go

@@ -167,7 +167,9 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err
 	var resp *http.Response
 	// retry 3 times
 	for i := 1; i <= 3; i++ {
+		start := time.Now()
 		resp, err = client.Do(req)
+		elapsed := time.Since(start)
 		var logID string
 		if resp != nil {
 			logID = getLogID(resp.Header)
@@ -175,7 +177,8 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err
 		if err == nil && resp.StatusCode == http.StatusOK {
 			log.Debug().
 				Str("X-TT-LOGID", logID).
-				Int("imageBufSize", size).
+				Int("image_bytes", size).
+				Float64("elapsed_seconds", elapsed.Seconds()).
 				Msg("request OCR service success")
 			break
 		}

hrp/pkg/uixt/swipe.go

@@ -66,42 +66,6 @@ func (dExt *DriverExt) SwipeRight(options ...DataOption) (err error) {
 
 type Action func(driver *DriverExt) error
 
-// findCondition indicates the condition to find a UI element
-// foundAction indicates the action to do after a UI element is found
-func (dExt *DriverExt) SwipeUntil(direction interface{}, findCondition Action, foundAction Action, options ...DataOption) error {
-	dataOptions := NewDataOptions(options...)
-	maxRetryTimes := dataOptions.MaxRetryTimes
-	interval := dataOptions.Interval
-
-	for i := 0; i < maxRetryTimes; i++ {
-		if err := findCondition(dExt); err == nil {
-			// do action after found
-			return foundAction(dExt)
-		}
-		if d, ok := direction.(string); ok {
-			if err := dExt.SwipeTo(d); err != nil {
-				log.Error().Err(err).Msgf("swipe %s failed", d)
-			}
-		} else if d, ok := direction.([]float64); ok {
-			if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3]); err != nil {
-				log.Error().Err(err).Msgf("swipe %v failed", d)
-			}
-		} else if d, ok := direction.([]interface{}); ok {
-			sx, _ := builtin.Interface2Float64(d[0])
-			sy, _ := builtin.Interface2Float64(d[1])
-			ex, _ := builtin.Interface2Float64(d[2])
-			ey, _ := builtin.Interface2Float64(d[3])
-			if err := dExt.SwipeRelative(sx, sy, ex, ey); err != nil {
-				log.Error().Err(err).Msgf("swipe (%v, %v) to (%v, %v) failed", sx, sy, ex, ey)
-			}
-		}
-		// wait for swipe action to completed and content to load completely
-		time.Sleep(time.Duration(1000*interval) * time.Millisecond)
-	}
-	return errors.Wrap(code.OCRTextNotFoundError,
-		fmt.Sprintf("swipe %v %d times, match condition failed", direction, maxRetryTimes))
-}
-
 func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...DataOption) error {
 	dataOptions := NewDataOptions(options...)
 	maxRetryTimes := dataOptions.MaxRetryTimes
@@ -125,40 +89,104 @@ func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action,
 		fmt.Sprintf("loop %d times, match find condition failed", maxRetryTimes))
 }
 
-func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error {
+func (dExt *DriverExt) prepareSwipeAction(action MobileAction) func(d *DriverExt) error {
+	identifierOption := WithDataIdentifier(action.Identifier)
+	durationOption := WithDataPressDuration(action.Duration)
+
+	if action.Steps == 0 {
+		action.Steps = 10
+	}
+	stepsOption := WithDataSteps(action.Steps)
+
+	dataOptions := make([]DataOption, 3)
+	dataOptions = append(dataOptions, identifierOption, durationOption, stepsOption)
+
+	return func(d *DriverExt) error {
+		defer func() {
+			// wait for swipe action to completed and content to load completely
+			time.Sleep(time.Duration(1000*action.WaitTime) * time.Millisecond)
+		}()
+
+		if d, ok := action.Params.(string); ok {
+			// enum direction: up, down, left, right
+			if err := dExt.SwipeTo(d, dataOptions...); err != nil {
+				log.Error().Err(err).Msgf("swipe %s failed", d)
+				return err
+			}
+		} else if d, ok := action.Params.([]float64); ok {
+			// custom direction: [fromX, fromY, toX, toY]
+			if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3], dataOptions...); err != nil {
+				log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed",
+					d[0], d[1], d[2], d[3])
+				return err
+			}
+		} else if d, ok := action.Params.([]interface{}); ok {
+			// loaded from json case
+			// custom direction: [fromX, fromY, toX, toY]
+			sx, _ := builtin.Interface2Float64(d[0])
+			sy, _ := builtin.Interface2Float64(d[1])
+			ex, _ := builtin.Interface2Float64(d[2])
+			ey, _ := builtin.Interface2Float64(d[3])
+			if err := dExt.SwipeRelative(sx, sy, ex, ey, dataOptions...); err != nil {
+				log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed",
+					sx, sy, ex, ey)
+				return err
+			}
+		} else {
+			return fmt.Errorf("invalid swipe params %v", action.Params)
+		}
+		return nil
+	}
+}
+
+func (dExt *DriverExt) swipeToTapTexts(texts []string, action MobileAction) error {
 	if len(action.Scope) != 4 {
 		action.Scope = []float64{0, 0, 1, 1}
 	}
 	if len(action.Offset) != 2 {
-		action.Offset = []int{0, -25}
+		action.Offset = []int{0, 0}
 	}
 
 	identifierOption := WithDataIdentifier(action.Identifier)
-	indexOption := WithDataIndex(action.Index)
 	offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
+	indexOption := WithDataIndex(action.Index)
 	scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
-
+	// default to retry 10 times
-	// default to retry 5 times
 	if action.MaxRetryTimes == 0 {
-		action.MaxRetryTimes = 5
+		action.MaxRetryTimes = 10
 	}
 	maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
 	waitTimeOption := WithDataWaitTime(action.WaitTime)
 
 	var point PointF
-	findAppAction := func(d *DriverExt) error {
+	findTexts := func(d *DriverExt) error {
-		return dExt.SwipeLeft()
-	}
-	findAppCondition := func(d *DriverExt) error {
 		var err error
-		point, err = d.FindScreenTextByOCR(appName, scopeOption, indexOption)
+		ocrTexts, err := d.GetScreenTextsByOCR()
-		return err
+		if err != nil {
+			return err
+		}
+		points, err := ocrTexts.FindTexts(texts, indexOption, scopeOption)
+		if err != nil {
+			return err
+		}
+		// FIXME: handle index
+		for _, point = range points {
+			if point != (PointF{X: 0, Y: 0}) {
+				return nil
+			}
+		}
+		return errors.New("failed to find text position")
 	}
-	foundAppAction := func(d *DriverExt) error {
+	foundTextAction := func(d *DriverExt) error {
-		// click app to launch
+		// tap text
 		return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
 	}
 
+	findAction := dExt.prepareSwipeAction(action)
+	return dExt.LoopUntil(findAction, findTexts, foundTextAction, maxRetryOption, waitTimeOption)
+}
+
+func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error {
 	// go to home screen
 	if err := dExt.Driver.Homescreen(); err != nil {
 		return errors.Wrap(err, "go to home screen failed")
@@ -169,6 +197,8 @@ func (dExt *DriverExt) swipeToTapApp(appName string, action MobileAction) error
 		dExt.SwipeRight()
 	}
 
-	// swipe next screen until app found
+	action.Offset = []int{0, -25}
-	return dExt.LoopUntil(findAppAction, findAppCondition, foundAppAction, maxRetryOption, waitTimeOption)
+	action.Params = "left"
+
+	return dExt.swipeToTapTexts([]string{appName}, action)
 }