feat: tap the first one matches text from given texts by ocr

examples/uitest/demo_douyin_follow_live.json

@@ -0,0 +1,152 @@
+{
+    "config": {
+        "name": "通过 关注天窗 进入指定主播抖音直播间",
+        "variables": {
+            "app_name": "抖音"
+        },
+        "ios": [
+            {
+                "port": 8100,
+                "mjpeg_port": 9100,
+                "log_on": true
+            }
+        ]
+    },
+    "teststeps": [
+        {
+            "name": "启动抖音",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "home"
+                    },
+                    {
+                        "method": "app_terminate",
+                        "params": "com.ss.iphone.ugc.Aweme"
+                    },
+                    {
+                        "method": "swipe_to_tap_app",
+                        "params": "$app_name",
+                        "identifier": "启动抖音",
+                        "max_retry_times": 5
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 5
+                    }
+                ]
+            },
+            "validate": [
+                {
+                    "check": "ui_ocr",
+                    "assert": "exists",
+                    "expect": "推荐",
+                    "msg": "抖音启动失败，「推荐」不存在"
+                }
+            ]
+        },
+        {
+            "name": "处理青少年弹窗",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "tap_ocr",
+                        "params": "我知道了",
+                        "ignore_NotFoundError": true
+                    }
+                ]
+            }
+        },
+        {
+            "name": "点击首页",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "tap_ocr",
+                        "params": "首页",
+                        "index": -1
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 10
+                    }
+                ]
+            }
+        },
+        {
+            "name": "点击关注页",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "tap_ocr",
+                        "params": "关注",
+                        "index": 1
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 10
+                    }
+                ]
+            }
+        },
+        {
+            "name": "向上滑动 2 次",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "swipe_to_tap_text",
+                        "params": [
+                            "理肤泉",
+                            "婉宝"
+                        ],
+                        "identifier": "click_live",
+                        "direction": [
+                            0.6,
+                            0.2,
+                            0.2,
+                            0.2
+                        ]
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 10
+                    },
+                    {
+                        "method": "swipe",
+                        "params": [
+                            0.9,
+                            0.7,
+                            0.9,
+                            0.3
+                        ],
+                        "identifier": "slide_in_live"
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 10
+                    },
+                    {
+                        "method": "screenshot"
+                    },
+                    {
+                        "method": "swipe",
+                        "params": [
+                            0.9,
+                            0.7,
+                            0.9,
+                            0.3
+                        ],
+                        "identifier": "slide_in_live"
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 10
+                    },
+                    {
+                        "method": "screenshot"
+                    }
+                ]
+            }
+        }
+    ]
+}

examples/uitest/demo_douyin_follow_live.yaml

@@ -0,0 +1,83 @@
+config:
+    name: 通过 关注天窗 进入指定主播抖音直播间
+    variables:
+        app_name: 抖音
+    ios:
+        - port: 8100
+          mjpeg_port: 9100
+          log_on: true
+teststeps:
+    - name: 启动抖音
+      ios:
+        actions:
+            - method: home
+            - method: app_terminate
+              params: com.ss.iphone.ugc.Aweme
+            - method: swipe_to_tap_app
+              params: $app_name
+              identifier: 启动抖音
+              max_retry_times: 5
+            - method: sleep
+              params: 5
+      validate:
+        - check: ui_ocr
+          assert: exists
+          expect: 推荐
+          msg: 抖音启动失败，「推荐」不存在
+    - name: 处理青少年弹窗
+      ios:
+        actions:
+            - method: tap_ocr
+              params: 我知道了
+              ignore_NotFoundError: true
+    - name: 点击首页
+      ios:
+        actions:
+            - method: tap_ocr
+              params: 首页
+              index: -1
+            - method: sleep
+              params: 10
+    - name: 点击关注页
+      ios:
+        actions:
+            - method: tap_ocr
+              params: 关注
+              index: 1
+            - method: sleep
+              params: 10
+    - name: 向上滑动 2 次
+      ios:
+        actions:
+            - method: swipe_to_tap_text
+              params:
+                - 理肤泉
+                - 婉宝
+              identifier: click_live
+              direction:
+                - 0.6
+                - 0.2
+                - 0.2
+                - 0.2
+            - method: sleep
+              params: 10
+            - method: swipe
+              params:
+                - 0.9
+                - 0.7
+                - 0.9
+                - 0.3
+              identifier: slide_in_live
+            - method: sleep
+              params: 10
+            - method: screenshot
+            - method: swipe
+              params:
+                - 0.9
+                - 0.7
+                - 0.9
+                - 0.3
+              identifier: slide_in_live
+            - method: sleep
+              params: 10
+            - method: screenshot

examples/uitest/demo_douyin_follow_live_test.go

@@ -0,0 +1,58 @@
+//go:build localtest
+
+package uitest
+
+import (
+	"testing"
+
+	"github.com/httprunner/httprunner/v4/hrp"
+)
+
+func TestIOSDouyinFollowLive(t *testing.T) {
+	testCase := &hrp.TestCase{
+		Config: hrp.NewConfig("通过 关注天窗 进入指定主播抖音直播间").
+			WithVariables(map[string]interface{}{
+				"app_name": "抖音",
+			}).
+			SetIOS(
+				hrp.WithLogOn(true),
+				hrp.WithWDAPort(8100),
+				hrp.WithWDAMjpegPort(9100),
+			),
+		TestSteps: []hrp.IStep{
+			hrp.NewStep("启动抖音").
+				IOS().
+				Home().
+				AppTerminate("com.ss.iphone.ugc.Aweme"). // 关闭已运行的抖音
+				SwipeToTapApp("$app_name", hrp.WithMaxRetryTimes(5), hrp.WithIdentifier("启动抖音")).Sleep(5).
+				Validate().
+				AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"),
+			hrp.NewStep("处理青少年弹窗").
+				IOS().
+				TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)),
+			hrp.NewStep("点击首页").
+				IOS().
+				TapByOCR("首页", hrp.WithIndex(-1)).Sleep(10),
+			hrp.NewStep("点击关注页").
+				IOS().
+				TapByOCR("关注", hrp.WithIndex(1)).Sleep(10),
+			hrp.NewStep("向上滑动 2 次").
+				IOS().SwipeToTapFromTexts([]string{"理肤泉", "婉宝"}, hrp.WithDirection([]float64{0.6, 0.2, 0.2, 0.2}), hrp.WithIdentifier("click_live")).Sleep(10).
+				Swipe(0.9, 0.7, 0.9, 0.3, hrp.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(). // 上划 1 次，等待 10s，截图保存
+				Swipe(0.9, 0.7, 0.9, 0.3, hrp.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(), // 再上划 1 次，等待 10s，截图保存
+		},
+	}
+
+	if err := testCase.Dump2JSON("demo_douyin_follow_live.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err := testCase.Dump2YAML("demo_douyin_follow_live.yaml"); err != nil {
+		t.Fatal(err)
+	}
+
+	runner := hrp.NewRunner(t).SetSaveTests(true)
+	err := runner.Run(testCase)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

hrp/pkg/uixt/android_driver.go

@@ -693,7 +693,7 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) {
 	}
 
 	var element WebElement
-	if valuetext, ok := data["text"]; ok {
+	if valuetext, ok := data["textview"]; ok {
 		element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()})
 	} else if valueid, ok := data["id"]; ok {
 		element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)})

hrp/pkg/uixt/ext.go

@@ -63,14 +63,15 @@ type MobileAction struct {
 	Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"`
 	Params interface{}  `json:"params,omitempty" yaml:"params,omitempty"`
 
-	Identifier          string `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
+	Identifier          string      `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
-	MaxRetryTimes       int    `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
+	MaxRetryTimes       int         `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
-	Index               int    `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
+	Direction           interface{} `json:"direction,omitempty" yaml:"direction,omitempty"`                       // used by swipe to tap text or app
-	Timeout             int    `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
+	Index               int         `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
-	IgnoreNotFoundError bool   `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
+	Timeout             int         `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
-	Text                string `json:"text,omitempty" yaml:"text,omitempty"`
+	IgnoreNotFoundError bool        `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
-	ID                  string `json:"id,omitempty" yaml:"id,omitempty"`
+	Text                string      `json:"text,omitempty" yaml:"text,omitempty"`
-	Description         string `json:"description,omitempty" yaml:"description,omitempty"`
+	ID                  string      `json:"id,omitempty" yaml:"id,omitempty"`
+	Description         string      `json:"description,omitempty" yaml:"description,omitempty"`
 }
 
 type ActionOption func(o *MobileAction)
@@ -87,6 +88,13 @@ func WithIndex(index int) ActionOption {
 	}
 }
 
+// WithDirection inputs direction (up, down, left, right, []float64{sx, sy, ex, ey})
+func WithDirection(direction interface{}) ActionOption {
+	return func(o *MobileAction) {
+		o.Direction = direction
+	}
+}
+
 func WithText(text string) ActionOption {
 	return func(o *MobileAction) {
 		o.Text = text
@@ -363,7 +371,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			}
 			foundAppAction := func(d *DriverExt) error {
 				// click app to launch
-				return d.TapAbsXY(point.X, point.Y-20, action.Identifier)
+				return d.TapAbsXY(point.X, point.Y-25, action.Identifier)
 			}
 
 			// go to home screen
@@ -386,27 +394,52 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid %s params, should be app name(string), got %v",
 			ACTION_SwipeToTapApp, action.Params)
 	case ACTION_SwipeToTapText:
+		var point PointF
+		var findText func(d *DriverExt) error
+
 		if text, ok := action.Params.(string); ok {
-			var point PointF
+			findText = func(d *DriverExt) error {
-			findText := func(d *DriverExt) error {
 				var err error
 				point, err = d.GetTextXY(text, action.Index)
 				return err
 			}
-			foundTextAction := func(d *DriverExt) error {
+		} else if texts, ok := action.Params.([]interface{}); ok {
-				// tap text
+			findText = func(d *DriverExt) error {
-				return d.TapAbsXY(point.X, point.Y, action.Identifier)
+				var err error
+				var ts []string
+				for _, t := range texts {
+					ts = append(ts, t.(string))
+				}
+				points, err := d.GetTextXYs(ts)
+				if err != nil {
+					return err
+				}
+				for _, point = range points {
+					if point != (PointF{}) {
+						return nil
+					}
+				}
+				return errors.New("failed to find text position")
 			}
-
+		} else {
-			// default to retry 10 times
+			return fmt.Errorf("invalid %s params, should be app text(string or []string), got %v",
-			if action.MaxRetryTimes == 0 {
+				ACTION_SwipeToTapText, action.Params)
-				action.MaxRetryTimes = 10
-			}
-			// swipe until live room found
-			return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
 		}
-		return fmt.Errorf("invalid %s params, should be app text(string), got %v",
+
-			ACTION_SwipeToTapText, action.Params)
+		foundTextAction := func(d *DriverExt) error {
+			// tap text
+			return d.TapAbsXY(point.X, point.Y, action.Identifier)
+		}
+
+		// default to retry 10 times
+		if action.MaxRetryTimes == 0 {
+			action.MaxRetryTimes = 10
+		}
+		if action.Direction != nil {
+			return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes)
+		}
+		// swipe until live room found
+		return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
 	case AppTerminate:
 		if bundleId, ok := action.Params.(string); ok {
 			success, err := dExt.Driver.AppTerminate(bundleId)
@@ -497,7 +530,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		param := fmt.Sprintf("%v", action.Params)
 		options := []DataOption{}
 		if action.Text != "" {
-			options = append(options, WithCustomOption("text", action.Text))
+			options = append(options, WithCustomOption("textview", action.Text))
 		}
 		if action.ID != "" {
 			options = append(options, WithCustomOption("id", action.ID))

hrp/pkg/uixt/ocr_off.go

@@ -2,9 +2,16 @@
 
 package uixt
 
-import "github.com/rs/zerolog/log"
+import (
+	"github.com/rs/zerolog/log"
+)
 
 func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
 	log.Fatal().Msg("OCR is not supported")
 	return
 }
+
+func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
+	log.Fatal().Msg("OCR is not supported")
+	return
+}

hrp/pkg/uixt/ocr_on.go

@@ -159,6 +159,48 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
 	return rects[idx], nil
 }
 
+func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[string]image.Rectangle, err error) {
+	ocrResults, err := s.getOCRResult(imageBuf)
+	if err != nil {
+		log.Error().Err(err).Msg("getOCRResult failed")
+		return
+	}
+
+	var ocrTexts []string
+	rects = map[string]image.Rectangle{}
+
+	for _, text := range texts {
+		for _, ocrResult := range ocrResults {
+			ocrTexts = append(ocrTexts, ocrResult.Text)
+
+			// not contains text
+			if !strings.Contains(ocrResult.Text, text) {
+				continue
+			}
+
+			rect := image.Rectangle{
+				// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
+				Min: image.Point{
+					X: int(ocrResult.Points[0].X),
+					Y: int(ocrResult.Points[0].Y),
+				},
+				Max: image.Point{
+					X: int(ocrResult.Points[2].X),
+					Y: int(ocrResult.Points[2].Y),
+				},
+			}
+			rects[text] = rect
+			break
+		}
+
+		if _, ok := rects[text]; !ok {
+			rects[text] = image.Rectangle{}
+		}
+	}
+
+	return rects, nil
+}
+
 type OCRService interface {
 	FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
 }
@@ -182,3 +224,32 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width,
 	x, y, width, height = dExt.MappingToRectInUIKit(rect)
 	return
 }
+
+func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
+	var bufSource *bytes.Buffer
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		err = fmt.Errorf("takeScreenShot error: %v", err)
+		return
+	}
+
+	service := &veDEMOCRService{}
+	rects, err := service.FindTexts(ocrTexts, bufSource.Bytes())
+	if err != nil {
+		log.Warn().Msgf("FindTexts failed: %s", err.Error())
+		err = fmt.Errorf("FindTexts failed: %v", err)
+		return
+	}
+
+	ps = map[string][]float64{}
+	log.Info().Interface("ocrTexts", ocrTexts).Msgf("FindTexts success")
+	for text, rect := range rects {
+		if rect == (image.Rectangle{}) {
+			ps[text] = []float64{}
+			continue
+		}
+		x, y, width, height := dExt.MappingToRectInUIKit(rect)
+		ps[text] = []float64{x, y, width, height}
+	}
+
+	return
+}

hrp/pkg/uixt/swipe.go

@@ -72,14 +72,20 @@ type FindCondition func(driver *DriverExt) error
 // FoundAction indicates the action to do after a UI element is found
 type FoundAction func(driver *DriverExt) error
 
-func (dExt *DriverExt) SwipeUntil(direction string, condition FindCondition, action FoundAction, maxTimes int) error {
+func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int) error {
 	for i := 0; i < maxTimes; i++ {
 		if err := condition(dExt); err == nil {
 			// do action after found
 			return action(dExt)
 		}
-		if err := dExt.SwipeTo(direction); err != nil {
+		if d, ok := direction.(string); ok {
-			log.Error().Err(err).Msgf("swipe %s failed", direction)
+			if err := dExt.SwipeTo(d); err != nil {
+				log.Error().Err(err).Msgf("swipe %s failed", d)
+			}
+		} else if d, ok := direction.([]float64); ok {
+			if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3]); err != nil {
+				log.Error().Err(err).Msgf("swipe %s failed", d)
+			}
 		}
 	}
 	return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes)

hrp/pkg/uixt/tap.go

@@ -41,6 +41,27 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er
 	return point, nil
 }
 
+func (dExt *DriverExt) GetTextXYs(ocrText []string) (points map[string]PointF, err error) {
+	ps, err := dExt.FindTextsByOCR(ocrText)
+	if err != nil {
+		return map[string]PointF{}, err
+	}
+
+	points = map[string]PointF{}
+	for text, point := range ps {
+		if len(point) == 0 {
+			points[text] = PointF{}
+			continue
+		}
+		points[text] = PointF{
+			X: point[0] + point[2]*0.5,
+			Y: point[1] + point[3]*0.5,
+		}
+	}
+
+	return points, nil
+}
+
 func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) {
 	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...)
 	if err != nil {

hrp/step.go

@@ -29,6 +29,7 @@ var (
 	WithText                = uixt.WithText
 	WithID                  = uixt.WithID
 	WithDescription         = uixt.WithDescription
+	WithDirection           = uixt.WithDirection
 )
 
 var (

hrp/step_android_ui.go

@@ -324,6 +324,18 @@ func (s *StepAndroid) SwipeToTapText(text string, options ...uixt.ActionOption)
 	return &StepAndroid{step: s.step}
 }
 
+func (s *StepAndroid) SwipeToTapFromTexts(texts []string, options ...uixt.ActionOption) *StepAndroid {
+	action := uixt.MobileAction{
+		Method: uixt.ACTION_SwipeToTapText,
+		Params: texts,
+	}
+	for _, option := range options {
+		option(&action)
+	}
+	s.step.Android.Actions = append(s.step.Android.Actions, action)
+	return &StepAndroid{step: s.step}
+}
+
 // Validate switches to step validation.
 func (s *StepAndroid) Validate() *StepAndroidValidation {
 	return &StepAndroidValidation{

hrp/step_ios_ui.go

@@ -244,6 +244,18 @@ func (s *StepIOS) SwipeToTapText(text string, options ...uixt.ActionOption) *Ste
 	return &StepIOS{step: s.step}
 }
 
+func (s *StepIOS) SwipeToTapFromTexts(texts []string, options ...uixt.ActionOption) *StepIOS {
+	action := uixt.MobileAction{
+		Method: uixt.ACTION_SwipeToTapText,
+		Params: texts,
+	}
+	for _, option := range options {
+		option(&action)
+	}
+	s.step.IOS.Actions = append(s.step.IOS.Actions, action)
+	return &StepIOS{step: s.step}
+}
+
 func (s *StepIOS) Input(text string, options ...uixt.ActionOption) *StepIOS {
 	action := uixt.MobileAction{
 		Method: uixt.ACTION_Input,