feat: locate ocr text with index

examples/uitest/demo_douyin_live.json

@@ -0,0 +1,89 @@
+{
+    "config": {
+        "name": "通过 feed 卡片进入微信直播间",
+        "ios": [
+            {
+                "port": 8700,
+                "mjpeg_port": 8800,
+                "log_on": true
+            }
+        ]
+    },
+    "teststeps": [
+        {
+            "name": "启动抖音",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "home"
+                    },
+                    {
+                        "method": "app_terminate",
+                        "params": "com.ss.iphone.ugc.Aweme"
+                    },
+                    {
+                        "method": "swipe_to_tap_app",
+                        "params": "抖音",
+                        "max_retry_times": 5
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 5
+                    }
+                ]
+            },
+            "validate": [
+                {
+                    "check": "ui_ocr",
+                    "assert": "exists",
+                    "expect": "推荐",
+                    "msg": "抖音启动失败，「推荐」不存在"
+                }
+            ]
+        },
+        {
+            "name": "在推荐页上划，直到出现「点击进入直播间」",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "swipe_to_tap_text",
+                        "params": "点击进入直播间",
+                        "identifier": "进入直播间",
+                        "max_retry_times": 100
+                    }
+                ]
+            }
+        },
+        {
+            "name": "向上滑动，等待 10s",
+            "ios": {
+                "actions": [
+                    {
+                        "method": "swipe",
+                        "params": "up",
+                        "identifier": "第一次上划"
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 2
+                    },
+                    {
+                        "method": "screenshot"
+                    },
+                    {
+                        "method": "swipe",
+                        "params": "up",
+                        "identifier": "第二次上划"
+                    },
+                    {
+                        "method": "sleep",
+                        "params": 2
+                    },
+                    {
+                        "method": "screenshot"
+                    }
+                ]
+            }
+        }
+    ]
+}

examples/uitest/demo_douyin_live.yaml

@@ -0,0 +1,45 @@
+config:
+    name: 通过 feed 卡片进入微信直播间
+    ios:
+        - port: 8700
+          mjpeg_port: 8800
+          log_on: true
+teststeps:
+    - name: 启动抖音
+      ios:
+        actions:
+            - method: home
+            - method: app_terminate
+              params: com.ss.iphone.ugc.Aweme
+            - method: swipe_to_tap_app
+              params: 抖音
+              max_retry_times: 5
+            - method: sleep
+              params: 5
+      validate:
+        - check: ui_ocr
+          assert: exists
+          expect: 推荐
+          msg: 抖音启动失败，「推荐」不存在
+    - name: 在推荐页上划，直到出现「点击进入直播间」
+      ios:
+        actions:
+            - method: swipe_to_tap_text
+              params: 点击进入直播间
+              identifier: 进入直播间
+              max_retry_times: 100
+    - name: 向上滑动，等待 10s
+      ios:
+        actions:
+            - method: swipe
+              params: up
+              identifier: 第一次上划
+            - method: sleep
+              params: 2
+            - method: screenshot
+            - method: swipe
+              params: up
+              identifier: 第二次上划
+            - method: sleep
+              params: 2
+            - method: screenshot

examples/uitest/demo_douyin_test.go

@@ -0,0 +1,52 @@
+package uitest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/httprunner/httprunner/v4/hrp"
+)
+
+func TestIOSDouyinLive(t *testing.T) {
+	testCase := &hrp.TestCase{
+		Config: hrp.NewConfig("通过 feed 卡片进入微信直播间").
+			SetIOS(hrp.WithLogOn(true), hrp.WithPort(8700), hrp.WithMjpegPort(8800)),
+		TestSteps: []hrp.IStep{
+			hrp.NewStep("启动抖音").
+				IOS().
+				Home().
+				AppTerminate("com.ss.iphone.ugc.Aweme"). // 关闭已运行的抖音
+				SwipeToTapApp("抖音", hrp.WithMaxRetryTimes(5)).Sleep(5).
+				Validate().
+				AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"),
+			// hrp.NewStep("处理青少年弹窗").
+			// 	IOS().
+			// 	TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)),
+			hrp.NewStep("在推荐页上划，直到出现「点击进入直播间」").
+				IOS().
+				SwipeToTapText("点击进入直播间", hrp.WithMaxRetryTimes(100), hrp.WithIdentifier("进入直播间")),
+			hrp.NewStep("向上滑动，等待 10s").
+				IOS().
+				SwipeUp(hrp.WithIdentifier("第一次上划")).Sleep(2).ScreenShot(). // 上划 1 次，等待 2s，截图保存
+				SwipeUp(hrp.WithIdentifier("第二次上划")).Sleep(2).ScreenShot(), // 再上划 1 次，等待 2s，截图保存
+		},
+	}
+
+	if err := testCase.Dump2JSON("demo_douyin_live.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err := testCase.Dump2YAML("demo_douyin_live.yaml"); err != nil {
+		t.Fatal(err)
+	}
+
+	runner := hrp.NewRunner(t)
+	sessionRunner, err := runner.NewSessionRunner(testCase)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sessionRunner.Start(nil); err != nil {
+		t.Fatal(err)
+	}
+	summary := sessionRunner.GetSummary()
+	fmt.Println(summary)
+}

examples/uitest/demo_weixin_live.json

@@ -48,7 +48,8 @@
                     {
                         "method": "tap_ocr",
                         "params": "视频号",
-                        "identifier": "进入视频号"
+                        "identifier": "进入视频号",
+                        "index": -1
                     }
                 ]
             }

examples/uitest/demo_weixin_live.yaml

@@ -27,6 +27,7 @@ teststeps:
             - method: tap_ocr
               params: 视频号
               identifier: 进入视频号
+              index: -1
     - name: 处理青少年弹窗
       ios:
         actions:

examples/uitest/demo_weixin_test.go

@@ -21,8 +21,8 @@ func TestIOSWeixinLive(t *testing.T) {
 				AssertLabelExists("通讯录", "微信启动失败，「通讯录」不存在"),
 			hrp.NewStep("进入直播页").
 				IOS().
-				Tap("发现").                                    // 进入「发现页」
+				Tap("发现").                                                       // 进入「发现页」
-				TapByOCR("视频号", hrp.WithIdentifier("进入视频号")), // 通过 OCR 识别「视频号」
+				TapByOCR("视频号", hrp.WithIdentifier("进入视频号"), hrp.WithIndex(-1)), // 通过 OCR 识别「视频号」
 			hrp.NewStep("处理青少年弹窗").
 				IOS().
 				TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)),

hrp/internal/uixt/ext.go

@@ -63,6 +63,7 @@ type MobileAction struct {
 
 	Identifier          string `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
 	MaxRetryTimes       int    `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
+	Index               int    `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
 	Timeout             int    `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
 	IgnoreNotFoundError bool   `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
 }
@@ -75,6 +76,12 @@ func WithIdentifier(identifier string) ActionOption {
 	}
 }
 
+func WithIndex(index int) ActionOption {
+	return func(o *MobileAction) {
+		o.Index = index
+	}
+}
+
 func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
 	return func(o *MobileAction) {
 		o.MaxRetryTimes = maxRetryTimes
@@ -235,13 +242,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
 	return dExt.Driver.FindElement(selector)
 }
 
-func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
 	// click on text, using OCR
 	if !isPathExists(search) {
-		return dExt.FindTextByOCR(search)
+		return dExt.FindTextByOCR(search, index...)
 	}
 	// click on image, using opencv
-	return dExt.FindImageRectInUIKit(search)
+	return dExt.FindImageRectInUIKit(search, index...)
 }
 
 func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
@@ -310,7 +317,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 			var x, y, width, height float64
 			findApp := func(d *DriverExt) error {
 				var err error
-				x, y, width, height, err = d.FindTextByOCR(appName)
+				x, y, width, height, err = d.FindTextByOCR(appName, action.Index)
 				return err
 			}
 			foundAppAction := func(d *DriverExt) error {
@@ -384,17 +391,17 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
 	case ACTION_Tap:
 		if param, ok := action.Params.(string); ok {
-			return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError)
+			return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
 	case ACTION_TapByOCR:
 		if ocrText, ok := action.Params.(string); ok {
-			return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError)
+			return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
 	case ACTION_TapByCV:
 		if imagePath, ok := action.Params.(string); ok {
-			return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError)
+			return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
 	case ACTION_DoubleTapXY:

hrp/internal/uixt/gesture_test.go

@@ -3,6 +3,7 @@
 package uixt
 
 import (
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -14,7 +15,7 @@ func TestDriverExt_GesturePassword(t *testing.T) {
 		password[i], _ = strconv.Atoi(split[i])
 	}
 
-	driverExt, err := InitWDAClient()
+	driverExt, err := InitWDAClient(nil)
 	checkErr(t, err)
 
 	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"

hrp/internal/uixt/ocr_off.go

@@ -4,7 +4,7 @@ package uixt
 
 import "github.com/rs/zerolog/log"
 
-func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
 	log.Fatal().Msg("OCR is not supported")
 	return
 }

hrp/internal/uixt/ocr_on.go

@@ -86,7 +86,11 @@ func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) {
 	return ocrResult.OCRResult, nil
 }
 
-func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) {
+func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) {
+	if len(index) == 0 {
+		index = []int{0} // index not specified
+	}
+
 	ocrResults, err := s.getOCRResult(imageBuf)
 	if err != nil {
 		return
@@ -110,30 +114,46 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rec
 				Y: int(ocrResult.Points[2].Y),
 			},
 		}
+		rects = append(rects, rect)
 
 		// contains text while not match exactly
 		if ocrResult.Text != text {
-			rects = append(rects, rect)
 			continue
 		}
 
-		// match exactly
+		// match exactly, and not specify index, return the first one
-		return rect, nil
+		if index[0] == 0 {
+			return rect, nil
+		}
 	}
 
-	// only find the first matched one
+	if len(rects) == 0 {
-	if len(rects) > 0 {
+		return image.Rectangle{}, fmt.Errorf("text %s not found", text)
-		return rects[0], nil
 	}
 
-	return image.Rectangle{}, fmt.Errorf("text %s not found", text)
+	// get index
+	idx := index[0]
+	if idx > 0 {
+		// NOTICE: index start from 1
+		idx = idx - 1
+	} else if idx < 0 {
+		idx = len(rects) + idx
+	}
+
+	// index out of range
+	if idx >= len(rects) {
+		return image.Rectangle{}, fmt.Errorf("text %s found %d, index %d out of range",
+			text, len(rects), idx)
+	}
+
+	return rects[idx], nil
 }
 
 type OCRService interface {
-	FindText(text string, imageBuf []byte) (rect image.Rectangle, err error)
+	FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
 }
 
-func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
 	var bufSource *bytes.Buffer
 	if bufSource, err = dExt.takeScreenShot(); err != nil {
 		err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -141,7 +161,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float6
 	}
 
 	service := &veDEMOCRService{}
-	rect, err := service.FindText(ocrText, bufSource.Bytes())
+	rect, err := service.FindText(ocrText, bufSource.Bytes(), index...)
 	if err != nil {
 		log.Warn().Err(err).Msg("FindText failed")
 		err = fmt.Errorf("FindText failed: %v", err)

hrp/internal/uixt/opencv_off.go

@@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
 	return
 }
 
-func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
 	log.Fatal().Msg("opencv is not supported")
 	return
 }

hrp/internal/uixt/opencv_on.go

@@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
 	return
 }
 
-func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
 	var bufSource, bufSearch *bytes.Buffer
 	if bufSearch, err = getBufFromDisk(imagePath); err != nil {
 		return 0, 0, 0, 0, err

hrp/internal/uixt/tap.go

@@ -27,8 +27,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
 	return dExt.tapFloat(x, y, identifier)
 }
 
-func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool) error {
+func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
-	x, y, width, height, err := dExt.FindTextByOCR(ocrText)
+	x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
 	if err != nil {
 		if ignoreNotFoundError {
 			return nil
@@ -39,7 +39,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun
 	return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
 }
 
-func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool) error {
+func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
 	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath)
 	if err != nil {
 		if ignoreNotFoundError {
@@ -51,18 +51,18 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou
 	return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
 }
 
-func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool) error {
+func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
-	return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError)
+	return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...)
 }
 
-func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool) (err error) {
+func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) {
 	// click on element, find by name attribute
 	ele, err := dExt.FindUIElement(param)
 	if err == nil {
 		return ele.Click()
 	}
 
-	x, y, width, height, err := dExt.FindUIRectInUIKit(param)
+	x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...)
 	if err != nil {
 		if ignoreNotFoundError {
 			return nil

hrp/step.go

@@ -19,6 +19,7 @@ const (
 var (
 	WithIdentifier          = uixt.WithIdentifier
 	WithMaxRetryTimes       = uixt.WithMaxRetryTimes
+	WithIndex               = uixt.WithIndex
 	WithTimeout             = uixt.WithTimeout
 	WithIgnoreNotFoundError = uixt.WithIgnoreNotFoundError
 )