From 498cc9432aa3e3d19929379f724bf721e2e559b6 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Sep 2022 22:25:13 +0800 Subject: [PATCH] feat: locate ocr text with index --- examples/uitest/demo_douyin_live.json | 89 +++++++++++++++++++++++++++ examples/uitest/demo_douyin_live.yaml | 45 ++++++++++++++ examples/uitest/demo_douyin_test.go | 52 ++++++++++++++++ examples/uitest/demo_weixin_live.json | 3 +- examples/uitest/demo_weixin_live.yaml | 1 + examples/uitest/demo_weixin_test.go | 4 +- hrp/internal/uixt/ext.go | 21 ++++--- hrp/internal/uixt/gesture_test.go | 3 +- hrp/internal/uixt/ocr_off.go | 2 +- hrp/internal/uixt/ocr_on.go | 42 +++++++++---- hrp/internal/uixt/opencv_off.go | 2 +- hrp/internal/uixt/opencv_on.go | 2 +- hrp/internal/uixt/tap.go | 14 ++--- hrp/step.go | 1 + 14 files changed, 249 insertions(+), 32 deletions(-) create mode 100644 examples/uitest/demo_douyin_live.json create mode 100644 examples/uitest/demo_douyin_live.yaml create mode 100644 examples/uitest/demo_douyin_test.go diff --git a/examples/uitest/demo_douyin_live.json b/examples/uitest/demo_douyin_live.json new file mode 100644 index 00000000..77cd4d9d --- /dev/null +++ b/examples/uitest/demo_douyin_live.json @@ -0,0 +1,89 @@ +{ + "config": { + "name": "通过 feed 卡片进入微信直播间", + "ios": [ + { + "port": 8700, + "mjpeg_port": 8800, + "log_on": true + } + ] + }, + "teststeps": [ + { + "name": "启动抖音", + "ios": { + "actions": [ + { + "method": "home" + }, + { + "method": "app_terminate", + "params": "com.ss.iphone.ugc.Aweme" + }, + { + "method": "swipe_to_tap_app", + "params": "抖音", + "max_retry_times": 5 + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_ocr", + "assert": "exists", + "expect": "推荐", + "msg": "抖音启动失败，「推荐」不存在" + } + ] + }, + { + "name": "在推荐页上划，直到出现「点击进入直播间」", + "ios": { + "actions": [ + { + "method": "swipe_to_tap_text", + "params": "点击进入直播间", + "identifier": "进入直播间", + "max_retry_times": 100 + } + ] + } + }, + { + "name": "向上滑动，等待 10s", + "ios": { + "actions": [ + { + "method": "swipe", + "params": "up", + "identifier": "第一次上划" + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "screenshot" + }, + { + "method": "swipe", + "params": "up", + "identifier": "第二次上划" + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "screenshot" + } + ] + } + } + ] +} diff --git a/examples/uitest/demo_douyin_live.yaml b/examples/uitest/demo_douyin_live.yaml new file mode 100644 index 00000000..f2c29f62 --- /dev/null +++ b/examples/uitest/demo_douyin_live.yaml @@ -0,0 +1,45 @@ +config: + name: 通过 feed 卡片进入微信直播间 + ios: + - port: 8700 + mjpeg_port: 8800 + log_on: true +teststeps: + - name: 启动抖音 + ios: + actions: + - method: home + - method: app_terminate + params: com.ss.iphone.ugc.Aweme + - method: swipe_to_tap_app + params: 抖音 + max_retry_times: 5 + - method: sleep + params: 5 + validate: + - check: ui_ocr + assert: exists + expect: 推荐 + msg: 抖音启动失败，「推荐」不存在 + - name: 在推荐页上划，直到出现「点击进入直播间」 + ios: + actions: + - method: swipe_to_tap_text + params: 点击进入直播间 + identifier: 进入直播间 + max_retry_times: 100 + - name: 向上滑动，等待 10s + ios: + actions: + - method: swipe + params: up + identifier: 第一次上划 + - method: sleep + params: 2 + - method: screenshot + - method: swipe + params: up + identifier: 第二次上划 + - method: sleep + params: 2 + - method: screenshot diff --git a/examples/uitest/demo_douyin_test.go b/examples/uitest/demo_douyin_test.go new file mode 100644 index 00000000..2880b4de --- /dev/null +++ b/examples/uitest/demo_douyin_test.go @@ -0,0 +1,52 @@ +package uitest + +import ( + "fmt" + "testing" + + "github.com/httprunner/httprunner/v4/hrp" +) + +func TestIOSDouyinLive(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("通过 feed 卡片进入微信直播间"). + SetIOS(hrp.WithLogOn(true), hrp.WithPort(8700), hrp.WithMjpegPort(8800)), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动抖音"). + IOS(). + Home(). + AppTerminate("com.ss.iphone.ugc.Aweme"). // 关闭已运行的抖音 + SwipeToTapApp("抖音", hrp.WithMaxRetryTimes(5)).Sleep(5). + Validate(). + AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"), + // hrp.NewStep("处理青少年弹窗"). + // IOS(). + // TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)), + hrp.NewStep("在推荐页上划，直到出现「点击进入直播间」"). + IOS(). + SwipeToTapText("点击进入直播间", hrp.WithMaxRetryTimes(100), hrp.WithIdentifier("进入直播间")), + hrp.NewStep("向上滑动，等待 10s"). + IOS(). + SwipeUp(hrp.WithIdentifier("第一次上划")).Sleep(2).ScreenShot(). // 上划 1 次，等待 2s，截图保存 + SwipeUp(hrp.WithIdentifier("第二次上划")).Sleep(2).ScreenShot(), // 再上划 1 次，等待 2s，截图保存 + }, + } + + if err := testCase.Dump2JSON("demo_douyin_live.json"); err != nil { + t.Fatal(err) + } + if err := testCase.Dump2YAML("demo_douyin_live.yaml"); err != nil { + t.Fatal(err) + } + + runner := hrp.NewRunner(t) + sessionRunner, err := runner.NewSessionRunner(testCase) + if err != nil { + t.Fatal(err) + } + if err := sessionRunner.Start(nil); err != nil { + t.Fatal(err) + } + summary := sessionRunner.GetSummary() + fmt.Println(summary) +} diff --git a/examples/uitest/demo_weixin_live.json b/examples/uitest/demo_weixin_live.json index f5f3f23a..618ec703 100644 --- a/examples/uitest/demo_weixin_live.json +++ b/examples/uitest/demo_weixin_live.json @@ -48,7 +48,8 @@ { "method": "tap_ocr", "params": "视频号", - "identifier": "进入视频号" + "identifier": "进入视频号", + "index": -1 } ] } diff --git a/examples/uitest/demo_weixin_live.yaml b/examples/uitest/demo_weixin_live.yaml index 44b19b50..3b064f09 100644 --- a/examples/uitest/demo_weixin_live.yaml +++ b/examples/uitest/demo_weixin_live.yaml @@ -27,6 +27,7 @@ teststeps: - method: tap_ocr params: 视频号 identifier: 进入视频号 + index: -1 - name: 处理青少年弹窗 ios: actions: diff --git a/examples/uitest/demo_weixin_test.go b/examples/uitest/demo_weixin_test.go index c692ddf1..f157e27c 100644 --- a/examples/uitest/demo_weixin_test.go +++ b/examples/uitest/demo_weixin_test.go @@ -21,8 +21,8 @@ func TestIOSWeixinLive(t *testing.T) { AssertLabelExists("通讯录", "微信启动失败，「通讯录」不存在"), hrp.NewStep("进入直播页"). IOS(). - Tap("发现"). // 进入「发现页」 - TapByOCR("视频号", hrp.WithIdentifier("进入视频号")), // 通过 OCR 识别「视频号」 + Tap("发现"). // 进入「发现页」 + TapByOCR("视频号", hrp.WithIdentifier("进入视频号"), hrp.WithIndex(-1)), // 通过 OCR 识别「视频号」 hrp.NewStep("处理青少年弹窗"). IOS(). TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)), diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go index 1b44922b..a89385a2 100644 --- a/hrp/internal/uixt/ext.go +++ b/hrp/internal/uixt/ext.go @@ -63,6 +63,7 @@ type MobileAction struct { Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found } @@ -75,6 +76,12 @@ func WithIdentifier(identifier string) ActionOption { } } +func WithIndex(index int) ActionOption { + return func(o *MobileAction) { + o.Index = index + } +} + func WithMaxRetryTimes(maxRetryTimes int) ActionOption { return func(o *MobileAction) { o.MaxRetryTimes = maxRetryTimes @@ -235,13 +242,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) { return dExt.Driver.FindElement(selector) } -func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search) + return dExt.FindTextByOCR(search, index...) } // click on image, using opencv - return dExt.FindImageRectInUIKit(search) + return dExt.FindImageRectInUIKit(search, index...) } func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { @@ -310,7 +317,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var x, y, width, height float64 findApp := func(d *DriverExt) error { var err error - x, y, width, height, err = d.FindTextByOCR(appName) + x, y, width, height, err = d.FindTextByOCR(appName, action.Index) return err } foundAppAction := func(d *DriverExt) error { @@ -384,17 +391,17 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) case ACTION_Tap: if param, ok := action.Params.(string); ok { - return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError) + return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index) } return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError) + return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: if imagePath, ok := action.Params.(string); ok { - return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError) + return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: diff --git a/hrp/internal/uixt/gesture_test.go b/hrp/internal/uixt/gesture_test.go index 5e4ce57b..31664954 100644 --- a/hrp/internal/uixt/gesture_test.go +++ b/hrp/internal/uixt/gesture_test.go @@ -3,6 +3,7 @@ package uixt import ( + "strconv" "strings" "testing" ) @@ -14,7 +15,7 @@ func TestDriverExt_GesturePassword(t *testing.T) { password[i], _ = strconv.Atoi(split[i]) } - driverExt, err := InitWDAClient() + driverExt, err := InitWDAClient(nil) checkErr(t, err) pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" diff --git a/hrp/internal/uixt/ocr_off.go b/hrp/internal/uixt/ocr_off.go index 7c3536b5..669dbe46 100644 --- a/hrp/internal/uixt/ocr_off.go +++ b/hrp/internal/uixt/ocr_off.go @@ -4,7 +4,7 @@ package uixt import "github.com/rs/zerolog/log" -func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) { log.Fatal().Msg("OCR is not supported") return } diff --git a/hrp/internal/uixt/ocr_on.go b/hrp/internal/uixt/ocr_on.go index 6583c3f5..327bc373 100644 --- a/hrp/internal/uixt/ocr_on.go +++ b/hrp/internal/uixt/ocr_on.go @@ -86,7 +86,11 @@ func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) { return ocrResult.OCRResult, nil } -func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) { +func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) { + if len(index) == 0 { + index = []int{0} // index not specified + } + ocrResults, err := s.getOCRResult(imageBuf) if err != nil { return @@ -110,30 +114,46 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rec Y: int(ocrResult.Points[2].Y), }, } + rects = append(rects, rect) // contains text while not match exactly if ocrResult.Text != text { - rects = append(rects, rect) continue } - // match exactly - return rect, nil + // match exactly, and not specify index, return the first one + if index[0] == 0 { + return rect, nil + } } - // only find the first matched one - if len(rects) > 0 { - return rects[0], nil + if len(rects) == 0 { + return image.Rectangle{}, fmt.Errorf("text %s not found", text) } - return image.Rectangle{}, fmt.Errorf("text %s not found", text) + // get index + idx := index[0] + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, fmt.Errorf("text %s found %d, index %d out of range", + text, len(rects), idx) + } + + return rects[idx], nil } type OCRService interface { - FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) + FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) } -func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) @@ -141,7 +161,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float6 } service := &veDEMOCRService{} - rect, err := service.FindText(ocrText, bufSource.Bytes()) + rect, err := service.FindText(ocrText, bufSource.Bytes(), index...) if err != nil { log.Warn().Err(err).Msg("FindText failed") err = fmt.Errorf("FindText failed: %v", err) diff --git a/hrp/internal/uixt/opencv_off.go b/hrp/internal/uixt/opencv_off.go index 4fc747c1..db1865ae 100644 --- a/hrp/internal/uixt/opencv_off.go +++ b/hrp/internal/uixt/opencv_off.go @@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) { log.Fatal().Msg("opencv is not supported") return } diff --git a/hrp/internal/uixt/opencv_on.go b/hrp/internal/uixt/opencv_on.go index 5fdc7197..d4acda45 100644 --- a/hrp/internal/uixt/opencv_on.go +++ b/hrp/internal/uixt/opencv_on.go @@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) { var bufSource, bufSearch *bytes.Buffer if bufSearch, err = getBufFromDisk(imagePath); err != nil { return 0, 0, 0, 0, err diff --git a/hrp/internal/uixt/tap.go b/hrp/internal/uixt/tap.go index 1d4eb168..4acff134 100644 --- a/hrp/internal/uixt/tap.go +++ b/hrp/internal/uixt/tap.go @@ -27,8 +27,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { return dExt.tapFloat(x, y, identifier) } -func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool) error { - x, y, width, height, err := dExt.FindTextByOCR(ocrText) +func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error { + x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...) if err != nil { if ignoreNotFoundError { return nil @@ -39,7 +39,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier) } -func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool) error { +func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error { x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath) if err != nil { if ignoreNotFoundError { @@ -51,18 +51,18 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier) } -func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool) error { - return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError) +func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error { + return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...) } -func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool) (err error) { +func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) { // click on element, find by name attribute ele, err := dExt.FindUIElement(param) if err == nil { return ele.Click() } - x, y, width, height, err := dExt.FindUIRectInUIKit(param) + x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...) if err != nil { if ignoreNotFoundError { return nil diff --git a/hrp/step.go b/hrp/step.go index f3585e40..13b6da61 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -19,6 +19,7 @@ const ( var ( WithIdentifier = uixt.WithIdentifier WithMaxRetryTimes = uixt.WithMaxRetryTimes + WithIndex = uixt.WithIndex WithTimeout = uixt.WithTimeout WithIgnoreNotFoundError = uixt.WithIgnoreNotFoundError )