From 2815d6733168c2341ae0f210a75c2ac955b0a766 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Tue, 11 Oct 2022 14:53:21 +0800 Subject: [PATCH] feat: tap the first one matches text from given texts by ocr --- examples/uitest/demo_douyin_follow_live.json | 152 ++++++++++++++++++ examples/uitest/demo_douyin_follow_live.yaml | 83 ++++++++++ .../uitest/demo_douyin_follow_live_test.go | 58 +++++++ hrp/pkg/uixt/android_driver.go | 2 +- hrp/pkg/uixt/ext.go | 81 +++++++--- hrp/pkg/uixt/ocr_off.go | 9 +- hrp/pkg/uixt/ocr_on.go | 71 ++++++++ hrp/pkg/uixt/swipe.go | 12 +- hrp/pkg/uixt/tap.go | 21 +++ hrp/step.go | 1 + hrp/step_android_ui.go | 12 ++ hrp/step_ios_ui.go | 12 ++ 12 files changed, 485 insertions(+), 29 deletions(-) create mode 100644 examples/uitest/demo_douyin_follow_live.json create mode 100644 examples/uitest/demo_douyin_follow_live.yaml create mode 100644 examples/uitest/demo_douyin_follow_live_test.go diff --git a/examples/uitest/demo_douyin_follow_live.json b/examples/uitest/demo_douyin_follow_live.json new file mode 100644 index 00000000..3be24779 --- /dev/null +++ b/examples/uitest/demo_douyin_follow_live.json @@ -0,0 +1,152 @@ +{ + "config": { + "name": "通过 关注天窗 进入指定主播抖音直播间", + "variables": { + "app_name": "抖音" + }, + "ios": [ + { + "port": 8100, + "mjpeg_port": 9100, + "log_on": true + } + ] + }, + "teststeps": [ + { + "name": "启动抖音", + "ios": { + "actions": [ + { + "method": "home" + }, + { + "method": "app_terminate", + "params": "com.ss.iphone.ugc.Aweme" + }, + { + "method": "swipe_to_tap_app", + "params": "$app_name", + "identifier": "启动抖音", + "max_retry_times": 5 + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_ocr", + "assert": "exists", + "expect": "推荐", + "msg": "抖音启动失败，「推荐」不存在" + } + ] + }, + { + "name": "处理青少年弹窗", + "ios": { + "actions": [ + { + "method": "tap_ocr", + "params": "我知道了", + "ignore_NotFoundError": true + } + ] + } + }, + { + "name": "点击首页", + "ios": { + "actions": [ + { + "method": "tap_ocr", + "params": "首页", + "index": -1 + }, + { + "method": "sleep", + "params": 10 + } + ] + } + }, + { + "name": "点击关注页", + "ios": { + "actions": [ + { + "method": "tap_ocr", + "params": "关注", + "index": 1 + }, + { + "method": "sleep", + "params": 10 + } + ] + } + }, + { + "name": "向上滑动 2 次", + "ios": { + "actions": [ + { + "method": "swipe_to_tap_text", + "params": [ + "理肤泉", + "婉宝" + ], + "identifier": "click_live", + "direction": [ + 0.6, + 0.2, + 0.2, + 0.2 + ] + }, + { + "method": "sleep", + "params": 10 + }, + { + "method": "swipe", + "params": [ + 0.9, + 0.7, + 0.9, + 0.3 + ], + "identifier": "slide_in_live" + }, + { + "method": "sleep", + "params": 10 + }, + { + "method": "screenshot" + }, + { + "method": "swipe", + "params": [ + 0.9, + 0.7, + 0.9, + 0.3 + ], + "identifier": "slide_in_live" + }, + { + "method": "sleep", + "params": 10 + }, + { + "method": "screenshot" + } + ] + } + } + ] +} diff --git a/examples/uitest/demo_douyin_follow_live.yaml b/examples/uitest/demo_douyin_follow_live.yaml new file mode 100644 index 00000000..4e1b616c --- /dev/null +++ b/examples/uitest/demo_douyin_follow_live.yaml @@ -0,0 +1,83 @@ +config: + name: 通过 关注天窗 进入指定主播抖音直播间 + variables: + app_name: 抖音 + ios: + - port: 8100 + mjpeg_port: 9100 + log_on: true +teststeps: + - name: 启动抖音 + ios: + actions: + - method: home + - method: app_terminate + params: com.ss.iphone.ugc.Aweme + - method: swipe_to_tap_app + params: $app_name + identifier: 启动抖音 + max_retry_times: 5 + - method: sleep + params: 5 + validate: + - check: ui_ocr + assert: exists + expect: 推荐 + msg: 抖音启动失败，「推荐」不存在 + - name: 处理青少年弹窗 + ios: + actions: + - method: tap_ocr + params: 我知道了 + ignore_NotFoundError: true + - name: 点击首页 + ios: + actions: + - method: tap_ocr + params: 首页 + index: -1 + - method: sleep + params: 10 + - name: 点击关注页 + ios: + actions: + - method: tap_ocr + params: 关注 + index: 1 + - method: sleep + params: 10 + - name: 向上滑动 2 次 + ios: + actions: + - method: swipe_to_tap_text + params: + - 理肤泉 + - 婉宝 + identifier: click_live + direction: + - 0.6 + - 0.2 + - 0.2 + - 0.2 + - method: sleep + params: 10 + - method: swipe + params: + - 0.9 + - 0.7 + - 0.9 + - 0.3 + identifier: slide_in_live + - method: sleep + params: 10 + - method: screenshot + - method: swipe + params: + - 0.9 + - 0.7 + - 0.9 + - 0.3 + identifier: slide_in_live + - method: sleep + params: 10 + - method: screenshot diff --git a/examples/uitest/demo_douyin_follow_live_test.go b/examples/uitest/demo_douyin_follow_live_test.go new file mode 100644 index 00000000..a20dd363 --- /dev/null +++ b/examples/uitest/demo_douyin_follow_live_test.go @@ -0,0 +1,58 @@ +//go:build localtest + +package uitest + +import ( + "testing" + + "github.com/httprunner/httprunner/v4/hrp" +) + +func TestIOSDouyinFollowLive(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("通过 关注天窗 进入指定主播抖音直播间"). + WithVariables(map[string]interface{}{ + "app_name": "抖音", + }). + SetIOS( + hrp.WithLogOn(true), + hrp.WithWDAPort(8100), + hrp.WithWDAMjpegPort(9100), + ), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动抖音"). + IOS(). + Home(). + AppTerminate("com.ss.iphone.ugc.Aweme"). // 关闭已运行的抖音 + SwipeToTapApp("$app_name", hrp.WithMaxRetryTimes(5), hrp.WithIdentifier("启动抖音")).Sleep(5). + Validate(). + AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"), + hrp.NewStep("处理青少年弹窗"). + IOS(). + TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)), + hrp.NewStep("点击首页"). + IOS(). + TapByOCR("首页", hrp.WithIndex(-1)).Sleep(10), + hrp.NewStep("点击关注页"). + IOS(). + TapByOCR("关注", hrp.WithIndex(1)).Sleep(10), + hrp.NewStep("向上滑动 2 次"). + IOS().SwipeToTapFromTexts([]string{"理肤泉", "婉宝"}, hrp.WithDirection([]float64{0.6, 0.2, 0.2, 0.2}), hrp.WithIdentifier("click_live")).Sleep(10). + Swipe(0.9, 0.7, 0.9, 0.3, hrp.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(). // 上划 1 次，等待 10s，截图保存 + Swipe(0.9, 0.7, 0.9, 0.3, hrp.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(), // 再上划 1 次，等待 10s，截图保存 + }, + } + + if err := testCase.Dump2JSON("demo_douyin_follow_live.json"); err != nil { + t.Fatal(err) + } + if err := testCase.Dump2YAML("demo_douyin_follow_live.yaml"); err != nil { + t.Fatal(err) + } + + runner := hrp.NewRunner(t).SetSaveTests(true) + err := runner.Run(testCase) + if err != nil { + t.Fatal(err) + } +} diff --git a/hrp/pkg/uixt/android_driver.go b/hrp/pkg/uixt/android_driver.go index 58073906..b4a2b713 100644 --- a/hrp/pkg/uixt/android_driver.go +++ b/hrp/pkg/uixt/android_driver.go @@ -693,7 +693,7 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) { } var element WebElement - if valuetext, ok := data["text"]; ok { + if valuetext, ok := data["textview"]; ok { element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()}) } else if valueid, ok := data["id"]; ok { element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)}) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index babc74fc..14cbd0e3 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -63,14 +63,15 @@ type MobileAction struct { Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"` Params interface{} `json:"params,omitempty" yaml:"params,omitempty"` - Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log - MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action - IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found - Text string `json:"text,omitempty" yaml:"text,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` + Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log + MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action + IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found + Text string `json:"text,omitempty" yaml:"text,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` } type ActionOption func(o *MobileAction) @@ -87,6 +88,13 @@ func WithIndex(index int) ActionOption { } } +// WithDirection inputs direction (up, down, left, right, []float64{sx, sy, ex, ey}) +func WithDirection(direction interface{}) ActionOption { + return func(o *MobileAction) { + o.Direction = direction + } +} + func WithText(text string) ActionOption { return func(o *MobileAction) { o.Text = text @@ -363,7 +371,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } foundAppAction := func(d *DriverExt) error { // click app to launch - return d.TapAbsXY(point.X, point.Y-20, action.Identifier) + return d.TapAbsXY(point.X, point.Y-25, action.Identifier) } // go to home screen @@ -386,27 +394,52 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params, should be app name(string), got %v", ACTION_SwipeToTapApp, action.Params) case ACTION_SwipeToTapText: + var point PointF + var findText func(d *DriverExt) error + if text, ok := action.Params.(string); ok { - var point PointF - findText := func(d *DriverExt) error { + findText = func(d *DriverExt) error { var err error point, err = d.GetTextXY(text, action.Index) return err } - foundTextAction := func(d *DriverExt) error { - // tap text - return d.TapAbsXY(point.X, point.Y, action.Identifier) + } else if texts, ok := action.Params.([]interface{}); ok { + findText = func(d *DriverExt) error { + var err error + var ts []string + for _, t := range texts { + ts = append(ts, t.(string)) + } + points, err := d.GetTextXYs(ts) + if err != nil { + return err + } + for _, point = range points { + if point != (PointF{}) { + return nil + } + } + return errors.New("failed to find text position") } - - // default to retry 10 times - if action.MaxRetryTimes == 0 { - action.MaxRetryTimes = 10 - } - // swipe until live room found - return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes) + } else { + return fmt.Errorf("invalid %s params, should be app text(string or []string), got %v", + ACTION_SwipeToTapText, action.Params) } - return fmt.Errorf("invalid %s params, should be app text(string), got %v", - ACTION_SwipeToTapText, action.Params) + + foundTextAction := func(d *DriverExt) error { + // tap text + return d.TapAbsXY(point.X, point.Y, action.Identifier) + } + + // default to retry 10 times + if action.MaxRetryTimes == 0 { + action.MaxRetryTimes = 10 + } + if action.Direction != nil { + return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes) + } + // swipe until live room found + return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes) case AppTerminate: if bundleId, ok := action.Params.(string); ok { success, err := dExt.Driver.AppTerminate(bundleId) @@ -497,7 +530,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { param := fmt.Sprintf("%v", action.Params) options := []DataOption{} if action.Text != "" { - options = append(options, WithCustomOption("text", action.Text)) + options = append(options, WithCustomOption("textview", action.Text)) } if action.ID != "" { options = append(options, WithCustomOption("id", action.ID)) diff --git a/hrp/pkg/uixt/ocr_off.go b/hrp/pkg/uixt/ocr_off.go index 669dbe46..03b2e505 100644 --- a/hrp/pkg/uixt/ocr_off.go +++ b/hrp/pkg/uixt/ocr_off.go @@ -2,9 +2,16 @@ package uixt -import "github.com/rs/zerolog/log" +import ( + "github.com/rs/zerolog/log" +) func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) { log.Fatal().Msg("OCR is not supported") return } + +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) { + log.Fatal().Msg("OCR is not supported") + return +} diff --git a/hrp/pkg/uixt/ocr_on.go b/hrp/pkg/uixt/ocr_on.go index 550da8e7..7a5faf91 100644 --- a/hrp/pkg/uixt/ocr_on.go +++ b/hrp/pkg/uixt/ocr_on.go @@ -159,6 +159,48 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) ( return rects[idx], nil } +func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[string]image.Rectangle, err error) { + ocrResults, err := s.getOCRResult(imageBuf) + if err != nil { + log.Error().Err(err).Msg("getOCRResult failed") + return + } + + var ocrTexts []string + rects = map[string]image.Rectangle{} + + for _, text := range texts { + for _, ocrResult := range ocrResults { + ocrTexts = append(ocrTexts, ocrResult.Text) + + // not contains text + if !strings.Contains(ocrResult.Text, text) { + continue + } + + rect := image.Rectangle{ + // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(ocrResult.Points[0].X), + Y: int(ocrResult.Points[0].Y), + }, + Max: image.Point{ + X: int(ocrResult.Points[2].X), + Y: int(ocrResult.Points[2].Y), + }, + } + rects[text] = rect + break + } + + if _, ok := rects[text]; !ok { + rects[text] = image.Rectangle{} + } + } + + return rects, nil +} + type OCRService interface { FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) } @@ -182,3 +224,32 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, x, y, width, height = dExt.MappingToRectInUIKit(rect) return } + +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + service := &veDEMOCRService{} + rects, err := service.FindTexts(ocrTexts, bufSource.Bytes()) + if err != nil { + log.Warn().Msgf("FindTexts failed: %s", err.Error()) + err = fmt.Errorf("FindTexts failed: %v", err) + return + } + + ps = map[string][]float64{} + log.Info().Interface("ocrTexts", ocrTexts).Msgf("FindTexts success") + for text, rect := range rects { + if rect == (image.Rectangle{}) { + ps[text] = []float64{} + continue + } + x, y, width, height := dExt.MappingToRectInUIKit(rect) + ps[text] = []float64{x, y, width, height} + } + + return +} diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 97ac8fc5..5cb8bffa 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -72,14 +72,20 @@ type FindCondition func(driver *DriverExt) error // FoundAction indicates the action to do after a UI element is found type FoundAction func(driver *DriverExt) error -func (dExt *DriverExt) SwipeUntil(direction string, condition FindCondition, action FoundAction, maxTimes int) error { +func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int) error { for i := 0; i < maxTimes; i++ { if err := condition(dExt); err == nil { // do action after found return action(dExt) } - if err := dExt.SwipeTo(direction); err != nil { - log.Error().Err(err).Msgf("swipe %s failed", direction) + if d, ok := direction.(string); ok { + if err := dExt.SwipeTo(d); err != nil { + log.Error().Err(err).Msgf("swipe %s failed", d) + } + } else if d, ok := direction.([]float64); ok { + if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3]); err != nil { + log.Error().Err(err).Msgf("swipe %s failed", d) + } } } return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes) diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index ada1ae73..10e1f79c 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -41,6 +41,27 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er return point, nil } +func (dExt *DriverExt) GetTextXYs(ocrText []string) (points map[string]PointF, err error) { + ps, err := dExt.FindTextsByOCR(ocrText) + if err != nil { + return map[string]PointF{}, err + } + + points = map[string]PointF{} + for text, point := range ps { + if len(point) == 0 { + points[text] = PointF{} + continue + } + points[text] = PointF{ + X: point[0] + point[2]*0.5, + Y: point[1] + point[3]*0.5, + } + } + + return points, nil +} + func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) { x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...) if err != nil { diff --git a/hrp/step.go b/hrp/step.go index 9f6989eb..244903bd 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -29,6 +29,7 @@ var ( WithText = uixt.WithText WithID = uixt.WithID WithDescription = uixt.WithDescription + WithDirection = uixt.WithDirection ) var ( diff --git a/hrp/step_android_ui.go b/hrp/step_android_ui.go index 8c175c21..6bb8b33d 100644 --- a/hrp/step_android_ui.go +++ b/hrp/step_android_ui.go @@ -324,6 +324,18 @@ func (s *StepAndroid) SwipeToTapText(text string, options ...uixt.ActionOption) return &StepAndroid{step: s.step} } +func (s *StepAndroid) SwipeToTapFromTexts(texts []string, options ...uixt.ActionOption) *StepAndroid { + action := uixt.MobileAction{ + Method: uixt.ACTION_SwipeToTapText, + Params: texts, + } + for _, option := range options { + option(&action) + } + s.step.Android.Actions = append(s.step.Android.Actions, action) + return &StepAndroid{step: s.step} +} + // Validate switches to step validation. func (s *StepAndroid) Validate() *StepAndroidValidation { return &StepAndroidValidation{ diff --git a/hrp/step_ios_ui.go b/hrp/step_ios_ui.go index 7ba60103..5f54c28a 100644 --- a/hrp/step_ios_ui.go +++ b/hrp/step_ios_ui.go @@ -244,6 +244,18 @@ func (s *StepIOS) SwipeToTapText(text string, options ...uixt.ActionOption) *Ste return &StepIOS{step: s.step} } +func (s *StepIOS) SwipeToTapFromTexts(texts []string, options ...uixt.ActionOption) *StepIOS { + action := uixt.MobileAction{ + Method: uixt.ACTION_SwipeToTapText, + Params: texts, + } + for _, option := range options { + option(&action) + } + s.step.IOS.Actions = append(s.step.IOS.Actions, action) + return &StepIOS{step: s.step} +} + func (s *StepIOS) Input(text string, options ...uixt.ActionOption) *StepIOS { action := uixt.MobileAction{ Method: uixt.ACTION_Input,