From 4dace0f589f48c026795323e2f49087f440029b0 Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Fri, 25 Aug 2023 14:01:46 +0800 Subject: [PATCH] implement TapByUITypes using WithScreenShotUITypes --- examples/uitest/android_expert_test.json | 173 +++++++++++----------- examples/uitest/expert_test.go | 26 ++-- examples/uitest/ios_expert_test.json | 177 +++++++++++------------ hrp/pkg/uixt/action.go | 13 +- hrp/pkg/uixt/service_vedem.go | 11 +- hrp/pkg/uixt/tap.go | 4 +- hrp/step_mobile_ui.go | 3 +- 7 files changed, 194 insertions(+), 213 deletions(-) diff --git a/examples/uitest/android_expert_test.json b/examples/uitest/android_expert_test.json index 59fb8fc5..97edc084 100644 --- a/examples/uitest/android_expert_test.json +++ b/examples/uitest/android_expert_test.json @@ -49,9 +49,7 @@ { "method": "swipe_to_tap_app", "params": "$app_name", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -147,9 +145,7 @@ }, { "method": "back", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -222,9 +218,7 @@ { "method": "tap_ocr", "params": "直播", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -243,9 +237,7 @@ 0.5, 0.5 ], - "options": { - - } + "options": {} }, { "method": "sleep", @@ -260,12 +252,12 @@ "actions": [ { "method": "tap_cv", - "params": [ - "dyhouse", - "shoppingbag" - ], "options": { - "identifier": "click_sales_rack" + "identifier": "click_sales_rack", + "screenshot_with_ui_types": [ + "dyhouse", + "shoppingbag" + ] } }, { @@ -354,78 +346,78 @@ "method": "video_crawler", "params": { "feed": { - "sleep_random": [ - 0, - 5, - 0.6, - 5, - 15, - 0.2, - 15, - 50, - 0.2 -], - "target_count": 10, - "target_labels": [ - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^广告$" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^图文$" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^特效\\|" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^模板\\|" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^购物\\|" - } -] -}, + "sleep_random": [ + 0, + 5, + 0.6, + 5, + 15, + 0.2, + 15, + 50, + 0.2 + ], + "target_count": 10, + "target_labels": [ + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^广告$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^图文$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^特效\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^模板\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^购物\\|" + } + ] + }, "live": { - "sleep_random": [ - 20, - 20 -], - "target_count": 0 -}, + "sleep_random": [ + 20, + 20 + ], + "target_count": 0 + }, "timeout": 600 } } @@ -466,7 +458,8 @@ "android": { "actions": [ { - "method": "screenshot" + "method": "screenshot", + "options": {} }, { "method": "sleep_random", @@ -480,4 +473,4 @@ "loops": 3 } ] -} +} \ No newline at end of file diff --git a/examples/uitest/expert_test.go b/examples/uitest/expert_test.go index 5cc8cadc..0067f625 100644 --- a/examples/uitest/expert_test.go +++ b/examples/uitest/expert_test.go @@ -24,13 +24,13 @@ func TestAndroidExpertTest(t *testing.T) { uixt.WithUIA2(true), ), TestSteps: []hrp.IStep{ + // 温启动 hrp.NewStep("app_launch 以及 ui_foreground_app equal 断言"). Android(). AppLaunch("$bundle_id"). Sleep(2). Validate(). AssertAppInForeground("$bundle_id"), - // 直播赛道 hrp.NewStep("home 以及 swipe_to_tap_app 默认配置"). Android(). Home(). @@ -44,6 +44,7 @@ func TestAndroidExpertTest(t *testing.T) { ). Validate(). AssertOCRExists("推荐", "进入抖音失败"), + // 直播赛道 hrp.NewStep("【直播】feed头像或卡片进房 swipe_to_tap_texts 自定义配置"). Android(). SwipeToTapTexts( @@ -93,19 +94,19 @@ func TestAndroidExpertTest(t *testing.T) { Android(). TapByOCR("直播"). Sleep(5), + // 生活服务赛道 hrp.NewStep("【生活服务】进入直播间 tap_xy"). Android(). TapXY(0.5, 0.5). Sleep(5), - // 生活服务赛道 hrp.NewStep("【生活服务】点击货架商品 tap_ocr 自定义配置"). Android(). TapByUITypes( - []string{"dyhouse", "shoppingbag"}, + uixt.WithScreenShotUITypes("dyhouse", "shoppingbag"), uixt.WithIdentifier("click_sales_rack"), ). Sleep(5), - // 重新进入应用程序 + // 冷启动 hrp.NewStep("app_terminate 以及 ui_foreground_app not_equal 断言"). Android(). AppTerminate("$bundle_id"). @@ -182,12 +183,12 @@ func TestIOSExpertTest(t *testing.T) { uixt.WithWDAMjpegPort(8800), ), TestSteps: []hrp.IStep{ - hrp.NewStep("app_launch 以及 ui_foreground_app equal 断言"). + // 温启动 + // iOS 不支持前台 App 断言操作 + hrp.NewStep("启动应用程序 app_launch"). IOS(). AppLaunch("$bundle_id"). Sleep(2), - // iOS 不支持前台 App 断言操作 - // 直播赛道 hrp.NewStep("home 以及 swipe_to_tap_app 默认配置"). IOS(). Home(). @@ -201,6 +202,7 @@ func TestIOSExpertTest(t *testing.T) { ). Validate(). AssertOCRExists("推荐", "进入抖音失败"), + // 直播赛道 hrp.NewStep("【直播】feed头像或卡片进房 swipe_to_tap_texts 自定义配置"). IOS(). SwipeToTapTexts( @@ -246,6 +248,7 @@ func TestIOSExpertTest(t *testing.T) { uixt.WithIndex(0), ). Sleep(5), + // 生活服务赛道 hrp.NewStep("选择直播页签 tap_ocr 默认配置"). IOS(). TapByOCR("直播"). @@ -254,20 +257,19 @@ func TestIOSExpertTest(t *testing.T) { IOS(). TapXY(0.5, 0.5). Sleep(5), - // 生活服务赛道 hrp.NewStep("【生活服务】点击货架商品 tap_ocr 自定义配置"). IOS(). TapByUITypes( - []string{"dyhouse", "shoppingbag"}, + uixt.WithScreenShotUITypes("dyhouse", "shoppingbag"), uixt.WithIdentifier("click_sales_rack"), ). Sleep(5), - // 重新进入应用程序 - hrp.NewStep("app_terminate 以及 ui_foreground_app not_equal 断言"). + // 冷启动 + // iOS 不支持前台 App 断言操作 + hrp.NewStep("终止应用程序 app_terminate"). IOS(). AppTerminate("$bundle_id"). Sleep(2), - // iOS 不支持前台 App 断言操作 hrp.NewStep("home 以及 swipe_to_tap_app 自定义配置"). IOS(). Home(). diff --git a/examples/uitest/ios_expert_test.json b/examples/uitest/ios_expert_test.json index a78aaee4..85fccf35 100644 --- a/examples/uitest/ios_expert_test.json +++ b/examples/uitest/ios_expert_test.json @@ -18,7 +18,7 @@ }, "teststeps": [ { - "name": "app_launch 以及 ui_foreground_app equal 断言", + "name": "启动应用程序 app_launch", "ios": { "actions": [ { @@ -42,9 +42,7 @@ { "method": "swipe_to_tap_app", "params": "$app_name", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -140,9 +138,7 @@ }, { "method": "back", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -215,9 +211,7 @@ { "method": "tap_ocr", "params": "直播", - "options": { - - } + "options": {} }, { "method": "sleep", @@ -236,9 +230,7 @@ 0.5, 0.5 ], - "options": { - - } + "options": {} }, { "method": "sleep", @@ -253,12 +245,12 @@ "actions": [ { "method": "tap_cv", - "params": [ - "dyhouse", - "shoppingbag" - ], "options": { - "identifier": "click_sales_rack" + "identifier": "click_sales_rack", + "screenshot_with_ui_types": [ + "dyhouse", + "shoppingbag" + ] } }, { @@ -269,7 +261,7 @@ } }, { - "name": "app_terminate 以及 ui_foreground_app not_equal 断言", + "name": "终止应用程序 app_terminate", "ios": { "actions": [ { @@ -339,78 +331,78 @@ "method": "video_crawler", "params": { "feed": { - "sleep_random": [ - 0, - 5, - 0.6, - 5, - 15, - 0.2, - 15, - 50, - 0.2 -], - "target_count": 10, - "target_labels": [ - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^广告$" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^图文$" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^特效\\|" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^模板\\|" - }, - { - "regex": true, - "scope": [ - 0, - 0.5, - 1, - 1 -], - "text": "^购物\\|" - } -] -}, + "sleep_random": [ + 0, + 5, + 0.6, + 5, + 15, + 0.2, + 15, + 50, + 0.2 + ], + "target_count": 10, + "target_labels": [ + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^广告$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^图文$" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^特效\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^模板\\|" + }, + { + "regex": true, + "scope": [ + 0, + 0.5, + 1, + 1 + ], + "text": "^购物\\|" + } + ] + }, "live": { - "sleep_random": [ - 20, - 20 -], - "target_count": 0 -}, + "sleep_random": [ + 20, + 20 + ], + "target_count": 0 + }, "timeout": 600 } } @@ -451,7 +443,8 @@ "ios": { "actions": [ { - "method": "screenshot" + "method": "screenshot", + "options": {} }, { "method": "sleep_random", @@ -465,4 +458,4 @@ "loops": 3 } ] -} +} \ No newline at end of file diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 2dfa40f0..0b3b14ca 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -547,17 +547,8 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) { case ACTION_TapByCV: if imagePath, ok := action.Params.(string); ok { return dExt.TapByCV(imagePath, action.GetOptions()...) - } - if uiParams, ok := action.Params.([]interface{}); ok { - var uiTypes []string - for _, uiParam := range uiParams { - uiType, ok := uiParam.(string) - if !ok { - continue - } - uiTypes = append(uiTypes, uiType) - } - return dExt.TapByUIDetection(uiTypes, action.Options.Options()...) + } else if err := dExt.TapByUIDetection(action.GetOptions()...); err == nil { + return nil } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index a57e0d55..cf505d59 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -478,6 +478,7 @@ func (u UIResults) FilterScope(scope AbsScope) (results UIResults) { type UIResultMap map[string]UIResults +// FilterUIResults filters ui icons, the former the uiTypes, the higher the priority func (u UIResultMap) FilterUIResults(uiTypes []string) (uiResults UIResults, err error) { var ok bool for _, uiType := range uiTypes { @@ -512,20 +513,22 @@ func (u UIResults) GetUIResult(options ...ActionOption) (UIResult, error) { return uiResults[idx], nil } -func (dExt *DriverExt) FindUIResult(uiTypes []string, options ...ActionOption) (point PointF, err error) { - screenResult, err := dExt.GetScreenResult(WithScreenShotUITypes(uiTypes...)) +func (dExt *DriverExt) FindUIResult(options ...ActionOption) (point PointF, err error) { + actionOptions := NewActionOptions(options...) + + screenResult, err := dExt.GetScreenResult(options...) if err != nil { return } - uiResults, err := screenResult.Icons.FilterUIResults(uiTypes) + uiResults, err := screenResult.Icons.FilterUIResults(actionOptions.ScreenShotWithUITypes) if err != nil { return } uiResult, err := uiResults.GetUIResult(dExt.ParseActionOptions(options...)...) point = uiResult.Center() - log.Info().Interface("text", uiTypes). + log.Info().Interface("text", actionOptions.ScreenShotWithUITypes). Interface("point", point).Msg("FindUIResult success") return } diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 83603ccf..b685b7bc 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -49,10 +49,10 @@ func (dExt *DriverExt) TapByCV(imagePath string, options ...ActionOption) error return dExt.TapAbsXY(point.X, point.Y, options...) } -func (dExt *DriverExt) TapByUIDetection(uiTypes []string, options ...ActionOption) error { +func (dExt *DriverExt) TapByUIDetection(options ...ActionOption) error { actionOptions := NewActionOptions(options...) - point, err := dExt.FindUIResult(uiTypes, options...) + point, err := dExt.FindUIResult(options...) if err != nil { if actionOptions.IgnoreNotFoundError { return nil diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 27b0b5e8..ceb83b19 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -127,10 +127,9 @@ func (s *StepMobile) TapByCV(imagePath string, options ...uixt.ActionOption) *St } // TapByUITypes taps on the target element specified by uiTypes, the higher the uiTypes, the higher the priority -func (s *StepMobile) TapByUITypes(uiTypes []string, options ...uixt.ActionOption) *StepMobile { +func (s *StepMobile) TapByUITypes(options ...uixt.ActionOption) *StepMobile { action := uixt.MobileAction{ Method: uixt.ACTION_TapByCV, - Params: uiTypes, Options: uixt.NewActionOptions(options...), }