From f84f3931301cd1ab7d6d5d8bd35a6a59366f0fc0 Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Wed, 7 Jun 2023 23:02:48 +0800 Subject: [PATCH] change tap_cv params to []string --- hrp/pkg/uixt/algorithm.go | 4 ++-- hrp/pkg/uixt/ext.go | 3 +++ hrp/pkg/uixt/tap.go | 39 ++++++++++++++++++++++++++++------- hrp/pkg/uixt/ui_vedem.go | 36 ++++++++++++++++++-------------- hrp/pkg/uixt/ui_vedem_test.go | 6 +++--- hrp/step_mobile_ui.go | 13 ++++++++++++ 6 files changed, 73 insertions(+), 28 deletions(-) diff --git a/hrp/pkg/uixt/algorithm.go b/hrp/pkg/uixt/algorithm.go index 21807bc3..ac6c11c5 100644 --- a/hrp/pkg/uixt/algorithm.go +++ b/hrp/pkg/uixt/algorithm.go @@ -192,7 +192,7 @@ func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOpt return } -func (dExt *DriverExt) FindDetectUIRectInUIKit(uiName string, options ...DataOption) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindDetectUIRectInUIKit(uiTypes []string, options ...DataOption) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.TakeScreenShotAfterAction(); err != nil { return 0, 0, 0, 0, err @@ -203,7 +203,7 @@ func (dExt *DriverExt) FindDetectUIRectInUIKit(uiName string, options ...DataOpt return 0, 0, 0, 0, err } var rect image.Rectangle - rect, err = service.FindUI(uiName, bufSource.Bytes()) + rect, err = service.FindUI(uiTypes, bufSource.Bytes()) if err != nil { return 0, 0, 0, 0, err } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index f2966bb3..44a5e566 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -616,6 +616,9 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { if imagePath, ok := action.Params.(string); ok { return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) } + if uiTypes, ok := action.Params.([]string); ok { + return dExt.TapByUIDetection(uiTypes, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) + } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: if location, ok := action.Params.([]interface{}); ok { diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 5898ba7f..33e4465d 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -63,18 +63,29 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin return points, nil } -func (dExt *DriverExt) GetImageXY(imageParam string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) { // close popup if necessary if dExt.ClosePopup { dExt.ClosePopupHandler() } - var x, y, width, height float64 - switch imageParam { - case ShoppingBag, DyHouse: - x, y, width, height, err = dExt.FindDetectUIRectInUIKit(imageParam, options...) - default: - x, y, width, height, err = dExt.FindImageRectInUIKit(imageParam, options...) + x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...) + if err != nil { + return PointF{}, err } + + point = PointF{ + X: x + width*0.5, + Y: y + height*0.5, + } + return point, nil +} + +func (dExt *DriverExt) GetUIXY(uiTypes []string, options ...DataOption) (point PointF, err error) { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } + x, y, width, height, err := dExt.FindDetectUIRectInUIKit(uiTypes, options...) if err != nil { return PointF{}, err } @@ -114,6 +125,20 @@ func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { return dExt.TapAbsXY(point.X, point.Y, options...) } +func (dExt *DriverExt) TapByUIDetection(uiTypes []string, options ...DataOption) error { + dataOptions := NewDataOptions(options...) + + point, err := dExt.GetUIXY(uiTypes, options...) + if err != nil { + if dataOptions.IgnoreNotFoundError { + return nil + } + return err + } + + return dExt.TapAbsXY(point.X, point.Y, options...) +} + func (dExt *DriverExt) Tap(param string, options ...DataOption) error { return dExt.TapOffset(param, 0.5, 0.5, options...) } diff --git a/hrp/pkg/uixt/ui_vedem.go b/hrp/pkg/uixt/ui_vedem.go index a4aff04e..e76390d0 100644 --- a/hrp/pkg/uixt/ui_vedem.go +++ b/hrp/pkg/uixt/ui_vedem.go @@ -18,17 +18,12 @@ import ( "github.com/httprunner/httprunner/v4/hrp/internal/json" ) -const ( - ShoppingBag = "shoppingbag" - DyHouse = "dyhouse" -) - -type UIResult map[string][]Box +type UIResultMap map[string][]Box type UIResponse struct { - Code int `json:"code"` - Message string `json:"message"` - Result UIResult `json:"result"` + Code int `json:"code"` + Message string `json:"message"` + Result UIResultMap `json:"result"` } type veDEMUIService struct{} @@ -53,10 +48,12 @@ func checkUIEnv() error { return nil } -func (s *veDEMUIService) getUIResult(uiType string, sourceImage []byte) (UIResult, error) { +func (s *veDEMUIService) getUIResult(uiTypes []string, sourceImage []byte) (UIResultMap, error) { bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) - bodyWriter.WriteField("types", uiType) + for _, uiType := range uiTypes { + bodyWriter.WriteField("types", uiType) + } formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") if err != nil { @@ -136,18 +133,25 @@ func (s *veDEMUIService) getUIResult(uiType string, sourceImage []byte) (UIResul return uiResult.Result, nil } -func (s *veDEMUIService) FindUI(uiType string, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { +func (s *veDEMUIService) FindUI(uiTypes []string, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { data := NewDataOptions(options...) - uiResultMap, err := s.getUIResult(uiType, byteSource) + uiResultMap, err := s.getUIResult(uiTypes, byteSource) if err != nil { log.Error().Err(err).Msg("getUIResult failed") return } - uiResult, ok := uiResultMap[uiType] - if !ok { - err = fmt.Errorf("UI type %v not detected", uiResult) + var uiResult []Box + var ok bool + for _, uiType := range uiTypes { + uiResult, ok = uiResultMap[uiType] + if ok && len(uiResult) != 0 { + break + } + } + if len(uiResult) == 0 { + err = fmt.Errorf("UI types %v not detected", uiTypes) log.Error().Err(err).Msg("getUIResult failed") return } diff --git a/hrp/pkg/uixt/ui_vedem_test.go b/hrp/pkg/uixt/ui_vedem_test.go index f373ed80..a9832f05 100644 --- a/hrp/pkg/uixt/ui_vedem_test.go +++ b/hrp/pkg/uixt/ui_vedem_test.go @@ -6,12 +6,12 @@ import ( "testing" ) -func checkUI(uiName string, source []byte) error { +func checkUI(uiTypes []string, source []byte) error { service, err := newVEDEMUIService() if err != nil { return err } - uiResults, err := service.getUIResult(uiName, source) + uiResults, err := service.FindUI(uiTypes, source) if err != nil { return err } @@ -26,7 +26,7 @@ func TestUIWithLocalFile(t *testing.T) { t.Fatal(err) } - if err := checkUI("dyhouse", file); err != nil { + if err := checkUI([]string{"dyhouse", "shoppingbag"}, file); err != nil { t.Fatal(err) } } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 9e489d48..34477a39 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -131,6 +131,19 @@ func (s *StepMobile) TapByCV(imagePath string, options ...uixt.ActionOption) *St return &StepMobile{step: s.step} } +// Tap taps on the target element by UI Detection +func (s *StepMobile) TapByUI(uiTypes []string, options ...uixt.ActionOption) *StepMobile { + action := uixt.MobileAction{ + Method: uixt.ACTION_TapByCV, + Params: uiTypes, + } + for _, option := range options { + option(&action) + } + s.mobileStep().Actions = append(s.mobileStep().Actions, action) + return &StepMobile{step: s.step} +} + // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates func (s *StepMobile) DoubleTapXY(x, y float64) *StepMobile { s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{