change tap_cv params to []string

hrp/pkg/uixt/algorithm.go

@@ -192,7 +192,7 @@ func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOpt
 	return
 }
 
-func (dExt *DriverExt) FindDetectUIRectInUIKit(uiName string, options ...DataOption) (x, y, width, height float64, err error) {
+func (dExt *DriverExt) FindDetectUIRectInUIKit(uiTypes []string, options ...DataOption) (x, y, width, height float64, err error) {
 	var bufSource *bytes.Buffer
 	if bufSource, err = dExt.TakeScreenShotAfterAction(); err != nil {
 		return 0, 0, 0, 0, err
@@ -203,7 +203,7 @@ func (dExt *DriverExt) FindDetectUIRectInUIKit(uiName string, options ...DataOpt
 		return 0, 0, 0, 0, err
 	}
 	var rect image.Rectangle
-	rect, err = service.FindUI(uiName, bufSource.Bytes())
+	rect, err = service.FindUI(uiTypes, bufSource.Bytes())
 	if err != nil {
 		return 0, 0, 0, 0, err
 	}

hrp/pkg/uixt/ext.go

@@ -616,6 +616,9 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		if imagePath, ok := action.Params.(string); ok {
 			return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
 		}
+		if uiTypes, ok := action.Params.([]string); ok {
+			return dExt.TapByUIDetection(uiTypes, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
+		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
 	case ACTION_DoubleTapXY:
 		if location, ok := action.Params.([]interface{}); ok {

hrp/pkg/uixt/tap.go

@@ -63,18 +63,29 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin
 	return points, nil
 }
 
-func (dExt *DriverExt) GetImageXY(imageParam string, options ...DataOption) (point PointF, err error) {
+func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) {
 	// close popup if necessary
 	if dExt.ClosePopup {
 		dExt.ClosePopupHandler()
 	}
-	var x, y, width, height float64
+	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...)
-	switch imageParam {
+	if err != nil {
-	case ShoppingBag, DyHouse:
+		return PointF{}, err
-		x, y, width, height, err = dExt.FindDetectUIRectInUIKit(imageParam, options...)
-	default:
-		x, y, width, height, err = dExt.FindImageRectInUIKit(imageParam, options...)
 	}
+
+	point = PointF{
+		X: x + width*0.5,
+		Y: y + height*0.5,
+	}
+	return point, nil
+}
+
+func (dExt *DriverExt) GetUIXY(uiTypes []string, options ...DataOption) (point PointF, err error) {
+	// close popup if necessary
+	if dExt.ClosePopup {
+		dExt.ClosePopupHandler()
+	}
+	x, y, width, height, err := dExt.FindDetectUIRectInUIKit(uiTypes, options...)
 	if err != nil {
 		return PointF{}, err
 	}
@@ -114,6 +125,20 @@ func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error {
 	return dExt.TapAbsXY(point.X, point.Y, options...)
 }
 
+func (dExt *DriverExt) TapByUIDetection(uiTypes []string, options ...DataOption) error {
+	dataOptions := NewDataOptions(options...)
+
+	point, err := dExt.GetUIXY(uiTypes, options...)
+	if err != nil {
+		if dataOptions.IgnoreNotFoundError {
+			return nil
+		}
+		return err
+	}
+
+	return dExt.TapAbsXY(point.X, point.Y, options...)
+}
+
 func (dExt *DriverExt) Tap(param string, options ...DataOption) error {
 	return dExt.TapOffset(param, 0.5, 0.5, options...)
 }

hrp/pkg/uixt/ui_vedem.go

@@ -18,17 +18,12 @@ import (
 	"github.com/httprunner/httprunner/v4/hrp/internal/json"
 )
 
-const (
+type UIResultMap map[string][]Box
-	ShoppingBag = "shoppingbag"
-	DyHouse     = "dyhouse"
-)
-
-type UIResult map[string][]Box
 
 type UIResponse struct {
-	Code    int      `json:"code"`
+	Code    int         `json:"code"`
-	Message string   `json:"message"`
+	Message string      `json:"message"`
-	Result  UIResult `json:"result"`
+	Result  UIResultMap `json:"result"`
 }
 
 type veDEMUIService struct{}
@@ -53,10 +48,12 @@ func checkUIEnv() error {
 	return nil
 }
 
-func (s *veDEMUIService) getUIResult(uiType string, sourceImage []byte) (UIResult, error) {
+func (s *veDEMUIService) getUIResult(uiTypes []string, sourceImage []byte) (UIResultMap, error) {
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
-	bodyWriter.WriteField("types", uiType)
+	for _, uiType := range uiTypes {
+		bodyWriter.WriteField("types", uiType)
+	}
 
 	formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png")
 	if err != nil {
@@ -136,18 +133,25 @@ func (s *veDEMUIService) getUIResult(uiType string, sourceImage []byte) (UIResul
 	return uiResult.Result, nil
 }
 
-func (s *veDEMUIService) FindUI(uiType string, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
+func (s *veDEMUIService) FindUI(uiTypes []string, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
 	data := NewDataOptions(options...)
 
-	uiResultMap, err := s.getUIResult(uiType, byteSource)
+	uiResultMap, err := s.getUIResult(uiTypes, byteSource)
 	if err != nil {
 		log.Error().Err(err).Msg("getUIResult failed")
 		return
 	}
 
-	uiResult, ok := uiResultMap[uiType]
+	var uiResult []Box
-	if !ok {
+	var ok bool
-		err = fmt.Errorf("UI type %v not detected", uiResult)
+	for _, uiType := range uiTypes {
+		uiResult, ok = uiResultMap[uiType]
+		if ok && len(uiResult) != 0 {
+			break
+		}
+	}
+	if len(uiResult) == 0 {
+		err = fmt.Errorf("UI types %v not detected", uiTypes)
 		log.Error().Err(err).Msg("getUIResult failed")
 		return
 	}

hrp/pkg/uixt/ui_vedem_test.go

@@ -6,12 +6,12 @@ import (
 	"testing"
 )
 
-func checkUI(uiName string, source []byte) error {
+func checkUI(uiTypes []string, source []byte) error {
 	service, err := newVEDEMUIService()
 	if err != nil {
 		return err
 	}
-	uiResults, err := service.getUIResult(uiName, source)
+	uiResults, err := service.FindUI(uiTypes, source)
 	if err != nil {
 		return err
 	}
@@ -26,7 +26,7 @@ func TestUIWithLocalFile(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := checkUI("dyhouse", file); err != nil {
+	if err := checkUI([]string{"dyhouse", "shoppingbag"}, file); err != nil {
 		t.Fatal(err)
 	}
 }

hrp/step_mobile_ui.go

@@ -131,6 +131,19 @@ func (s *StepMobile) TapByCV(imagePath string, options ...uixt.ActionOption) *St
 	return &StepMobile{step: s.step}
 }
 
+// Tap taps on the target element by UI Detection
+func (s *StepMobile) TapByUI(uiTypes []string, options ...uixt.ActionOption) *StepMobile {
+	action := uixt.MobileAction{
+		Method: uixt.ACTION_TapByCV,
+		Params: uiTypes,
+	}
+	for _, option := range options {
+		option(&action)
+	}
+	s.mobileStep().Actions = append(s.mobileStep().Actions, action)
+	return &StepMobile{step: s.step}
+}
+
 // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates
 func (s *StepMobile) DoubleTapXY(x, y float64) *StepMobile {
 	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{