feat: TapByCV WithTapRandomRect

internal/version/VERSION

@@ -1 +1 @@
-v5.0.0-beta-2503171757
+v5.0.0-beta-2503171806

uixt/ai/cv.go

@@ -249,6 +249,15 @@ func (box Box) Center() PointF {
 	}
 }
 
+func (box Box) RandomPoint() PointF {
+	width, height := float64(box.Width), float64(box.Height)
+	point := PointF{
+		X: box.Point.X + width*rand.Float64(),
+		Y: box.Point.Y + height*rand.Float64(),
+	}
+	return point
+}
+
 type UIResults []UIResult
 
 func (u UIResults) FilterScope(scope option.AbsScope) (results UIResults) {

uixt/driver_ext_screenshot.go

@@ -166,7 +166,7 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (
 	return textRect, nil
 }
 
-func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) {
+func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (uiResult ai.UIResult, err error) {
 	options := option.NewActionOptions(opts...)
 	if options.ScreenShotFileName == "" {
 		opts = append(opts, option.WithScreenShotFileName(
@@ -182,11 +182,10 @@ func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF
 	if err != nil {
 		return
 	}
-	uiResult, err := uiResults.GetUIResult(opts...)
+	uiResult, err = uiResults.GetUIResult(opts...)
-	point = uiResult.Center()
 
 	log.Info().Interface("text", options.ScreenShotWithUITypes).
-		Interface("point", point).Msg("FindUIResult success")
+		Interface("uiResult", uiResult).Msg("FindUIResult success")
 	return
 }
 

uixt/driver_ext_tap.go

@@ -32,15 +32,22 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error {
 }
 
 func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error {
-	options := option.NewActionOptions(opts...)
+	actionOptions := option.NewActionOptions(opts...)
 
-	point, err := dExt.FindUIResult(opts...)
+	uiResult, err := dExt.FindUIResult(opts...)
 	if err != nil {
-		if options.IgnoreNotFoundError {
+		if actionOptions.IgnoreNotFoundError {
 			return nil
 		}
 		return err
 	}
 
+	var point ai.PointF
+	if actionOptions.TapRandomRect {
+		point = uiResult.RandomPoint()
+	} else {
+		point = uiResult.Center()
+	}
+
 	return dExt.TapAbsXY(point.X, point.Y, opts...)
 }

uixt/driver_ext_test.go

@@ -193,4 +193,10 @@ func TestDriverExt_Action_Risk(t *testing.T) {
 
 	// tap random point in ocr text rect
 	err = driver.TapByOCR("首页", option.WithTapRandomRect(true))
+	assert.Nil(t, err)
+
+	err = driver.TapByCV(
+		option.WithScreenShotUITypes("deepseek_send"),
+		option.WithTapRandomRect(true))
+	assert.Nil(t, err)
 }