From 5e2f33b362942acb38106005e4957b1480d8004e Mon Sep 17 00:00:00 2001
From: "lilong.129" <lilong.129@bytedance.com>
Date: Mon, 17 Mar 2025 18:06:32 +0800
Subject: [PATCH] feat: TapByCV WithTapRandomRect

---
 internal/version/VERSION      |  2 +-
 uixt/ai/cv.go                 |  9 +++++++++
 uixt/driver_ext_screenshot.go |  7 +++----
 uixt/driver_ext_tap.go        | 13 ++++++++++---
 uixt/driver_ext_test.go       |  6 ++++++
 5 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/internal/version/VERSION b/internal/version/VERSION
index a2e325c3..5200bf63 100644
--- a/internal/version/VERSION
+++ b/internal/version/VERSION
@@ -1 +1 @@
-v5.0.0-beta-2503171757
+v5.0.0-beta-2503171806
diff --git a/uixt/ai/cv.go b/uixt/ai/cv.go
index 81b6c196..2fe70839 100644
--- a/uixt/ai/cv.go
+++ b/uixt/ai/cv.go
@@ -249,6 +249,15 @@ func (box Box) Center() PointF {
 	}
 }
 
+func (box Box) RandomPoint() PointF {
+	width, height := float64(box.Width), float64(box.Height)
+	point := PointF{
+		X: box.Point.X + width*rand.Float64(),
+		Y: box.Point.Y + height*rand.Float64(),
+	}
+	return point
+}
+
 type UIResults []UIResult
 
 func (u UIResults) FilterScope(scope option.AbsScope) (results UIResults) {
diff --git a/uixt/driver_ext_screenshot.go b/uixt/driver_ext_screenshot.go
index 854baeb6..674b4a5f 100644
--- a/uixt/driver_ext_screenshot.go
+++ b/uixt/driver_ext_screenshot.go
@@ -166,7 +166,7 @@ func (dExt *XTDriver) FindScreenText(text string, opts ...option.ActionOption) (
 	return textRect, nil
 }
 
-func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF, err error) {
+func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (uiResult ai.UIResult, err error) {
 	options := option.NewActionOptions(opts...)
 	if options.ScreenShotFileName == "" {
 		opts = append(opts, option.WithScreenShotFileName(
@@ -182,11 +182,10 @@ func (dExt *XTDriver) FindUIResult(opts ...option.ActionOption) (point ai.PointF
 	if err != nil {
 		return
 	}
-	uiResult, err := uiResults.GetUIResult(opts...)
-	point = uiResult.Center()
+	uiResult, err = uiResults.GetUIResult(opts...)
 
 	log.Info().Interface("text", options.ScreenShotWithUITypes).
-		Interface("point", point).Msg("FindUIResult success")
+		Interface("uiResult", uiResult).Msg("FindUIResult success")
 	return
 }
 
diff --git a/uixt/driver_ext_tap.go b/uixt/driver_ext_tap.go
index 7283fd18..dd65f50a 100644
--- a/uixt/driver_ext_tap.go
+++ b/uixt/driver_ext_tap.go
@@ -32,15 +32,22 @@ func (dExt *XTDriver) TapByOCR(text string, opts ...option.ActionOption) error {
 }
 
 func (dExt *XTDriver) TapByCV(opts ...option.ActionOption) error {
-	options := option.NewActionOptions(opts...)
+	actionOptions := option.NewActionOptions(opts...)
 
-	point, err := dExt.FindUIResult(opts...)
+	uiResult, err := dExt.FindUIResult(opts...)
 	if err != nil {
-		if options.IgnoreNotFoundError {
+		if actionOptions.IgnoreNotFoundError {
 			return nil
 		}
 		return err
 	}
 
+	var point ai.PointF
+	if actionOptions.TapRandomRect {
+		point = uiResult.RandomPoint()
+	} else {
+		point = uiResult.Center()
+	}
+
 	return dExt.TapAbsXY(point.X, point.Y, opts...)
 }
diff --git a/uixt/driver_ext_test.go b/uixt/driver_ext_test.go
index 7b4f5a37..a5a98eaa 100644
--- a/uixt/driver_ext_test.go
+++ b/uixt/driver_ext_test.go
@@ -193,4 +193,10 @@ func TestDriverExt_Action_Risk(t *testing.T) {
 
 	// tap random point in ocr text rect
 	err = driver.TapByOCR("首页", option.WithTapRandomRect(true))
+	assert.Nil(t, err)
+
+	err = driver.TapByCV(
+		option.WithScreenShotUITypes("deepseek_send"),
+		option.WithTapRandomRect(true))
+	assert.Nil(t, err)
 }