merge master

docs/CHANGELOG.md

@@ -26,6 +26,7 @@ UI related:
 - refactor: ui validation methods, support parsing expect value
 - fix: reuse the same request body during `GetImage` retry
 - fix: iOS `tap_xy` scale adaption error
+- feat: support new action: `close_popups`
 
 others:
 

examples/uitest/android_expert_test.json

@@ -59,15 +59,12 @@
             }
         },
         {
-            "name": "处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言",
+            "name": "处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言",
             "android": {
                 "actions": [
                     {
-                        "method": "tap_ocr",
+                        "method": "close_popups",
-                        "params": "我知道了",
+                        "options": {}
-                        "options": {
-                            "ignore_NotFoundError": true
-                        }
                     }
                 ]
             },
@@ -317,14 +314,14 @@
             }
         },
         {
-            "name": "处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言",
+            "name": "处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言",
             "android": {
                 "actions": [
                     {
-                        "method": "tap_ocr",
+                        "method": "close_popups",
-                        "params": "我知道了",
                         "options": {
-                            "ignore_NotFoundError": true
+                            "max_retry_times": 3,
+                            "interval": 2
                         }
                     }
                 ]

examples/uitest/expert_test.go

@@ -1,5 +1,3 @@
-//go:build localtest
-
 package uitest
 
 import (
@@ -36,12 +34,9 @@ func TestAndroidExpertTest(t *testing.T) {
 				Home().
 				SwipeToTapApp("$app_name").
 				Sleep(10),
-			hrp.NewStep("处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言").
+			hrp.NewStep("处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言").
 				Android().
-				TapByOCR(
+				ClosePopups().
-					"我知道了",
-					uixt.WithIgnoreNotFoundError(true),
-				).
 				Validate().
 				AssertOCRExists("推荐", "进入抖音失败"),
 			// 直播赛道
@@ -118,11 +113,11 @@ func TestAndroidExpertTest(t *testing.T) {
 				Home().
 				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)).
 				Sleep(10),
-			hrp.NewStep("处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言").
+			hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言").
 				Android().
-				TapByOCR(
+				ClosePopups(
-					"我知道了",
+					uixt.WithMaxRetryTimes(3),
-					uixt.WithIgnoreNotFoundError(true),
+					uixt.WithInterval(2),
 				).
 				Validate().
 				AssertOCRExists("推荐", "进入抖音失败"),
@@ -194,12 +189,9 @@ func TestIOSExpertTest(t *testing.T) {
 				Home().
 				SwipeToTapApp("$app_name").
 				Sleep(10),
-			hrp.NewStep("处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言").
+			hrp.NewStep("处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言").
 				IOS().
-				TapByOCR(
+				ClosePopups().
-					"我知道了",
-					uixt.WithIgnoreNotFoundError(true),
-				).
 				Validate().
 				AssertOCRExists("推荐", "进入抖音失败"),
 			// 直播赛道
@@ -275,11 +267,11 @@ func TestIOSExpertTest(t *testing.T) {
 				Home().
 				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)).
 				Sleep(10),
-			hrp.NewStep("处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言").
+			hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言").
 				IOS().
-				TapByOCR(
+				ClosePopups(
-					"我知道了",
+					uixt.WithMaxRetryTimes(3),
-					uixt.WithIgnoreNotFoundError(true),
+					uixt.WithInterval(2),
 				).
 				Validate().
 				AssertOCRExists("推荐", "进入抖音失败"),

examples/uitest/ios_expert_test.json

@@ -52,15 +52,12 @@
             }
         },
         {
-            "name": "处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言",
+            "name": "处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言",
             "ios": {
                 "actions": [
                     {
-                        "method": "tap_ocr",
+                        "method": "close_popups",
-                        "params": "我知道了",
+                        "options": {}
-                        "options": {
-                            "ignore_NotFoundError": true
-                        }
                     }
                 ]
             },
@@ -302,14 +299,14 @@
             }
         },
         {
-            "name": "处理青少年弹窗 tap ocr 以及 ui_ocr exists 断言",
+            "name": "处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言",
             "ios": {
                 "actions": [
                     {
-                        "method": "tap_ocr",
+                        "method": "close_popups",
-                        "params": "我知道了",
                         "options": {
-                            "ignore_NotFoundError": true
+                            "max_retry_times": 3,
+                            "interval": 2
                         }
                     }
                 ]

hrp/pkg/uixt/action.go

@@ -58,6 +58,7 @@ const (
 	ACTION_SwipeToTapText  ActionMethod = "swipe_to_tap_text"  // swipe up & down to find text and tap
 	ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap
 	ACTION_VideoCrawler    ActionMethod = "video_crawler"
+	ACTION_ClosePopups     ActionMethod = "close_popups"
 )
 
 type MobileAction struct {
@@ -105,18 +106,20 @@ type ActionOptions struct {
 	Scope    Scope    `json:"scope,omitempty" yaml:"scope,omitempty"`
 	AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"`
 
-	Regex  bool  `json:"regex,omitempty" yaml:"regex,omitempty"`   // use regex to match text
+	Regex    bool  `json:"regex,omitempty" yaml:"regex,omitempty"`         // use regex to match text
-	Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point
+	Offset   []int `json:"offset,omitempty" yaml:"offset,omitempty"`       // used to tap offset of point
-	Index  int   `json:"index,omitempty" yaml:"index,omitempty"`   // index of the target element
+	Index    int   `json:"index,omitempty" yaml:"index,omitempty"`         // index of the target element
+	MatchOne bool  `json:"match_one,omitempty" yaml:"match_one,omitempty"` // match one of the targets if existed
 
 	// set custiom options such as textview, id, description
 	Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"`
 
 	// screenshot related
-	ScreenShotWithOCR      bool     `json:"screenshot_with_ocr,omitempty" yaml:"screenshot_with_ocr,omitempty"`
+	ScreenShotWithOCR         bool     `json:"screenshot_with_ocr,omitempty" yaml:"screenshot_with_ocr,omitempty"`
-	ScreenShotWithUpload   bool     `json:"screenshot_with_upload,omitempty" yaml:"screenshot_with_upload,omitempty"`
+	ScreenShotWithUpload      bool     `json:"screenshot_with_upload,omitempty" yaml:"screenshot_with_upload,omitempty"`
-	ScreenShotWithLiveType bool     `json:"screenshot_with_live_type,omitempty" yaml:"screenshot_with_live_type,omitempty"`
+	ScreenShotWithLiveType    bool     `json:"screenshot_with_live_type,omitempty" yaml:"screenshot_with_live_type,omitempty"`
-	ScreenShotWithUITypes  []string `json:"screenshot_with_ui_types,omitempty" yaml:"screenshot_with_ui_types,omitempty"`
+	ScreenShotWithUITypes     []string `json:"screenshot_with_ui_types,omitempty" yaml:"screenshot_with_ui_types,omitempty"`
+	ScreenShotWithClosePopups bool     `json:"screenshot_with_close_popups,omitempty" yaml:"screenshot_with_close_popups,omitempty"`
 }
 
 func (o *ActionOptions) Options() []ActionOption {
@@ -183,6 +186,12 @@ func (o *ActionOptions) Options() []ActionOption {
 	if o.Regex {
 		options = append(options, WithRegex(true))
 	}
+	if o.Index != 0 {
+		options = append(options, WithIndex(o.Index))
+	}
+	if o.MatchOne {
+		options = append(options, WithMatchOne(true))
+	}
 
 	// custom options
 	if o.Custom != nil {
@@ -210,12 +219,12 @@ func (o *ActionOptions) Options() []ActionOption {
 
 func (o *ActionOptions) screenshotActions() []string {
 	actions := []string{}
-	if o.ScreenShotWithOCR {
-		actions = append(actions, "ocr")
-	}
 	if o.ScreenShotWithUpload {
 		actions = append(actions, "upload")
 	}
+	if o.ScreenShotWithOCR {
+		actions = append(actions, "ocr")
+	}
 	if o.ScreenShotWithLiveType {
 		actions = append(actions, "liveType")
 	}
@@ -223,6 +232,9 @@ func (o *ActionOptions) screenshotActions() []string {
 	if len(o.ScreenShotWithUITypes) > 0 {
 		actions = append(actions, "ui")
 	}
+	if o.ScreenShotWithClosePopups {
+		actions = append(actions, "close")
+	}
 	return actions
 }
 
@@ -377,6 +389,12 @@ func WithRegex(regex bool) ActionOption {
 	}
 }
 
+func WithMatchOne(matchOne bool) ActionOption {
+	return func(o *ActionOptions) {
+		o.MatchOne = matchOne
+	}
+}
+
 func WithFrequency(frequency int) ActionOption {
 	return func(o *ActionOptions) {
 		o.Frequency = frequency
@@ -425,6 +443,12 @@ func WithScreenShotUITypes(uiTypes ...string) ActionOption {
 	}
 }
 
+func WithScreenShotClosePopups(closeOn bool) ActionOption {
+	return func(o *ActionOptions) {
+		o.ScreenShotWithClosePopups = closeOn
+	}
+}
+
 func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption {
 	actionOptions := NewActionOptions(options...)
 
@@ -496,8 +520,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) {
 		if texts, ok := action.Params.([]string); ok {
 			return dExt.swipeToTapTexts(texts, action.GetOptions()...)
 		}
-		return fmt.Errorf("invalid %s params, should be app text([]string), got %v",
+		if texts, err := convertToStringSlice(action.Params); err == nil {
-			ACTION_SwipeToTapText, action.Params)
+			return dExt.swipeToTapTexts(texts, action.GetOptions()...)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_SwipeToTapTexts, action.Params)
 	case ACTION_AppTerminate:
 		if bundleId, ok := action.Params.(string); ok {
 			success, err := dExt.Driver.AppTerminate(bundleId)
@@ -545,10 +571,11 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) {
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
 	case ACTION_TapByCV:
+		actionOptions := NewActionOptions(action.GetOptions()...)
 		if imagePath, ok := action.Params.(string); ok {
 			return dExt.TapByCV(imagePath, action.GetOptions()...)
-		} else if err := dExt.TapByUIDetection(action.GetOptions()...); err == nil {
+		} else if len(actionOptions.ScreenShotWithUITypes) > 0 {
-			return nil
+			return dExt.TapByUIDetection(action.GetOptions()...)
 		}
 		return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
 	case ACTION_DoubleTapXY:
@@ -617,6 +644,8 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) {
 			return errors.Wrapf(err, "invalid video crawler params: %v(%T)", action.Params, action.Params)
 		}
 		return dExt.VideoCrawler(configs)
+	case ACTION_ClosePopups:
+		return dExt.ClosePopups(action.GetOptions()...)
 	}
 	return nil
 }
@@ -636,6 +665,21 @@ func convertToFloat64(val interface{}) (float64, error) {
 	}
 }
 
+func convertToStringSlice(val interface{}) ([]string, error) {
+	if valSlice, ok := val.([]interface{}); ok {
+		var res []string
+		for _, iVal := range valSlice {
+			valString, ok := iVal.(string)
+			if !ok {
+				return nil, fmt.Errorf("invalid type for converting one of the elements to string: %T, value: %v", iVal, iVal)
+			}
+			res = append(res, valString)
+		}
+		return res, nil
+	}
+	return nil, fmt.Errorf("invalid type for conversion to []string")
+}
+
 // getSimulationDuration returns simulation duration by given params (in seconds)
 func getSimulationDuration(params []interface{}) (milliseconds int64) {
 	if len(params) == 1 {

hrp/pkg/uixt/ext.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"sort"
 	"strings"
 	"syscall"
 	"time"
@@ -61,6 +62,7 @@ type ScreenResult struct {
 	VideoType   string      `json:"video_type,omitempty"` // video type: feed, live-preview or live
 	Feed        *FeedVideo  `json:"feed,omitempty"`
 	Live        *LiveRoom   `json:"live,omitempty"`
+	Popup       *PopupInfo  `json:"popup,omitempty"`
 
 	SwipeStartTime  int64 `json:"swipe_start_time"`  // 滑动开始时间戳
 	SwipeFinishTime int64 `json:"swipe_finish_time"` // 滑动结束时间戳
@@ -72,11 +74,59 @@ type ScreenResult struct {
 	TotalElapsed int64 `json:"total_elapsed"` // current_swipe_finish -> next_swipe_start 整体耗时(ms)
 }
 
+type ScreenResultMap map[string]*ScreenResult
+
+// getScreenShotUrls returns screenShotsUrls using imagePath as key and uploaded URL as value
+func (screenResults ScreenResultMap) getScreenShotUrls() map[string]string {
+	screenShotsUrls := make(map[string]string)
+	for imagePath, screenResult := range screenResults {
+		if screenResult.UploadedURL == "" {
+			continue
+		}
+		screenShotsUrls[imagePath] = screenResult.UploadedURL
+	}
+	return screenShotsUrls
+}
+
+// updatePopupCloseStatus checks if popup closed normally in every screenResult with close_popups on:
+func (screenResults ScreenResultMap) updatePopupCloseStatus() {
+	var popupScreenResultList []*ScreenResult
+	for _, screenResult := range screenResults {
+		if screenResult.Popup == nil {
+			continue
+		}
+		popupScreenResultList = append(popupScreenResultList, screenResult)
+	}
+	if len(popupScreenResultList) == 0 {
+		return
+	}
+	sort.Slice(popupScreenResultList, func(i, j int) bool {
+		return popupScreenResultList[i].Popup.RetryCount < popupScreenResultList[j].Popup.RetryCount
+	})
+
+	for i := 0; i < len(popupScreenResultList)-1; i++ {
+		curPopup := popupScreenResultList[i].Popup
+		nextPopup := popupScreenResultList[i+1].Popup
+
+		// popup not existed, no need to close
+		if curPopup.CloseArea.IsEmpty() {
+			continue
+		}
+		// popup existed, but identical popups occurs during next retry
+		if curPopup.Text == nextPopup.Text && curPopup.Type == nextPopup.Type {
+			popupScreenResultList[i].Popup.CloseStatus = "fail"
+			continue
+		}
+		// popup existed, but no popup or different popup occurs during next retry (IsClosed=true)
+		popupScreenResultList[i].Popup.CloseStatus = "success"
+	}
+}
+
 type cacheStepData struct {
 	// cache step screenshot paths
 	screenShots []string
 	// cache step screenshot ocr results, key is image path, value is ScreenResult
-	screenResults map[string]*ScreenResult
+	screenResults ScreenResultMap
 	// cache feed/live video stat
 	videoCrawler *VideoCrawler
 }
@@ -223,14 +273,8 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} {
 	cacheData["video_stat"] = dExt.cacheStepData.videoCrawler
 	cacheData["screenshots"] = dExt.cacheStepData.screenShots
 
-	screenShotsUrls := make(map[string]string)
+	cacheData["screenshots_urls"] = dExt.cacheStepData.screenResults.getScreenShotUrls()
-	for imagePath, screenResult := range dExt.cacheStepData.screenResults {
+	dExt.cacheStepData.screenResults.updatePopupCloseStatus()
-		if screenResult.UploadedURL == "" {
-			continue
-		}
-		screenShotsUrls[imagePath] = screenResult.UploadedURL
-	}
-	cacheData["screenshots_urls"] = screenShotsUrls
 
 	screenSize, err := dExt.Driver.WindowSize()
 	if err != nil {
@@ -255,6 +299,8 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} {
 			"screenshot_take_elapsed": screenResult.ScreenshotTakeElapsed,
 			"screenshot_cv_elapsed":   screenResult.ScreenshotCVElapsed,
 			"total_elapsed":           screenResult.TotalElapsed,
+			"icons":                   screenResult.Icons,
+			"popup":                   screenResult.Popup,
 		}
 
 		screenResults[imagePath] = data

hrp/pkg/uixt/ios_driver.go

@@ -201,6 +201,12 @@ func (wd *wdaDriver) WindowSize() (size Size, err error) {
 		return Size{}, err
 	}
 	size = reply.Value.Size
+	scale, err := wd.Scale()
+	if err != nil {
+		return Size{}, errors.Wrap(err, "get window size scale failed")
+	}
+	size.Height = size.Height * int(scale)
+	size.Width = size.Width * int(scale)
 	return
 }
 

hrp/pkg/uixt/popups.go

@@ -1,9 +1,12 @@
 package uixt
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
+	"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
 )
 
@@ -70,3 +73,85 @@ func (dExt *DriverExt) AutoPopupHandler() error {
 
 	return dExt.handleTextPopup(screenResult.Texts)
 }
+
+// ClosePopupsResult represents the result of recognized popup to close
+type ClosePopupsResult struct {
+	Type      string `json:"type"`
+	PopupArea Box    `json:"popupArea"`
+	CloseArea Box    `json:"closeArea"`
+	Text      string `json:"text"`
+}
+
+type PopupInfo struct {
+	CloseStatus string `json:"close_status"`
+	Type        string `json:"type"`
+	Text        string `json:"text"`
+	RetryCount  int    `json:"retry_count"`
+	PicName     string `json:"pic_name"`
+	PicURL      string `json:"pic_url"`
+	PopupArea   Box    `json:"popup_area"`
+	CloseArea   Box    `json:"close_area"`
+}
+
+func (dExt *DriverExt) ClosePopups(options ...ActionOption) error {
+	actionOptions := NewActionOptions(options...)
+
+	// default to retry 5 times
+	if actionOptions.MaxRetryTimes == 0 {
+		options = append(options, WithMaxRetryTimes(5))
+	}
+	// set default swipe interval to 1 second
+	if builtin.IsZeroFloat64(actionOptions.Interval) {
+		options = append(options, WithInterval(1))
+	}
+	return dExt.ClosePopupsHandler(options...)
+}
+
+func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error {
+	actionOptions := NewActionOptions(options...)
+	maxRetryTimes := actionOptions.MaxRetryTimes
+	interval := actionOptions.Interval
+
+	for retryCount := 0; retryCount < maxRetryTimes; retryCount++ {
+		screenResult, err := dExt.GetScreenResult(
+			WithScreenShotClosePopups(true), WithScreenShotUpload(true))
+		if err != nil {
+			log.Error().Err(err).Msg("get screen result failed for popup handler")
+			continue
+		}
+		// 1. there are no popups here (fast return normally)
+		// 2. failed to close popup （maybe tap error, return error）
+		// 3. successful to close popup (sleep and wait for next retry if existed)
+		if screenResult.Popup == nil {
+			break
+		}
+		screenResult.Popup.RetryCount = retryCount
+		if screenResult.Popup.CloseArea.IsEmpty() {
+			break
+		}
+
+		if err = dExt.tapPopupHandler(screenResult.Popup); err != nil {
+			return err
+		}
+		// sleep for another popup (if existed) to pop
+		time.Sleep(time.Duration(1000*interval) * time.Millisecond)
+	}
+	return nil
+}
+
+func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error {
+	if popup == nil {
+		return nil
+	}
+	if popup.CloseArea.IsEmpty() {
+		return nil
+	}
+	log.Info().Str("type", popup.Type).Str("text", popup.Text).Msg("close popup")
+	popupCenter := popup.CloseArea.Center()
+	if err := dExt.TapAbsXY(popupCenter.X, popupCenter.Y); err != nil {
+		log.Error().Err(err).Msg("tap popup failed")
+		return errors.Wrap(code.MobileUIPopupError, err.Error())
+	}
+	// tap popup success
+	return nil
+}

hrp/pkg/uixt/service_vedem.go

@@ -8,6 +8,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -65,8 +66,9 @@ type ImageResult struct {
 	// Media（媒体）
 	// Chat（语音）
 	// Event（赛事）
-	LiveType string      `json:"liveType"` // 直播间类型
+	LiveType string            `json:"liveType"`    // 直播间类型
-	UIResult UIResultMap `json:"uiResult"` // 图标检测
+	UIResult UIResultMap       `json:"uiResult"`    // 图标检测
+	CPResult ClosePopupsResult `json:"closeResult"` // 弹窗按钮检测
 }
 
 type APIResponseImage struct {
@@ -154,6 +156,7 @@ func (t OCRTexts) FindText(text string, options ...ActionOption) (result OCRText
 }
 
 func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (results OCRTexts, err error) {
+	actionOptions := NewActionOptions(options...)
 	for _, text := range texts {
 		ocrText, err := t.FindText(text, options...)
 		if err != nil {
@@ -162,11 +165,16 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (results OC
 		results = append(results, ocrText)
 	}
 
-	if len(results) != len(texts) {
+	if len(results) == len(texts) {
-		return nil, errors.Wrap(code.CVResultNotFoundError,
+		return results, nil
-			fmt.Sprintf("texts %s not found in %v", texts, t.texts()))
 	}
-	return results, nil
+
+	if actionOptions.MatchOne && len(results) > 0 {
+		return results, nil
+	}
+
+	return nil, errors.Wrap(code.CVResultNotFoundError,
+		fmt.Sprintf("texts %s not found in %v", texts, t.texts()))
 }
 
 func newVEDEMImageService() (*veDEMImageService, error) {
@@ -350,6 +358,11 @@ type IImageService interface {
 func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *ScreenResult, err error) {
 	startTime := time.Now()
 	fileName := builtin.GenNameWithTimestamp("%d_screenshot")
+	actionOptions := NewActionOptions(options...)
+	screenshotActions := actionOptions.screenshotActions()
+	if len(screenshotActions) != 0 {
+		fileName = builtin.GenNameWithTimestamp("%d_" + strings.Join(screenshotActions, "_"))
+	}
 	bufSource, imagePath, err := dExt.takeScreenShot(fileName)
 	if err != nil {
 		return
@@ -379,6 +392,17 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
 				LiveType: imageResult.LiveType,
 			}
 		}
+		if actionOptions.ScreenShotWithClosePopups {
+			screenResult.Popup = &PopupInfo{
+				Type:      imageResult.CPResult.Type,
+				Text:      imageResult.CPResult.Text,
+				PicName:   imagePath,
+				PicURL:    imageResult.URL,
+				PopupArea: imageResult.CPResult.PopupArea,
+				CloseArea: imageResult.CPResult.CloseArea,
+			}
+		}
+
 	}
 
 	dExt.cacheStepData.screenResults[imagePath] = screenResult
@@ -429,21 +453,25 @@ func getRectangleCenterPoint(rect image.Rectangle) (point PointF) {
 	return point
 }
 
-func getCenterPoint(point PointF, width, height float64) PointF {
+type Box struct {
-	return PointF{
-		X: point.X + width*0.5,
-		Y: point.Y + height*0.5,
-	}
-}
-
-type UIResult struct {
 	Point  PointF  `json:"point"`
 	Width  float64 `json:"width"`
 	Height float64 `json:"height"`
 }
 
-func (u UIResult) Center() PointF {
+func (box Box) IsEmpty() bool {
-	return getCenterPoint(u.Point, u.Width, u.Height)
+	return builtin.IsZeroFloat64(box.Width) && builtin.IsZeroFloat64(box.Height)
+}
+
+func (box Box) Center() PointF {
+	return PointF{
+		X: box.Point.X + box.Width*0.5,
+		Y: box.Point.Y + box.Height*0.5,
+	}
+}
+
+type UIResult struct {
+	Box
 }
 
 type UIResults []UIResult

hrp/pkg/uixt/service_vedem_test.go

@@ -98,3 +98,25 @@ func TestDriverExtOCR(t *testing.T) {
 	t.Logf("point.X: %v, point.Y: %v", point.X, point.Y)
 	driverExt.Driver.TapFloat(point.X, point.Y-20)
 }
+
+func TestClosePopup(t *testing.T) {
+	setupAndroid(t)
+
+	screenResult, err := driverExt.GetScreenResult(
+		WithScreenShotClosePopups(true), WithScreenShotUpload(true))
+	if err != nil {
+		t.Logf("get screen result failed for popup handler: %v", err)
+		return
+	}
+	t.Logf("screen result: %v", screenResult)
+
+	if screenResult.Popup == nil {
+		t.Log("there are no popups here")
+		return
+	}
+	t.Logf("popup info: %v", screenResult.Popup)
+
+	if err = driverExt.tapPopupHandler(screenResult.Popup); err != nil {
+		t.Fatal(err)
+	}
+}

hrp/pkg/uixt/swipe.go

@@ -7,6 +7,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
+	"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
 )
 
@@ -132,18 +133,23 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
 		return errors.New("no text to tap")
 	}
 
+	options = append(options, WithMatchOne(true))
 	var point PointF
 	findTexts := func(d *DriverExt) error {
 		var err error
-		screenTexts, err := d.GetScreenTexts()
+		screenResult, err := d.GetScreenResult(
+			WithScreenShotOCR(true),
+			WithScreenShotUpload(true),
+			WithScreenShotClosePopups(true),
+		)
 		if err != nil {
 			return err
 		}
-		points, err := screenTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...)
+		points, err := screenResult.Texts.FindTexts(texts, dExt.ParseActionOptions(options...)...)
 		if err != nil {
 			log.Error().Err(err).Msg("swipeToTapTexts failed")
 			// target texts not found, try to auto handle popup
-			if e := dExt.handleTextPopup(screenTexts); e != nil {
+			if e := dExt.tapPopupHandler(screenResult.Popup); e != nil {
 				log.Error().Err(e).Msg("auto handle popup failed")
 			}
 			return err
@@ -187,7 +193,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) er
 		options = append(options, WithOffset(0, -25))
 	}
 	// set default swipe interval to 1 second
-	if actionOptions.Interval == 0 {
+	if builtin.IsZeroFloat64(actionOptions.Interval) {
 		options = append(options, WithInterval(1))
 	}
 

hrp/pkg/uixt/tap.go

@@ -15,12 +15,8 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error {
 		return fmt.Errorf("x, y percentage should be <= 1, got x=%v, y=%v", x, y)
 	}
 
-	scale, err := dExt.Driver.Scale()
+	x = x * float64(dExt.windowSize.Width)
-	if err != nil {
+	y = y * float64(dExt.windowSize.Height)
-		return fmt.Errorf("failed to get scale: %v", err)
-	}
-	x = x * float64(dExt.windowSize.Width) * scale
-	y = y * float64(dExt.windowSize.Height) * scale
 
 	return dExt.TapAbsXY(x, y, options...)
 }

hrp/pkg/uixt/video_crawler.go

@@ -317,7 +317,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			if feedVideo.VideoID == crawler.lastFeed.VideoID {
 				// app event tracking not changed
 				// check and handle popups
-				if err = crawler.driverExt.AutoPopupHandler(); err != nil {
+				if err = crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(1)); err != nil {
 					return err
 				}
 			}

hrp/step_mobile_ui.go

@@ -327,6 +327,15 @@ func (s *StepMobile) StopCamera() *StepMobile {
 	return &StepMobile{step: s.step}
 }
 
+func (s *StepMobile) ClosePopups(options ...uixt.ActionOption) *StepMobile {
+	s.mobileStep().Actions = append(s.mobileStep().Actions, uixt.MobileAction{
+		Method:  uixt.ACTION_ClosePopups,
+		Params:  nil,
+		Options: uixt.NewActionOptions(options...),
+	})
+	return &StepMobile{step: s.step}
+}
+
 // Validate switches to step validation.
 func (s *StepMobile) Validate() *StepMobileUIValidation {
 	return &StepMobileUIValidation{
@@ -615,7 +624,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
 		}
 
 		// automatic handling of pop-up windows on each step finished
-		if err2 := uiDriver.AutoPopupHandler(); err2 != nil {
+		if err2 := uiDriver.ClosePopups(); err2 != nil {
 			log.Error().Err(err2).Str("step", step.Name).Msg("auto handle popup failed")
 		}