feat: auto handle popups on each step finished

hrp/pkg/uixt/action.go

@@ -506,7 +506,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 	case ACTION_ScreenShot:
 		// take screenshot
 		log.Info().Msg("take screenshot for current screen")
-		_, _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot"))
+		_, _, err := dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot"))
 		return err
 	case ACTION_StartCamera:
 		return dExt.Driver.StartCamera()

hrp/pkg/uixt/ext.go

@@ -124,9 +124,8 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error)
 	return dExt, nil
 }
 
-// TakeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
+// takeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
-// if fileName is empty, it will not save image file and only return raw image data
+func (dExt *DriverExt) takeScreenShot(fileName string) (raw *bytes.Buffer, path string, err error) {
-func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, path string, err error) {
 	// iOS 优先使用 MJPEG 流进行截图，性能最优
 	// 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口
 	if dExt.frame != nil {
@@ -145,17 +144,13 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, pa
 	}
 
 	// save screenshot to file
-	if len(fileName) > 0 && fileName[0] != "" {
+	path = filepath.Join(env.ScreenShotsPath, fileName)
-		path := filepath.Join(env.ScreenShotsPath, fileName[0])
+	path, err = dExt.saveScreenShot(compressed, path)
-		path, err := dExt.saveScreenShot(compressed, path)
+	if err != nil {
-		if err != nil {
+		log.Error().Err(err).Msg("save screenshot file failed")
-			log.Error().Err(err).Msg("save screenshot file failed")
+		return nil, "", err
-			return nil, "", err
-		}
-		return compressed, path, nil
 	}
-
+	return compressed, path, nil
-	return compressed, "", nil
 }
 
 func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error) {

hrp/pkg/uixt/ocr_vedem.go

@@ -59,7 +59,7 @@ type ImageResult struct {
 	LiveType  string     `json:"liveType"`  // 直播间类型
 }
 
-type ImageResponse struct {
+type APIResponseImage struct {
 	Code    int         `json:"code"`
 	Message string      `json:"message"`
 	Result  ImageResult `json:"result"`
@@ -279,7 +279,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
 		return
 	}
 
-	var imageResponse ImageResponse
+	var imageResponse APIResponseImage
 	err = json.Unmarshal(results, &imageResponse)
 	if err != nil {
 		log.Error().Err(err).
@@ -334,19 +334,20 @@ type IImageService interface {
 }
 
 // GetScreenResult takes a screenshot, returns the image recognization result
-func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
+func (dExt *DriverExt) GetScreenResult() (screenResult *ScreenResult, err error) {
 	var bufSource *bytes.Buffer
 	var imagePath string
-	if bufSource, imagePath, err = dExt.TakeScreenShot(
+	if bufSource, imagePath, err = dExt.takeScreenShot(
 		builtin.GenNameWithTimestamp("%d_ocr")); err != nil {
 		return
 	}
 
-	imageResult, err = dExt.ImageService.GetImage(bufSource)
+	imageResult, err := dExt.ImageService.GetImage(bufSource)
 	if err != nil {
 		log.Error().Err(err).Msg("GetScreenResult failed")
 		return
 	}
+	imageResult.imagePath = imagePath
 
 	imageUrl := imageResult.URL
 	if imageUrl != "" {
@@ -354,20 +355,25 @@ func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
 		log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot")
 	}
 
-	dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{
+	screenResult = &ScreenResult{
-		Texts: imageResult.OCRResult.ToOCRTexts(),
+		Texts:      imageResult.OCRResult.ToOCRTexts(),
+		Tags:       nil,
+		Popularity: Popularity{},
 	}
+	if imageResult.LiveType != "" {
+		screenResult.Tags = []string{imageResult.LiveType}
+	}
+	dExt.cacheStepData.screenResults[imagePath] = screenResult
 
-	imageResult.imagePath = imagePath
+	return screenResult, nil
-	return imageResult, nil
 }
 
 func (dExt *DriverExt) GetScreenTexts() (ocrTexts OCRTexts, err error) {
-	imageResult, err := dExt.GetScreenResult()
+	screenResult, err := dExt.GetScreenResult()
 	if err != nil {
 		return
 	}
-	return imageResult.OCRResult.ToOCRTexts(), nil
+	return screenResult.Texts, nil
 }
 
 func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) {

hrp/pkg/uixt/opencv.go

@@ -101,7 +101,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
 	if bufSearch, err = getBufFromDisk(search); err != nil {
 		return nil, err
 	}
-	if bufSource, _, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil {
+	if bufSource, _, err = dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil {
 		return nil, err
 	}
 

hrp/pkg/uixt/popups.go

@@ -24,16 +24,16 @@ var popups = [][]string{
 	{"管理使用时间", ".*忽略.*"},
 }
 
-func (dExt *DriverExt) autoPopupHandler(screenResult *ScreenResult) error {
+func (dExt *DriverExt) AutoPopupHandler(screenTexts OCRTexts) error {
 	for _, popup := range popups {
 		if len(popup) != 2 {
 			continue
 		}
 
-		points, err := screenResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
+		points, err := screenTexts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
 		if err == nil {
 			log.Warn().Interface("popup", popup).
-				Interface("texts", screenResult.Texts).Msg("text popup found")
+				Interface("texts", screenTexts).Msg("text popup found")
 			point := points[1].Center()
 			log.Info().Str("text", points[1].Text).Msg("close popup")
 			if err := dExt.TapAbsXY(point.X, point.Y); err != nil {

hrp/pkg/uixt/video_crawler.go

@@ -250,17 +250,13 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
 			}
 
 			// take screenshot and get screen texts by OCR
-			imageResult, err := l.driver.GetScreenResult()
+			screenResult, err := l.driver.GetScreenResult()
 			if err != nil {
 				log.Error().Err(err).Msg("OCR GetTexts failed")
 				time.Sleep(3 * time.Second)
 				continue
 			}
-			screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath]
+			screenResult.Tags = append([]string{"live"}, screenResult.Tags...)
-			screenResult.Tags = []string{"live"}
-			if imageResult.LiveType != "" {
-				screenResult.Tags = append(screenResult.Tags, imageResult.LiveType)
-			}
 
 			// check live type and incr live count
 			if err := l.currentStat.incrLive(screenResult, l.driver); err != nil {
@@ -364,7 +360,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			return errors.Wrap(code.InterruptError, "feed crawler interrupted")
 		default:
 			// take screenshot and get screen texts by OCR
-			imageResult, err := dExt.GetScreenResult()
+			screenResult, err := dExt.GetScreenResult()
 			if err != nil {
 				if strings.Contains(err.Error(), "connect: connection refused") {
 					return err
@@ -373,17 +369,15 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 				time.Sleep(3 * time.Second)
 				continue
 			}
-			screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath]
 
 			// automatic handling of pop-up windows
-			if err := dExt.autoPopupHandler(screenResult); err != nil {
+			if err := dExt.AutoPopupHandler(screenResult.Texts); err != nil {
 				log.Error().Err(err).Msg("auto handle popup failed")
 				return err
 			}
 
 			// check if live video && run live crawler
-			texts := imageResult.OCRResult.ToOCRTexts()
+			if enterPoint, isLive := liveCrawler.checkLiveVideo(screenResult.Texts); isLive {
-			if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
 				log.Info().Msg("live video found")
 				if !liveCrawler.currentStat.isLiveTargetAchieved() {
 					if err := liveCrawler.Run(dExt, enterPoint); err != nil {

hrp/step_mobile_ui.go

@@ -6,7 +6,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
-	"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
 	"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
 )
@@ -598,12 +597,17 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
 			}
 		}
 
-		// take screenshot after each step
+		// take screenshot and get screen texts by OCR
-		if _, _, err2 := uiDriver.TakeScreenShot(
+		screenResult, err2 := uiDriver.GetScreenResult()
-			builtin.GenNameWithTimestamp("%d_step_") + step.Name); err2 != nil {
+		if err2 != nil {
 			log.Error().Err(err2).Str("step", step.Name).Msg("take screenshot failed on step finished")
 		}
 
+		// automatic handling of pop-up windows on each step finished
+		if err3 := uiDriver.AutoPopupHandler(screenResult.Texts); err3 != nil {
+			log.Error().Err(err3).Msg("auto handle popup failed")
+		}
+
 		// save attachments
 		cacheData := uiDriver.GetStepCacheData()
 		for key, value := range cacheData {