diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index ee12b7a9..b47ba5a1 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -506,7 +506,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { case ACTION_ScreenShot: // take screenshot log.Info().Msg("take screenshot for current screen") - _, _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot")) + _, _, err := dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot")) return err case ACTION_StartCamera: return dExt.Driver.StartCamera() diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index a5f50b40..0390cc28 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -124,9 +124,8 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return dExt, nil } -// TakeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder -// if fileName is empty, it will not save image file and only return raw image data -func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, path string, err error) { +// takeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder +func (dExt *DriverExt) takeScreenShot(fileName string) (raw *bytes.Buffer, path string, err error) { // iOS 优先使用 MJPEG 流进行截图，性能最优 // 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口 if dExt.frame != nil { @@ -145,17 +144,13 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, pa } // save screenshot to file - if len(fileName) > 0 && fileName[0] != "" { - path := filepath.Join(env.ScreenShotsPath, fileName[0]) - path, err := dExt.saveScreenShot(compressed, path) - if err != nil { - log.Error().Err(err).Msg("save screenshot file failed") - return nil, "", err - } - return compressed, path, nil + path = filepath.Join(env.ScreenShotsPath, fileName) + path, err = dExt.saveScreenShot(compressed, path) + if err != nil { + log.Error().Err(err).Msg("save screenshot file failed") + return nil, "", err } - - return compressed, "", nil + return compressed, path, nil } func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error) { diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 3345428e..beb5f0a4 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -59,7 +59,7 @@ type ImageResult struct { LiveType string `json:"liveType"` // 直播间类型 } -type ImageResponse struct { +type APIResponseImage struct { Code int `json:"code"` Message string `json:"message"` Result ImageResult `json:"result"` @@ -279,7 +279,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( return } - var imageResponse ImageResponse + var imageResponse APIResponseImage err = json.Unmarshal(results, &imageResponse) if err != nil { log.Error().Err(err). @@ -334,19 +334,20 @@ type IImageService interface { } // GetScreenResult takes a screenshot, returns the image recognization result -func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) { +func (dExt *DriverExt) GetScreenResult() (screenResult *ScreenResult, err error) { var bufSource *bytes.Buffer var imagePath string - if bufSource, imagePath, err = dExt.TakeScreenShot( + if bufSource, imagePath, err = dExt.takeScreenShot( builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } - imageResult, err = dExt.ImageService.GetImage(bufSource) + imageResult, err := dExt.ImageService.GetImage(bufSource) if err != nil { log.Error().Err(err).Msg("GetScreenResult failed") return } + imageResult.imagePath = imagePath imageUrl := imageResult.URL if imageUrl != "" { @@ -354,20 +355,25 @@ func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) { log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot") } - dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{ - Texts: imageResult.OCRResult.ToOCRTexts(), + screenResult = &ScreenResult{ + Texts: imageResult.OCRResult.ToOCRTexts(), + Tags: nil, + Popularity: Popularity{}, } + if imageResult.LiveType != "" { + screenResult.Tags = []string{imageResult.LiveType} + } + dExt.cacheStepData.screenResults[imagePath] = screenResult - imageResult.imagePath = imagePath - return imageResult, nil + return screenResult, nil } func (dExt *DriverExt) GetScreenTexts() (ocrTexts OCRTexts, err error) { - imageResult, err := dExt.GetScreenResult() + screenResult, err := dExt.GetScreenResult() if err != nil { return } - return imageResult.OCRResult.ToOCRTexts(), nil + return screenResult.Texts, nil } func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) { diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index e72f0cde..75c1d0ca 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -101,7 +101,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, if bufSearch, err = getBufFromDisk(search); err != nil { return nil, err } - if bufSource, _, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil { + if bufSource, _, err = dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil { return nil, err } diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index ef2fab94..5bde89df 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -24,16 +24,16 @@ var popups = [][]string{ {"管理使用时间", ".*忽略.*"}, } -func (dExt *DriverExt) autoPopupHandler(screenResult *ScreenResult) error { +func (dExt *DriverExt) AutoPopupHandler(screenTexts OCRTexts) error { for _, popup := range popups { if len(popup) != 2 { continue } - points, err := screenResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true)) + points, err := screenTexts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true)) if err == nil { log.Warn().Interface("popup", popup). - Interface("texts", screenResult.Texts).Msg("text popup found") + Interface("texts", screenTexts).Msg("text popup found") point := points[1].Center() log.Info().Str("text", points[1].Text).Msg("close popup") if err := dExt.TapAbsXY(point.X, point.Y); err != nil { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 085bc69b..9021b86c 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -250,17 +250,13 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } // take screenshot and get screen texts by OCR - imageResult, err := l.driver.GetScreenResult() + screenResult, err := l.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") time.Sleep(3 * time.Second) continue } - screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath] - screenResult.Tags = []string{"live"} - if imageResult.LiveType != "" { - screenResult.Tags = append(screenResult.Tags, imageResult.LiveType) - } + screenResult.Tags = append([]string{"live"}, screenResult.Tags...) // check live type and incr live count if err := l.currentStat.incrLive(screenResult, l.driver); err != nil { @@ -364,7 +360,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return errors.Wrap(code.InterruptError, "feed crawler interrupted") default: // take screenshot and get screen texts by OCR - imageResult, err := dExt.GetScreenResult() + screenResult, err := dExt.GetScreenResult() if err != nil { if strings.Contains(err.Error(), "connect: connection refused") { return err @@ -373,17 +369,15 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { time.Sleep(3 * time.Second) continue } - screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath] // automatic handling of pop-up windows - if err := dExt.autoPopupHandler(screenResult); err != nil { + if err := dExt.AutoPopupHandler(screenResult.Texts); err != nil { log.Error().Err(err).Msg("auto handle popup failed") return err } // check if live video && run live crawler - texts := imageResult.OCRResult.ToOCRTexts() - if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { + if enterPoint, isLive := liveCrawler.checkLiveVideo(screenResult.Texts); isLive { log.Info().Msg("live video found") if !liveCrawler.currentStat.isLiveTargetAchieved() { if err := liveCrawler.Run(dExt, enterPoint); err != nil { diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index ac95207e..7e787775 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/code" "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" ) @@ -598,12 +597,17 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err } } - // take screenshot after each step - if _, _, err2 := uiDriver.TakeScreenShot( - builtin.GenNameWithTimestamp("%d_step_") + step.Name); err2 != nil { + // take screenshot and get screen texts by OCR + screenResult, err2 := uiDriver.GetScreenResult() + if err2 != nil { log.Error().Err(err2).Str("step", step.Name).Msg("take screenshot failed on step finished") } + // automatic handling of pop-up windows on each step finished + if err3 := uiDriver.AutoPopupHandler(screenResult.Texts); err3 != nil { + log.Error().Err(err3).Msg("auto handle popup failed") + } + // save attachments cacheData := uiDriver.GetStepCacheData() for key, value := range cacheData {