GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-16 15:37:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: auto handle popups on each step finished

Browse Source
This commit is contained in:
lilong.129
2023-06-22 12:48:29 +08:00
parent e1cf7d9679
commit a98c497995
7 changed files with 43 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hrp/pkg/uixt/action.go
View File

@@ -506,7 +506,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
case ACTION_ScreenShot:
// take screenshot
log.Info().Msg("take screenshot for current screen")
_, _, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot"))
_, _, err := dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_screenshot"))
return err
case ACTION_StartCamera:
return dExt.Driver.StartCamera()

21
hrp/pkg/uixt/ext.go
View File

@@ -124,9 +124,8 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error)
return dExt, nil
}
// TakeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
// if fileName is empty, it will not save image file and only return raw image data
func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, path string, err error) {
// takeScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
func (dExt *DriverExt) takeScreenShot(fileName string) (raw *bytes.Buffer, path string, err error) {
// iOS 优先使用 MJPEG 流进行截图，性能最优
// 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口
if dExt.frame != nil {
@@ -145,17 +144,13 @@ func (dExt *DriverExt) TakeScreenShot(fileName ...string) (raw *bytes.Buffer, pa
}
// save screenshot to file
if len(fileName) > 0 && fileName[0] != "" {
path := filepath.Join(env.ScreenShotsPath, fileName[0])
path, err := dExt.saveScreenShot(compressed, path)
if err != nil {
log.Error().Err(err).Msg("save screenshot file failed")
return nil, "", err
}
return compressed, path, nil
path = filepath.Join(env.ScreenShotsPath, fileName)
path, err = dExt.saveScreenShot(compressed, path)
if err != nil {
log.Error().Err(err).Msg("save screenshot file failed")
return nil, "", err
}
return compressed, "", nil
return compressed, path, nil
}
func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error) {

28
hrp/pkg/uixt/ocr_vedem.go
View File

@@ -59,7 +59,7 @@ type ImageResult struct {
LiveType string `json:"liveType"` // 直播间类型
}
type ImageResponse struct {
type APIResponseImage struct {
Code int `json:"code"`
Message string `json:"message"`
Result ImageResult `json:"result"`
@@ -279,7 +279,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
return
}
var imageResponse ImageResponse
var imageResponse APIResponseImage
err = json.Unmarshal(results, &imageResponse)
if err != nil {
log.Error().Err(err).
@@ -334,19 +334,20 @@ type IImageService interface {
}
// GetScreenResult takes a screenshot, returns the image recognization result
func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
func (dExt *DriverExt) GetScreenResult() (screenResult *ScreenResult, err error) {
var bufSource *bytes.Buffer
var imagePath string
if bufSource, imagePath, err = dExt.TakeScreenShot(
if bufSource, imagePath, err = dExt.takeScreenShot(
builtin.GenNameWithTimestamp("%d_ocr")); err != nil {
return
}
imageResult, err = dExt.ImageService.GetImage(bufSource)
imageResult, err := dExt.ImageService.GetImage(bufSource)
if err != nil {
log.Error().Err(err).Msg("GetScreenResult failed")
return
}
imageResult.imagePath = imagePath
imageUrl := imageResult.URL
if imageUrl != "" {
@@ -354,20 +355,25 @@ func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
log.Debug().Str("imagePath", imagePath).Str("imageUrl", imageUrl).Msg("log screenshot")
}
dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{
Texts: imageResult.OCRResult.ToOCRTexts(),
screenResult = &ScreenResult{
Texts: imageResult.OCRResult.ToOCRTexts(),
Tags: nil,
Popularity: Popularity{},
}
if imageResult.LiveType != "" {
screenResult.Tags = []string{imageResult.LiveType}
}
dExt.cacheStepData.screenResults[imagePath] = screenResult
imageResult.imagePath = imagePath
return imageResult, nil
return screenResult, nil
}
func (dExt *DriverExt) GetScreenTexts() (ocrTexts OCRTexts, err error) {
imageResult, err := dExt.GetScreenResult()
screenResult, err := dExt.GetScreenResult()
if err != nil {
return
}
return imageResult.OCRResult.ToOCRTexts(), nil
return screenResult.Texts, nil
}
func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) {

2
hrp/pkg/uixt/opencv.go
View File

@@ -101,7 +101,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
if bufSearch, err = getBufFromDisk(search); err != nil {
return nil, err
}
if bufSource, _, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil {
if bufSource, _, err = dExt.takeScreenShot(builtin.GenNameWithTimestamp("%d_cv")); err != nil {
return nil, err
}

6
hrp/pkg/uixt/popups.go
View File

@@ -24,16 +24,16 @@ var popups = [][]string{
{"管理使用时间", ".*忽略.*"},
}
func (dExt *DriverExt) autoPopupHandler(screenResult *ScreenResult) error {
func (dExt *DriverExt) AutoPopupHandler(screenTexts OCRTexts) error {
for _, popup := range popups {
if len(popup) != 2 {
continue
}
points, err := screenResult.Texts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
points, err := screenTexts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
if err == nil {
log.Warn().Interface("popup", popup).
Interface("texts", screenResult.Texts).Msg("text popup found")
Interface("texts", screenTexts).Msg("text popup found")
point := points[1].Center()
log.Info().Str("text", points[1].Text).Msg("close popup")
if err := dExt.TapAbsXY(point.X, point.Y); err != nil {

16
hrp/pkg/uixt/video_crawler.go
View File

@@ -250,17 +250,13 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
}
// take screenshot and get screen texts by OCR
imageResult, err := l.driver.GetScreenResult()
screenResult, err := l.driver.GetScreenResult()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
time.Sleep(3 * time.Second)
continue
}
screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath]
screenResult.Tags = []string{"live"}
if imageResult.LiveType != "" {
screenResult.Tags = append(screenResult.Tags, imageResult.LiveType)
}
screenResult.Tags = append([]string{"live"}, screenResult.Tags...)
// check live type and incr live count
if err := l.currentStat.incrLive(screenResult, l.driver); err != nil {
@@ -364,7 +360,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
return errors.Wrap(code.InterruptError, "feed crawler interrupted")
default:
// take screenshot and get screen texts by OCR
imageResult, err := dExt.GetScreenResult()
screenResult, err := dExt.GetScreenResult()
if err != nil {
if strings.Contains(err.Error(), "connect: connection refused") {
return err
@@ -373,17 +369,15 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
time.Sleep(3 * time.Second)
continue
}
screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath]
// automatic handling of pop-up windows
if err := dExt.autoPopupHandler(screenResult); err != nil {
if err := dExt.AutoPopupHandler(screenResult.Texts); err != nil {
log.Error().Err(err).Msg("auto handle popup failed")
return err
}
// check if live video && run live crawler
texts := imageResult.OCRResult.ToOCRTexts()
if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
if enterPoint, isLive := liveCrawler.checkLiveVideo(screenResult.Texts); isLive {
log.Info().Msg("live video found")
if !liveCrawler.currentStat.isLiveTargetAchieved() {
if err := liveCrawler.Run(dExt, enterPoint); err != nil {

12
hrp/step_mobile_ui.go
View File

@@ -6,7 +6,6 @@ import (
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/httprunner/httprunner/v4/hrp/internal/code"
"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
)
@@ -598,12 +597,17 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
}
}
// take screenshot after each step
if _, _, err2 := uiDriver.TakeScreenShot(
builtin.GenNameWithTimestamp("%d_step_") + step.Name); err2 != nil {
// take screenshot and get screen texts by OCR
screenResult, err2 := uiDriver.GetScreenResult()
if err2 != nil {
log.Error().Err(err2).Str("step", step.Name).Msg("take screenshot failed on step finished")
}
// automatic handling of pop-up windows on each step finished
if err3 := uiDriver.AutoPopupHandler(screenResult.Texts); err3 != nil {
log.Error().Err(err3).Msg("auto handle popup failed")
}
// save attachments
cacheData := uiDriver.GetStepCacheData()
for key, value := range cacheData {