GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-06-25 17:44:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: GetScreenResult

Browse Source
This commit is contained in:
lilong.129
2023-05-31 16:10:24 +08:00
parent 55e6a42b0f
commit 7d7044f332
8 changed files with 72 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/worldcup/main.go
View File

@@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i
func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error {
utcTimeStr := utcTime.Format("15:04:05")
_, ocrTexts, err := wc.driver.GetScreenTextsByOCR()
imageResult, err := wc.driver.GetScreenResult()
if err != nil {
log.Error().Err(err).Msg("get ocr texts failed")
return err
@@ -159,7 +159,7 @@ func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error {
// filter ocr texts with time format
secondsMap := map[string]int{}
var secondsTexts []string
for _, ocrText := range ocrTexts {
for _, ocrText := range imageResult.OCRResult.ToOCRTexts() {
seconds, err := convertTimeToSeconds(ocrText.Text)
if err == nil {
secondsTexts = append(secondsTexts, ocrText.Text)
@@ -212,8 +212,9 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error {
time.Sleep(5 * time.Second)
// 青少年弹窗处理
if _, ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil {
if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil {
if imageResult, err := wc.driver.GetScreenResult(); err == nil {
if points, err := imageResult.OCRResult.ToOCRTexts().
FindTexts([]string{"青少年模式", "我知道了"}); err == nil {
point := points[1].Center()
_ = wc.driver.TapAbsXY(point.X, point.Y)
}

4
hrp/pkg/uixt/demo/main_test.go
View File

@@ -35,13 +35,13 @@ func TestIOSDemo(t *testing.T) {
// 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」
for {
// take screenshot and get screen texts by OCR
_, texts, err := driverExt.GetScreenTextsByOCR()
imageResult, err := driverExt.GetScreenResult()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
t.Fatal(err)
}
points, err := texts.FindTexts([]string{"青少年模式", "我知道了"})
points, err := imageResult.OCRResult.ToOCRTexts().FindTexts([]string{"青少年模式", "我知道了"})
if err != nil {
time.Sleep(1 * time.Second)
continue

6
hrp/pkg/uixt/ext.go
View File

@@ -110,7 +110,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error)
return nil, err
}
if dExt.ImageService, err = newVEDEMImageService(); err != nil {
if dExt.ImageService, err = newVEDEMImageService("ocr", "upload", "liveType"); err != nil {
return nil, err
}
@@ -241,14 +241,14 @@ func init() {
func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) {
// click on text, using OCR
if !isPathExists(search) {
return dExt.FindScreenTextByOCR(search, options...)
return dExt.FindScreenText(search, options...)
}
// click on image, using opencv
return dExt.FindImageRectInUIKit(search, options...)
}
func (dExt *DriverExt) IsOCRExist(text string) bool {
_, err := dExt.FindScreenTextByOCR(text)
_, err := dExt.FindScreenText(text)
return err == nil
}

2
hrp/pkg/uixt/ocr_test.go
View File

@@ -10,7 +10,7 @@ func TestDriverExtOCR(t *testing.T) {
driverExt, err := iosDevice.NewDriver(nil)
checkErr(t, err)
point, err := driverExt.FindScreenTextByOCR("抖音")
point, err := driverExt.FindScreenText("抖音")
checkErr(t, err)
t.Logf("point.X: %v, point.Y: %v", point.X, point.Y)

95
hrp/pkg/uixt/ocr_vedem.go
View File

@@ -28,29 +28,35 @@ type OCRResult struct {
Points []PointF `json:"points"`
}
func (o OCRResult) ToOCRText() OCRText {
rect := image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(o.Points[0].X),
Y: int(o.Points[0].Y),
},
Max: image.Point{
X: int(o.Points[2].X),
Y: int(o.Points[2].Y),
},
}
type OCRResults []OCRResult
return OCRText{
Text: o.Text,
Rect: rect,
func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) {
for _, ocrResult := range o {
rect := image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(ocrResult.Points[0].X),
Y: int(ocrResult.Points[0].Y),
},
Max: image.Point{
X: int(ocrResult.Points[2].X),
Y: int(ocrResult.Points[2].Y),
},
}
ocrText := OCRText{
Text: ocrResult.Text,
Rect: rect,
}
ocrTexts = append(ocrTexts, ocrText)
}
return
}
type ImageResult struct {
URL string `json:"url"` // image uploaded url
OCRResult []OCRResult `json:"ocrResult"` // OCR texts
LiveType string `json:"liveType"` // 直播间类型
imagePath string
URL string `json:"url"` // image uploaded url
OCRResult OCRResults `json:"ocrResult"` // OCR texts
LiveType string `json:"liveType"` // 直播间类型
}
type ImageResponse struct {
@@ -156,22 +162,27 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (
return results, nil
}
func newVEDEMImageService() (*veDEMImageService, error) {
func newVEDEMImageService(actions ...string) (*veDEMImageService, error) {
if err := checkEnv(); err != nil {
return nil, err
}
return &veDEMImageService{}, nil
if len(actions) == 0 {
actions = []string{"ocr"}
}
return &veDEMImageService{
actions: actions,
}, nil
}
// veDEMImageService implements IImageService interface
type veDEMImageService struct{}
var actions = []string{
"ocr", // get ocr texts
"upload", // get image uploaded url
"liveType", // get live type
// "popup",
// "close",
// actions:
// ocr - get ocr texts
// upload - get image uploaded url
// liveType - get live type
// popup - get popup windows
// close - get close popup
type veDEMImageService struct {
actions []string
}
func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
@@ -179,7 +190,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
for _, action := range actions {
for _, action := range s.actions {
bodyWriter.WriteField("actions", action)
}
@@ -320,24 +331,21 @@ type IImageService interface {
GetImage(imageBuf *bytes.Buffer) (imageResult ImageResult, err error)
}
// GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts.
func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRTexts, err error) {
// GetScreenResult takes a screenshot, returns the image recognization result
func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
var bufSource *bytes.Buffer
var imagePath string
if bufSource, imagePath, err = dExt.TakeScreenShot(
builtin.GenNameWithTimestamp("%d_ocr")); err != nil {
return
}
imageResult, err := dExt.ImageService.GetImage(bufSource)
imageResult, err = dExt.ImageService.GetImage(bufSource)
if err != nil {
log.Error().Err(err).Msg("GetScreenTextsByOCR failed")
log.Error().Err(err).Msg("GetScreenResult failed")
return
}
for _, ocrResult := range imageResult.OCRResult {
ocrTexts = append(ocrTexts, ocrResult.ToOCRText())
}
imageUrl := imageResult.URL
if imageUrl != "" {
dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl
@@ -345,18 +353,19 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText
}
dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{
Texts: ocrTexts,
Texts: imageResult.OCRResult.ToOCRTexts(),
}
return imagePath, ocrTexts, nil
imageResult.imagePath = imagePath
return imageResult, nil
}
func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) {
_, ocrTexts, err := dExt.GetScreenTextsByOCR()
func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) {
imageResult, err := dExt.GetScreenResult()
if err != nil {
return
}
result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...)
result, err := imageResult.OCRResult.ToOCRTexts().FindText(text, dExt.ParseActionOptions(options...)...)
if err != nil {
log.Warn().Msgf("FindText failed: %s", err.Error())
return
@@ -364,7 +373,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption)
point = result.Center()
log.Info().Str("text", text).
Interface("point", point).Msgf("FindScreenTextByOCR success")
Interface("point", point).Msgf("FindScreenText success")
return
}

5
hrp/pkg/uixt/swipe.go
View File

@@ -135,11 +135,12 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
var point PointF
findTexts := func(d *DriverExt) error {
var err error
_, ocrTexts, err := d.GetScreenTextsByOCR()
imageResult, err := d.GetScreenResult()
if err != nil {
return err
}
points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...)
points, err := imageResult.OCRResult.ToOCRTexts().
FindTexts(texts, dExt.ParseActionOptions(options...)...)
if err != nil {
return err
}

2
hrp/pkg/uixt/tap.go
View File

@@ -24,7 +24,7 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error {
func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
point, err := dExt.FindScreenTextByOCR(ocrText, options...)
point, err := dExt.FindScreenText(ocrText, options...)
if err != nil {
if actionOptions.IgnoreNotFoundError {
return nil

9
hrp/pkg/uixt/video_crawler.go
View File

@@ -257,12 +257,12 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
}
// take screenshot and get screen texts by OCR
imagePath, _, err := l.driver.GetScreenTextsByOCR()
imageResult, err := l.driver.GetScreenResult()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
continue
}
screenResult := l.driver.cacheStepData.screenResults[imagePath]
screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath]
screenResult.Tags = []string{"live"}
// check live type and incr live count
@@ -365,12 +365,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
return errors.Wrap(code.InterruptError, "feed crawler interrupted")
default:
// take screenshot and get screen texts by OCR
imagePath, texts, err := dExt.GetScreenTextsByOCR()
imageResult, err := dExt.GetScreenResult()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
continue
}
screenResult := dExt.cacheStepData.screenResults[imagePath]
screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath]
// automatic handling of pop-up windows
if err := dExt.autoPopupHandler(screenResult); err != nil {
@@ -379,6 +379,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
}
// check if live video && run live crawler
texts := imageResult.OCRResult.ToOCRTexts()
if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
log.Info().Msg("live video found")
if !liveCrawler.currentStat.isLiveTargetAchieved() {