From 7d7044f332f5a1fb38b8e03e1b1c9ccd000631b6 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 31 May 2023 16:10:24 +0800 Subject: [PATCH] refactor: GetScreenResult --- examples/worldcup/main.go | 9 ++-- hrp/pkg/uixt/demo/main_test.go | 4 +- hrp/pkg/uixt/ext.go | 6 +-- hrp/pkg/uixt/ocr_test.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 95 +++++++++++++++++++--------------- hrp/pkg/uixt/swipe.go | 5 +- hrp/pkg/uixt/tap.go | 2 +- hrp/pkg/uixt/video_crawler.go | 9 ++-- 8 files changed, 72 insertions(+), 60 deletions(-) diff --git a/examples/worldcup/main.go b/examples/worldcup/main.go index eea2f47e..dc47a33e 100644 --- a/examples/worldcup/main.go +++ b/examples/worldcup/main.go @@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { utcTimeStr := utcTime.Format("15:04:05") - _, ocrTexts, err := wc.driver.GetScreenTextsByOCR() + imageResult, err := wc.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("get ocr texts failed") return err @@ -159,7 +159,7 @@ func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error { // filter ocr texts with time format secondsMap := map[string]int{} var secondsTexts []string - for _, ocrText := range ocrTexts { + for _, ocrText := range imageResult.OCRResult.ToOCRTexts() { seconds, err := convertTimeToSeconds(ocrText.Text) if err == nil { secondsTexts = append(secondsTexts, ocrText.Text) @@ -212,8 +212,9 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error { time.Sleep(5 * time.Second) // 青少年弹窗处理 - if _, ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil { - if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil { + if imageResult, err := wc.driver.GetScreenResult(); err == nil { + if points, err := imageResult.OCRResult.ToOCRTexts(). + FindTexts([]string{"青少年模式", "我知道了"}); err == nil { point := points[1].Center() _ = wc.driver.TapAbsXY(point.X, point.Y) } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 8a042a0d..34f001d8 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -35,13 +35,13 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { // take screenshot and get screen texts by OCR - _, texts, err := driverExt.GetScreenTextsByOCR() + imageResult, err := driverExt.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") t.Fatal(err) } - points, err := texts.FindTexts([]string{"青少年模式", "我知道了"}) + points, err := imageResult.OCRResult.ToOCRTexts().FindTexts([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index f414e414..8afe2233 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -110,7 +110,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error) return nil, err } - if dExt.ImageService, err = newVEDEMImageService(); err != nil { + if dExt.ImageService, err = newVEDEMImageService("ocr", "upload", "liveType"); err != nil { return nil, err } @@ -241,14 +241,14 @@ func init() { func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindScreenTextByOCR(search, options...) + return dExt.FindScreenText(search, options...) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, options...) } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, err := dExt.FindScreenTextByOCR(text) + _, err := dExt.FindScreenText(text) return err == nil } diff --git a/hrp/pkg/uixt/ocr_test.go b/hrp/pkg/uixt/ocr_test.go index 053ccaee..da8ed745 100644 --- a/hrp/pkg/uixt/ocr_test.go +++ b/hrp/pkg/uixt/ocr_test.go @@ -10,7 +10,7 @@ func TestDriverExtOCR(t *testing.T) { driverExt, err := iosDevice.NewDriver(nil) checkErr(t, err) - point, err := driverExt.FindScreenTextByOCR("抖音") + point, err := driverExt.FindScreenText("抖音") checkErr(t, err) t.Logf("point.X: %v, point.Y: %v", point.X, point.Y) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index eae5a924..17fa7747 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -28,29 +28,35 @@ type OCRResult struct { Points []PointF `json:"points"` } -func (o OCRResult) ToOCRText() OCRText { - rect := image.Rectangle{ - // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 - Min: image.Point{ - X: int(o.Points[0].X), - Y: int(o.Points[0].Y), - }, - Max: image.Point{ - X: int(o.Points[2].X), - Y: int(o.Points[2].Y), - }, - } +type OCRResults []OCRResult - return OCRText{ - Text: o.Text, - Rect: rect, +func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) { + for _, ocrResult := range o { + rect := image.Rectangle{ + // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(ocrResult.Points[0].X), + Y: int(ocrResult.Points[0].Y), + }, + Max: image.Point{ + X: int(ocrResult.Points[2].X), + Y: int(ocrResult.Points[2].Y), + }, + } + ocrText := OCRText{ + Text: ocrResult.Text, + Rect: rect, + } + ocrTexts = append(ocrTexts, ocrText) } + return } type ImageResult struct { - URL string `json:"url"` // image uploaded url - OCRResult []OCRResult `json:"ocrResult"` // OCR texts - LiveType string `json:"liveType"` // 直播间类型 + imagePath string + URL string `json:"url"` // image uploaded url + OCRResult OCRResults `json:"ocrResult"` // OCR texts + LiveType string `json:"liveType"` // 直播间类型 } type ImageResponse struct { @@ -156,22 +162,27 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) ( return results, nil } -func newVEDEMImageService() (*veDEMImageService, error) { +func newVEDEMImageService(actions ...string) (*veDEMImageService, error) { if err := checkEnv(); err != nil { return nil, err } - return &veDEMImageService{}, nil + if len(actions) == 0 { + actions = []string{"ocr"} + } + return &veDEMImageService{ + actions: actions, + }, nil } // veDEMImageService implements IImageService interface -type veDEMImageService struct{} - -var actions = []string{ - "ocr", // get ocr texts - "upload", // get image uploaded url - "liveType", // get live type - // "popup", - // "close", +// actions: +// ocr - get ocr texts +// upload - get image uploaded url +// liveType - get live type +// popup - get popup windows +// close - get close popup +type veDEMImageService struct { + actions []string } func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( @@ -179,7 +190,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) ( bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) - for _, action := range actions { + for _, action := range s.actions { bodyWriter.WriteField("actions", action) } @@ -320,24 +331,21 @@ type IImageService interface { GetImage(imageBuf *bytes.Buffer) (imageResult ImageResult, err error) } -// GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts. -func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRTexts, err error) { +// GetScreenResult takes a screenshot, returns the image recognization result +func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) { var bufSource *bytes.Buffer + var imagePath string if bufSource, imagePath, err = dExt.TakeScreenShot( builtin.GenNameWithTimestamp("%d_ocr")); err != nil { return } - imageResult, err := dExt.ImageService.GetImage(bufSource) + imageResult, err = dExt.ImageService.GetImage(bufSource) if err != nil { - log.Error().Err(err).Msg("GetScreenTextsByOCR failed") + log.Error().Err(err).Msg("GetScreenResult failed") return } - for _, ocrResult := range imageResult.OCRResult { - ocrTexts = append(ocrTexts, ocrResult.ToOCRText()) - } - imageUrl := imageResult.URL if imageUrl != "" { dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl @@ -345,18 +353,19 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText } dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{ - Texts: ocrTexts, + Texts: imageResult.OCRResult.ToOCRTexts(), } - return imagePath, ocrTexts, nil + imageResult.imagePath = imagePath + return imageResult, nil } -func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) { - _, ocrTexts, err := dExt.GetScreenTextsByOCR() +func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) { + imageResult, err := dExt.GetScreenResult() if err != nil { return } - result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...) + result, err := imageResult.OCRResult.ToOCRTexts().FindText(text, dExt.ParseActionOptions(options...)...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return @@ -364,7 +373,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) point = result.Center() log.Info().Str("text", text). - Interface("point", point).Msgf("FindScreenTextByOCR success") + Interface("point", point).Msgf("FindScreenText success") return } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 06488ef2..b445babb 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -135,11 +135,12 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) var point PointF findTexts := func(d *DriverExt) error { var err error - _, ocrTexts, err := d.GetScreenTextsByOCR() + imageResult, err := d.GetScreenResult() if err != nil { return err } - points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...) + points, err := imageResult.OCRResult.ToOCRTexts(). + FindTexts(texts, dExt.ParseActionOptions(options...)...) if err != nil { return err } diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index b4707839..84b3541b 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -24,7 +24,7 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error { func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error { actionOptions := NewActionOptions(options...) - point, err := dExt.FindScreenTextByOCR(ocrText, options...) + point, err := dExt.FindScreenText(ocrText, options...) if err != nil { if actionOptions.IgnoreNotFoundError { return nil diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 2e4294cc..001734df 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -257,12 +257,12 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } // take screenshot and get screen texts by OCR - imagePath, _, err := l.driver.GetScreenTextsByOCR() + imageResult, err := l.driver.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - screenResult := l.driver.cacheStepData.screenResults[imagePath] + screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath] screenResult.Tags = []string{"live"} // check live type and incr live count @@ -365,12 +365,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return errors.Wrap(code.InterruptError, "feed crawler interrupted") default: // take screenshot and get screen texts by OCR - imagePath, texts, err := dExt.GetScreenTextsByOCR() + imageResult, err := dExt.GetScreenResult() if err != nil { log.Error().Err(err).Msg("OCR GetTexts failed") continue } - screenResult := dExt.cacheStepData.screenResults[imagePath] + screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath] // automatic handling of pop-up windows if err := dExt.autoPopupHandler(screenResult); err != nil { @@ -379,6 +379,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // check if live video && run live crawler + texts := imageResult.OCRResult.ToOCRTexts() if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") if !liveCrawler.currentStat.isLiveTargetAchieved() {