refactor: GetScreenResult

examples/worldcup/main.go

@@ -150,7 +150,7 @@ func NewWorldCupLive(device uixt.Device, matchName, bundleID string, duration, i
 
 func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error {
 	utcTimeStr := utcTime.Format("15:04:05")
-	_, ocrTexts, err := wc.driver.GetScreenTextsByOCR()
+	imageResult, err := wc.driver.GetScreenResult()
 	if err != nil {
 		log.Error().Err(err).Msg("get ocr texts failed")
 		return err
@@ -159,7 +159,7 @@ func (wc *WorldCupLive) getCurrentLiveTime(utcTime time.Time) error {
 	// filter ocr texts with time format
 	secondsMap := map[string]int{}
 	var secondsTexts []string
-	for _, ocrText := range ocrTexts {
+	for _, ocrText := range imageResult.OCRResult.ToOCRTexts() {
 		seconds, err := convertTimeToSeconds(ocrText.Text)
 		if err == nil {
 			secondsTexts = append(secondsTexts, ocrText.Text)
@@ -212,8 +212,9 @@ func (wc *WorldCupLive) EnterLive(bundleID string) error {
 	time.Sleep(5 * time.Second)
 
 	// 青少年弹窗处理
-	if _, ocrTexts, err := wc.driver.GetScreenTextsByOCR(); err == nil {
+	if imageResult, err := wc.driver.GetScreenResult(); err == nil {
-		if points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"}); err == nil {
+		if points, err := imageResult.OCRResult.ToOCRTexts().
+			FindTexts([]string{"青少年模式", "我知道了"}); err == nil {
 			point := points[1].Center()
 			_ = wc.driver.TapAbsXY(point.X, point.Y)
 		}

hrp/pkg/uixt/demo/main_test.go

@@ -35,13 +35,13 @@ func TestIOSDemo(t *testing.T) {
 	// 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」
 	for {
 		// take screenshot and get screen texts by OCR
-		_, texts, err := driverExt.GetScreenTextsByOCR()
+		imageResult, err := driverExt.GetScreenResult()
 		if err != nil {
 			log.Error().Err(err).Msg("OCR GetTexts failed")
 			t.Fatal(err)
 		}
 
-		points, err := texts.FindTexts([]string{"青少年模式", "我知道了"})
+		points, err := imageResult.OCRResult.ToOCRTexts().FindTexts([]string{"青少年模式", "我知道了"})
 		if err != nil {
 			time.Sleep(1 * time.Second)
 			continue

hrp/pkg/uixt/ext.go

@@ -110,7 +110,7 @@ func NewDriverExt(device Device, driver WebDriver) (dExt *DriverExt, err error)
 		return nil, err
 	}
 
-	if dExt.ImageService, err = newVEDEMImageService(); err != nil {
+	if dExt.ImageService, err = newVEDEMImageService("ocr", "upload", "liveType"); err != nil {
 		return nil, err
 	}
 
@@ -241,14 +241,14 @@ func init() {
 func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) {
 	// click on text, using OCR
 	if !isPathExists(search) {
-		return dExt.FindScreenTextByOCR(search, options...)
+		return dExt.FindScreenText(search, options...)
 	}
 	// click on image, using opencv
 	return dExt.FindImageRectInUIKit(search, options...)
 }
 
 func (dExt *DriverExt) IsOCRExist(text string) bool {
-	_, err := dExt.FindScreenTextByOCR(text)
+	_, err := dExt.FindScreenText(text)
 	return err == nil
 }
 

hrp/pkg/uixt/ocr_test.go

@@ -10,7 +10,7 @@ func TestDriverExtOCR(t *testing.T) {
 	driverExt, err := iosDevice.NewDriver(nil)
 	checkErr(t, err)
 
-	point, err := driverExt.FindScreenTextByOCR("抖音")
+	point, err := driverExt.FindScreenText("抖音")
 	checkErr(t, err)
 
 	t.Logf("point.X: %v, point.Y: %v", point.X, point.Y)

hrp/pkg/uixt/ocr_vedem.go

@@ -28,28 +28,34 @@ type OCRResult struct {
 	Points []PointF `json:"points"`
 }
 
-func (o OCRResult) ToOCRText() OCRText {
+type OCRResults []OCRResult
+
+func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) {
+	for _, ocrResult := range o {
 		rect := image.Rectangle{
 			// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
 			Min: image.Point{
-			X: int(o.Points[0].X),
+				X: int(ocrResult.Points[0].X),
-			Y: int(o.Points[0].Y),
+				Y: int(ocrResult.Points[0].Y),
 			},
 			Max: image.Point{
-			X: int(o.Points[2].X),
+				X: int(ocrResult.Points[2].X),
-			Y: int(o.Points[2].Y),
+				Y: int(ocrResult.Points[2].Y),
 			},
 		}
-
+		ocrText := OCRText{
-	return OCRText{
+			Text: ocrResult.Text,
-		Text: o.Text,
 			Rect: rect,
 		}
+		ocrTexts = append(ocrTexts, ocrText)
+	}
+	return
 }
 
 type ImageResult struct {
+	imagePath string
 	URL       string     `json:"url"`       // image uploaded url
-	OCRResult []OCRResult `json:"ocrResult"` // OCR texts
+	OCRResult OCRResults `json:"ocrResult"` // OCR texts
 	LiveType  string     `json:"liveType"`  // 直播间类型
 }
 
@@ -156,22 +162,27 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (
 	return results, nil
 }
 
-func newVEDEMImageService() (*veDEMImageService, error) {
+func newVEDEMImageService(actions ...string) (*veDEMImageService, error) {
 	if err := checkEnv(); err != nil {
 		return nil, err
 	}
-	return &veDEMImageService{}, nil
+	if len(actions) == 0 {
+		actions = []string{"ocr"}
+	}
+	return &veDEMImageService{
+		actions: actions,
+	}, nil
 }
 
 // veDEMImageService implements IImageService interface
-type veDEMImageService struct{}
+// actions:
-
+// 	ocr - get ocr texts
-var actions = []string{
+// 	upload - get image uploaded url
-	"ocr",      // get ocr texts
+// 	liveType - get live type
-	"upload",   // get image uploaded url
+// 	popup - get popup windows
-	"liveType", // get live type
+// 	close - get close popup
-	// "popup",
+type veDEMImageService struct {
-	// "close",
+	actions []string
 }
 
 func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
@@ -179,7 +190,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer) (
 
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
-	for _, action := range actions {
+	for _, action := range s.actions {
 		bodyWriter.WriteField("actions", action)
 	}
 
@@ -320,24 +331,21 @@ type IImageService interface {
 	GetImage(imageBuf *bytes.Buffer) (imageResult ImageResult, err error)
 }
 
-// GetScreenTextsByOCR takes a screenshot, returns the image path and OCR texts.
+// GetScreenResult takes a screenshot, returns the image recognization result
-func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRTexts, err error) {
+func (dExt *DriverExt) GetScreenResult() (imageResult ImageResult, err error) {
 	var bufSource *bytes.Buffer
+	var imagePath string
 	if bufSource, imagePath, err = dExt.TakeScreenShot(
 		builtin.GenNameWithTimestamp("%d_ocr")); err != nil {
 		return
 	}
 
-	imageResult, err := dExt.ImageService.GetImage(bufSource)
+	imageResult, err = dExt.ImageService.GetImage(bufSource)
 	if err != nil {
-		log.Error().Err(err).Msg("GetScreenTextsByOCR failed")
+		log.Error().Err(err).Msg("GetScreenResult failed")
 		return
 	}
 
-	for _, ocrResult := range imageResult.OCRResult {
-		ocrTexts = append(ocrTexts, ocrResult.ToOCRText())
-	}
-
 	imageUrl := imageResult.URL
 	if imageUrl != "" {
 		dExt.cacheStepData.screenShotsUrls[imagePath] = imageUrl
@@ -345,18 +353,19 @@ func (dExt *DriverExt) GetScreenTextsByOCR() (imagePath string, ocrTexts OCRText
 	}
 
 	dExt.cacheStepData.screenResults[imagePath] = &ScreenResult{
-		Texts: ocrTexts,
+		Texts: imageResult.OCRResult.ToOCRTexts(),
 	}
 
-	return imagePath, ocrTexts, nil
+	imageResult.imagePath = imagePath
+	return imageResult, nil
 }
 
-func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption) (point PointF, err error) {
+func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) {
-	_, ocrTexts, err := dExt.GetScreenTextsByOCR()
+	imageResult, err := dExt.GetScreenResult()
 	if err != nil {
 		return
 	}
-	result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...)
+	result, err := imageResult.OCRResult.ToOCRTexts().FindText(text, dExt.ParseActionOptions(options...)...)
 	if err != nil {
 		log.Warn().Msgf("FindText failed: %s", err.Error())
 		return
@@ -364,7 +373,7 @@ func (dExt *DriverExt) FindScreenTextByOCR(text string, options ...ActionOption)
 	point = result.Center()
 
 	log.Info().Str("text", text).
-		Interface("point", point).Msgf("FindScreenTextByOCR success")
+		Interface("point", point).Msgf("FindScreenText success")
 	return
 }
 

hrp/pkg/uixt/swipe.go

@@ -135,11 +135,12 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
 	var point PointF
 	findTexts := func(d *DriverExt) error {
 		var err error
-		_, ocrTexts, err := d.GetScreenTextsByOCR()
+		imageResult, err := d.GetScreenResult()
 		if err != nil {
 			return err
 		}
-		points, err := ocrTexts.FindTexts(texts, dExt.ParseActionOptions(options...)...)
+		points, err := imageResult.OCRResult.ToOCRTexts().
+			FindTexts(texts, dExt.ParseActionOptions(options...)...)
 		if err != nil {
 			return err
 		}

hrp/pkg/uixt/tap.go

@@ -24,7 +24,7 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error {
 func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error {
 	actionOptions := NewActionOptions(options...)
 
-	point, err := dExt.FindScreenTextByOCR(ocrText, options...)
+	point, err := dExt.FindScreenText(ocrText, options...)
 	if err != nil {
 		if actionOptions.IgnoreNotFoundError {
 			return nil

hrp/pkg/uixt/video_crawler.go

@@ -257,12 +257,12 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
 			}
 
 			// take screenshot and get screen texts by OCR
-			imagePath, _, err := l.driver.GetScreenTextsByOCR()
+			imageResult, err := l.driver.GetScreenResult()
 			if err != nil {
 				log.Error().Err(err).Msg("OCR GetTexts failed")
 				continue
 			}
-			screenResult := l.driver.cacheStepData.screenResults[imagePath]
+			screenResult := l.driver.cacheStepData.screenResults[imageResult.imagePath]
 			screenResult.Tags = []string{"live"}
 
 			// check live type and incr live count
@@ -365,12 +365,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			return errors.Wrap(code.InterruptError, "feed crawler interrupted")
 		default:
 			// take screenshot and get screen texts by OCR
-			imagePath, texts, err := dExt.GetScreenTextsByOCR()
+			imageResult, err := dExt.GetScreenResult()
 			if err != nil {
 				log.Error().Err(err).Msg("OCR GetTexts failed")
 				continue
 			}
-			screenResult := dExt.cacheStepData.screenResults[imagePath]
+			screenResult := dExt.cacheStepData.screenResults[imageResult.imagePath]
 
 			// automatic handling of pop-up windows
 			if err := dExt.autoPopupHandler(screenResult); err != nil {
@@ -379,6 +379,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			}
 
 			// check if live video && run live crawler
+			texts := imageResult.OCRResult.ToOCRTexts()
 			if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
 				log.Info().Msg("live video found")
 				if !liveCrawler.currentStat.isLiveTargetAchieved() {