feat: get current feed info from app event trackings

hrp/pkg/uixt/service_vedem.go

@@ -343,6 +343,19 @@ type IImageService interface {
 
 // GetScreenResult takes a screenshot, returns the image recognization result
 func (dExt *DriverExt) GetScreenResult() (screenResult *ScreenResult, err error) {
+	if dExt.plugin != nil {
+		// get screen info from app event trackings
+		if feedVideo, err := getCurrentFeedVideo(dExt.plugin); err == nil && feedVideo != nil {
+			screenResult = &ScreenResult{
+				Feed:  feedVideo,
+				Texts: nil,
+				Tags:  nil,
+			}
+			dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
+			return screenResult, nil
+		}
+	}
+
 	startTime := time.Now()
 	var bufSource *bytes.Buffer
 	var imagePath string

hrp/pkg/uixt/video_crawler.go

@@ -87,45 +87,51 @@ func (s *VideoStat) isTargetAchieved() bool {
 // incrFeed increases feed count and feed stat
 func (s *VideoStat) incrFeed(screenResult *ScreenResult, driverExt *DriverExt) error {
 	screenResult.VideoType = "feed"
-	screenResult.Feed = &FeedVideo{}
 
-	// find feed author
+	var author string
-	actionOptions := []ActionOption{
+	if screenResult.Texts != nil {
-		WithRegex(true),
+		// handle screenshot
-		driverExt.GenAbsScope(0, 0.5, 1, 1).Option(),
+		// find feed author
-	}
-	ocrText, err := screenResult.Texts.FindText("^@", actionOptions...)
-	if err != nil {
-		return errors.Wrap(err, "find feed author failed")
-	}
-	author := fmt.Sprintf("@%s", removeNonAlphanumeric(ocrText.Text))
-	log.Info().Str("author", author).Msg("found feed author by OCR")
-	screenResult.Feed.UserName = author
-
-	// find target labels
-	for _, targetLabel := range s.configs.Feed.TargetLabels {
-		scope := targetLabel.Scope
 		actionOptions := []ActionOption{
-			WithRegex(targetLabel.Regex),
+			WithRegex(true),
-			driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(),
+			driverExt.GenAbsScope(0, 0.5, 1, 1).Option(),
 		}
-		if _, err := screenResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil {
+		ocrText, err := screenResult.Texts.FindText("^@", actionOptions...)
-			key := targetLabel.Text
+		if err != nil {
-			if _, ok := s.FeedStat[key]; !ok {
+			return errors.Wrap(err, "find feed author failed")
-				s.FeedStat[key] = 0
+		}
+		author = fmt.Sprintf("@%s", removeNonAlphanumeric(ocrText.Text))
+		log.Info().Str("author", author).Msg("found feed author by OCR")
+
+		// find target labels
+		for _, targetLabel := range s.configs.Feed.TargetLabels {
+			scope := targetLabel.Scope
+			actionOptions := []ActionOption{
+				WithRegex(targetLabel.Regex),
+				driverExt.GenAbsScope(scope[0], scope[1], scope[2], scope[3]).Option(),
+			}
+			if _, err := screenResult.Texts.FindText(targetLabel.Text, actionOptions...); err == nil {
+				key := targetLabel.Text
+				if _, ok := s.FeedStat[key]; !ok {
+					s.FeedStat[key] = 0
+				}
+				s.FeedStat[key]++
+				screenResult.Tags = append(screenResult.Tags, key)
 			}
-			s.FeedStat[key]++
-			screenResult.Tags = append(screenResult.Tags, key)
 		}
 	}
 
-	// get feed trackings by author
+	if screenResult.Feed == nil {
-	if driverExt.plugin != nil {
+		// get feed trackings by author
-		feedVideo, err := getFeedVideo(driverExt.plugin, author)
+		if driverExt.plugin != nil {
-		if err != nil {
+			feedVideo, err := getFeedVideo(driverExt.plugin, author)
-			return errors.Wrap(err, "get feed video from plugin failed")
+			if err != nil {
+				return errors.Wrap(err, "get feed video from plugin failed")
+			}
+			screenResult.Feed = feedVideo
+		} else {
+			screenResult.Feed = &FeedVideo{}
 		}
-		screenResult.Feed = feedVideo
 	}
 
 	// get simulation play duration
@@ -520,3 +526,37 @@ type LiveRoom struct {
 	// 记录仿真决策信息
 	WatchDuration int64 `json:"watch_duration"` // 观看时长(ms)
 }
+
+func getCurrentFeedVideo(plugin funplugin.IPlugin) (feedVideo *FeedVideo, err error) {
+	if !plugin.Has("GetCurrentFeedVideo") {
+		return nil, errors.New("plugin missing GetCurrentFeedVideo method")
+	}
+
+	// FIXME: wait for cache update
+	time.Sleep(1000 * time.Millisecond)
+
+	resp, err := plugin.Call("GetCurrentFeedVideo")
+	if err != nil {
+		return nil, errors.Wrap(err, "call plugin GetCurrentFeedVideo failed")
+	}
+
+	if resp == nil {
+		return nil, errors.New("feed not found")
+	}
+
+	feedBytes, err := json.Marshal(resp)
+	if err != nil {
+		return nil, errors.New("json marshal feed video info failed")
+	}
+
+	feedVideo = &FeedVideo{}
+	err = json.Unmarshal(feedBytes, feedVideo)
+	if err != nil {
+		return nil, errors.Wrap(err, "json unmarshal feed video info failed")
+	}
+
+	log.Info().
+		Interface("feedVideoCaption", feedVideo.Caption).
+		Msg("get current feed video success")
+	return feedVideo, nil
+}