feat: get feed video info by funplugin

hrp/pkg/uixt/action.go

@@ -3,7 +3,6 @@ package uixt
 import (
 	"encoding/json"
 	"fmt"
-	"math"
 	"math/rand"
 	"time"
 
@@ -533,7 +532,8 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
 		return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
 	case ACTION_SleepRandom:
 		if params, ok := action.Params.([]interface{}); ok {
-			return sleepRandom(time.Now(), params)
+			sleepStrict(time.Now(), getSimulationDuration(params))
+			return nil
 		}
 		return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
 	case ACTION_ScreenShot:
@@ -575,13 +575,20 @@ func convertToFloat64(val interface{}) (float64, error) {
 	}
 }
 
-// sleepRandom sleeps random time with given params
+// getSimulationDuration returns simulation duration by given params (in seconds)
-// startTime is used to correct sleep duration caused by process time
+func getSimulationDuration(params []interface{}) (milliseconds int64) {
-func sleepRandom(startTime time.Time, params []interface{}) error {
 	if len(params) == 1 {
-		// constant sleep time
+		// given constant duration time
-		params = append(params, params[0], 1.0)
+		seconds, err := convertToFloat64(params[0])
-	} else if len(params) == 2 {
+		if err != nil {
+			log.Error().Err(err).Interface("params", params).Msg("invalid params")
+			return 0
+		}
+		return int64(seconds * 1000)
+	}
+
+	if len(params) == 2 {
+		// given [min, max], missing weight
 		// append default weight 1
 		params = append(params, 1.0)
 	}
@@ -593,15 +600,18 @@ func sleepRandom(startTime time.Time, params []interface{}) error {
 	for i := 0; i+3 <= len(params); i += 3 {
 		min, err := convertToFloat64(params[i])
 		if err != nil {
-			return errors.Wrapf(err, "invalid minimum time: %v", params[i])
+			log.Error().Err(err).Interface("min", params[i]).Msg("invalid minimum time")
+			return 0
 		}
 		max, err := convertToFloat64(params[i+1])
 		if err != nil {
-			return errors.Wrapf(err, "invalid maximum time: %v", params[i+1])
+			log.Error().Err(err).Interface("max", params[i+1]).Msg("invalid maximum time")
+			return 0
 		}
 		weight, err := convertToFloat64(params[i+2])
 		if err != nil {
-			return errors.Wrapf(err, "invalid weight value: %v", params[i+2])
+			log.Error().Err(err).Interface("weight", params[i+2]).Msg("invalid weight value")
+			return 0
 		}
 		totalProb += weight
 		sections = append(sections,
@@ -610,8 +620,8 @@ func sleepRandom(startTime time.Time, params []interface{}) error {
 	}
 
 	if totalProb == 0 {
-		log.Warn().Msg("total weight is 0, skip sleep")
+		log.Warn().Msg("total weight is 0, skip simulation")
-		return nil
+		return 0
 	}
 
 	r := rand.Float64()
@@ -619,22 +629,36 @@ func sleepRandom(startTime time.Time, params []interface{}) error {
 	for _, s := range sections {
 		accProb += s.weight / totalProb
 		if r < accProb {
-			elapsed := time.Since(startTime).Seconds()
+			seconds := s.min + rand.Float64()*(s.max-s.min)
-			randomSeconds := s.min + rand.Float64()*(s.max-s.min)
+			log.Info().Float64("randomSeconds", seconds).
-			dur := randomSeconds - elapsed
+				Interface("strategy_params", params).Msg("get simulation duration")
-
+			return int64(seconds * 1000)
-			// if elapsed time is greater than random seconds, skip sleep to reduce deviation caused by process time
-			if dur <= 0 {
-				log.Info().Float64("elapsed", elapsed).Float64("randomSeconds", randomSeconds).
-					Interface("strategy_params", params).Msg("elapsed duration >= random seconds, skip sleep")
-			} else {
-				log.Info().Float64("sleepDuration", dur).Float64("elapsed", elapsed).Float64("randomSeconds", randomSeconds).
-					Interface("strategy_params", params).Msg("sleep remaining random seconds")
-				time.Sleep(time.Duration(math.Ceil(dur*1000)) * time.Millisecond)
-			}
-
-			return nil
 		}
 	}
-	return nil
+
+	log.Warn().Interface("strategy_params", params).
+		Msg("get simulation duration failed, skip simulation")
+	return 0
+}
+
+// sleepStrict sleeps strict duration with given params
+// startTime is used to correct sleep duration caused by process time
+func sleepStrict(startTime time.Time, strictMilliseconds int64) {
+	elapsed := time.Since(startTime).Milliseconds()
+	dur := strictMilliseconds - elapsed
+
+	// if elapsed time is greater than given duration, skip sleep to reduce deviation caused by process time
+	if dur <= 0 {
+		log.Info().
+			Int64("elapsed(ms)", elapsed).
+			Int64("strictSleep(ms)", strictMilliseconds).
+			Msg("elapsed >= simulation duration, skip sleep")
+		return
+	}
+
+	log.Info().Int64("sleepDuration(ms)", dur).
+		Int64("elapsed(ms)", elapsed).
+		Int64("strictSleep(ms)", strictMilliseconds).
+		Msg("sleep remaining duration time")
+	time.Sleep(time.Duration(dur) * time.Millisecond)
 }

hrp/pkg/uixt/action_test.go

@@ -15,33 +15,32 @@ func checkErr(t *testing.T, err error, msg ...string) {
 	}
 }
 
-func TestSleepRandom(t *testing.T) {
+func TestGetSimulationDuration(t *testing.T) {
-	startTime1 := time.Now()
+	params := []interface{}{1.23}
-	params := []interface{}{1}
+	duration := getSimulationDuration(params)
-	err := sleepRandom(startTime1, params)
+	if duration != 1230 {
-	checkErr(t, err)
+		t.Fatal("getSimulationDuration failed")
-	dur := time.Since(startTime1).Seconds()
-	t.Log(dur)
-	if dur < 1 || dur > 1.1 {
-		t.Fatal("sleepRandom failed")
 	}
 
-	params = []interface{}{0, 2}
-	err = sleepRandom(startTime1, params)
-	checkErr(t, err)
-	dur = time.Since(startTime1).Seconds()
-	t.Log(dur)
-	if dur < 1 || dur > 2 {
-		t.Fatal("sleepRandom failed")
-	}
-
-	startTime2 := time.Now()
 	params = []interface{}{1, 2}
-	err = sleepRandom(startTime2, params)
+	duration = getSimulationDuration(params)
-	checkErr(t, err)
+	if duration < 1000 || duration > 2000 {
-	dur = time.Since(startTime2).Seconds()
+		t.Fatal("getSimulationDuration failed")
+	}
+
+	params = []interface{}{1, 5, 0.7, 5, 10, 0.3}
+	duration = getSimulationDuration(params)
+	if duration < 1000 || duration > 10000 {
+		t.Fatal("getSimulationDuration failed")
+	}
+}
+
+func TestSleepStrict(t *testing.T) {
+	startTime := time.Now()
+	sleepStrict(startTime, 1230)
+	dur := time.Since(startTime).Milliseconds()
 	t.Log(dur)
-	if dur < 1 || dur > 2 {
+	if dur < 1230 || dur > 1232 {
 		t.Fatal("sleepRandom failed")
 	}
 }

hrp/pkg/uixt/ext.go

@@ -50,18 +50,12 @@ func WithThreshold(threshold float64) CVOption {
 	}
 }
 
-type Popularity struct {
-	Stars     string `json:"stars,omitempty"`      // 点赞数
-	Comments  string `json:"comments,omitempty"`   // 评论数
-	Favorites string `json:"favorites,omitempty"`  // 收藏数
-	Shares    string `json:"shares,omitempty"`     // 分享数
-	LiveUsers string `json:"live_users,omitempty"` // 直播间人数
-}
-
 type ScreenResult struct {
-	Texts      OCRTexts   `json:"texts"`      // dumped OCRTexts
+	Texts     OCRTexts   `json:"texts"`                // dumped raw OCRTexts
-	Tags       []string   `json:"tags"`       // tags for image, e.g. ["feed", "ad", "live"]
+	Tags      []string   `json:"tags"`                 // tags for image, e.g. ["feed", "ad", "live"]
-	Popularity Popularity `json:"popularity"` // video popularity data
+	VideoType string     `json:"video_type,omitempty"` // video type: feed, live-preview or live
+	Feed      *FeedVideo `json:"feed,omitempty"`
+	Live      *LiveRoom  `json:"live,omitempty"`
 }
 
 type cacheStepData struct {
@@ -227,9 +221,8 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} {
 	for imagePath, screenResult := range dExt.cacheStepData.screenResults {
 		o, _ := json.Marshal(screenResult.Texts)
 		data := map[string]interface{}{
-			"tags":       screenResult.Tags,
+			"tags":  screenResult.Tags,
-			"texts":      string(o),
+			"texts": string(o),
-			"popularity": screenResult.Popularity,
 			"resolution": map[string]int{
 				"width":  screenSize.Width,
 				"height": screenSize.Height,

hrp/pkg/uixt/service_vedem.go

@@ -364,12 +364,14 @@ func (dExt *DriverExt) GetScreenResult() (screenResult *ScreenResult, err error)
 	}
 
 	screenResult = &ScreenResult{
-		Texts:      imageResult.OCRResult.ToOCRTexts(),
+		Texts: imageResult.OCRResult.ToOCRTexts(),
-		Tags:       nil,
+		Tags:  nil,
-		Popularity: Popularity{},
 	}
 	if imageResult.LiveType != "" {
-		screenResult.Tags = []string{imageResult.LiveType}
+		screenResult.VideoType = "live"
+		screenResult.Live = &LiveRoom{
+			LiveType: imageResult.LiveType,
+		}
 	}
 	dExt.cacheStepData.screenResults[imagePath] = screenResult
 

hrp/pkg/uixt/video_crawler.go

@@ -4,10 +4,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/httprunner/funplugin"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
 )
 
 type VideoStat struct {
@@ -82,16 +84,24 @@ func (s *VideoStat) isTargetAchieved() bool {
 
 // incrFeed increases feed count and feed stat
 func (s *VideoStat) incrFeed(screenResult *ScreenResult, driverExt *DriverExt) error {
-	// feed author
+	screenResult.VideoType = "feed"
+
+	// find feed author
 	actionOptions := []ActionOption{
 		WithRegex(true),
 		driverExt.GenAbsScope(0, 0.5, 1, 1).Option(),
 	}
-	if ocrText, err := screenResult.Texts.FindText("^@", actionOptions...); err == nil {
+	ocrText, err := screenResult.Texts.FindText("^@", actionOptions...)
-		log.Debug().Str("author", ocrText.Text).Msg("found feed author")
+	if err != nil {
-		screenResult.Tags = append(screenResult.Tags, ocrText.Text)
+		return errors.Wrap(err, "find feed author failed")
+	}
+	author := ocrText.Text
+	log.Info().Str("author", author).Msg("found feed author by OCR")
+	screenResult.Feed = &FeedVideo{
+		UserName: author,
 	}
 
+	// find target labels
 	for _, targetLabel := range s.configs.Feed.TargetLabels {
 		scope := targetLabel.Scope
 		actionOptions := []ActionOption{
@@ -108,21 +118,23 @@ func (s *VideoStat) incrFeed(screenResult *ScreenResult, driverExt *DriverExt) e
 		}
 	}
 
-	// add popularity data for feed
+	// get feed trackings by author
-	popularityData := screenResult.Texts.FilterScope(driverExt.GenAbsScope(0.8, 0.5, 1, 0.8))
+	if driverExt.plugin != nil {
-	if len(popularityData) != 4 {
+		feedVideo, err := getFeedVideo(driverExt.plugin, author)
-		log.Warn().Interface("popularity", popularityData).Msg("get feed popularity data failed")
+		if err != nil {
-	} else {
+			log.Error().Err(err).Msg("get feed video from plugin failed")
-		screenResult.Popularity = Popularity{
+			return err
-			Stars:     popularityData[0].Text,
-			Comments:  popularityData[1].Text,
-			Favorites: popularityData[2].Text,
-			Shares:    popularityData[3].Text,
 		}
+		screenResult.Feed = feedVideo
+	}
+
+	// get simulation play duration
+	if screenResult.Feed.PlayDuration == 0 {
+		screenResult.Feed.PlayDuration = getSimulationDuration(s.configs.Feed.SleepRandom)
 	}
 
 	log.Info().Strs("tags", screenResult.Tags).
-		Interface("popularity", screenResult.Popularity).
+		Interface("feed", screenResult.Feed).
 		Msg("found feed success")
 	s.FeedCount++
 	return nil
@@ -130,20 +142,19 @@ func (s *VideoStat) incrFeed(screenResult *ScreenResult, driverExt *DriverExt) e
 
 // incrLive increases live count and live stat
 func (s *VideoStat) incrLive(screenResult *ScreenResult, driverExt *DriverExt) error {
+	screenResult.VideoType = "live"
 	// TODO: check live type
 
-	// add popularity data for live
+	if screenResult.Live == nil {
-	popularityData := screenResult.Texts.FilterScope(driverExt.GenAbsScope(0.7, 0.05, 1, 0.15))
+		screenResult.Live = &LiveRoom{}
-	if len(popularityData) != 1 {
-		log.Warn().Interface("popularity", popularityData).Msg("get live popularity data failed")
-	} else {
-		screenResult.Popularity = Popularity{
-			LiveUsers: popularityData[0].Text,
-		}
 	}
 
+	// TODO: add popularity data for live
+
+	screenResult.Live.WatchDuration = getSimulationDuration(s.configs.Live.SleepRandom)
+
 	log.Info().Strs("tags", screenResult.Tags).
-		Interface("popularity", screenResult.Popularity).
+		Interface("live", screenResult.Live).
 		Msg("found live success")
 	s.LiveCount++
 	return nil
@@ -221,6 +232,7 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
 		return err
 	}
 	time.Sleep(5 * time.Second)
+	lastSwipeTime := time.Now()
 
 	for !l.currentStat.isLiveTargetAchieved() {
 		select {
@@ -231,24 +243,6 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
 			log.Warn().Msg("interrupted in live crawler")
 			return errors.Wrap(code.InterruptError, "live crawler interrupted")
 		default:
-			// check if live room
-			if err := l.driver.Driver.AssertForegroundApp(l.configs.AppPackageName, "live"); err != nil {
-				return err
-			}
-
-			// swipe to next live video
-			err := l.driver.SwipeUp()
-			if err != nil {
-				log.Error().Err(err).Msg("swipe up failed")
-				// TODO: retry maximum 3 times
-				continue
-			}
-
-			// sleep custom random time
-			if err := sleepRandom(time.Now(), l.configs.Live.SleepRandom); err != nil {
-				log.Error().Err(err).Msg("sleep random failed")
-			}
-
 			// take screenshot and get screen texts by OCR
 			screenResult, err := l.driver.GetScreenResult()
 			if err != nil {
@@ -256,12 +250,28 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
 				time.Sleep(3 * time.Second)
 				continue
 			}
-			screenResult.Tags = append([]string{"live"}, screenResult.Tags...)
 
 			// check live type and incr live count
 			if err := l.currentStat.incrLive(screenResult, l.driver); err != nil {
 				log.Error().Err(err).Msg("incr live failed")
 			}
+
+			// simulation watch live video
+			sleepStrict(lastSwipeTime, screenResult.Live.WatchDuration)
+
+			// swipe to next live video
+			err = l.driver.SwipeUp()
+			if err != nil {
+				log.Error().Err(err).Msg("swipe up failed")
+				// TODO: retry maximum 3 times
+				continue
+			}
+			lastSwipeTime = time.Now()
+
+			// check if live room
+			if err := l.driver.Driver.AssertForegroundApp(l.configs.AppPackageName, "live"); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -388,19 +398,18 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 						continue
 					}
 				}
-				screenResult.Tags = []string{"live-preview"}
+				// 直播预览流
+				screenResult.VideoType = "live-preview"
 			} else {
-				screenResult.Tags = []string{"feed"}
+				// 点播
-
 				// check feed type and incr feed count
 				if err := currVideoStat.incrFeed(screenResult, dExt); err != nil {
 					log.Error().Err(err).Msg("incr feed failed")
+					continue
 				}
-			}
 
-			// sleep custom random time
+				// simulation watch feed video
-			if err := sleepRandom(lastSwipeTime, configs.Feed.SleepRandom); err != nil {
+				sleepStrict(lastSwipeTime, screenResult.Feed.PlayDuration)
-				log.Error().Err(err).Msg("sleep random failed")
 			}
 
 			// check if target count achieved
@@ -424,3 +433,53 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 		}
 	}
 }
+
+func getFeedVideo(plugin funplugin.IPlugin, authorName string) (feedVideo *FeedVideo, err error) {
+	if !plugin.Has("GetFeedVideo") {
+		return nil, errors.New("plugin missing GetFeedVideo method")
+	}
+
+	resp, err := plugin.Call("GetFeedVideo", authorName)
+	if err != nil {
+		return nil, errors.Wrap(err, "call plugin GetFeedVideo failed")
+	}
+
+	feedBytes, err := json.Marshal(resp)
+	if err != nil {
+		return nil, errors.New("json marshal feed video info failed")
+	}
+
+	err = json.Unmarshal(feedBytes, &feedVideo)
+	if err != nil {
+		return nil, errors.Wrap(err, "json unmarshal feed video info failed")
+	}
+
+	log.Info().Interface("feedVideo", feedVideo).Msg("get feed video success")
+	return feedVideo, nil
+}
+
+type FeedVideo struct {
+	// 视频基础数据
+	UserName string `json:"user_name"` // 视频作者
+	Duration int64  `json:"duration"`  // 视频时长(ms)
+	Caption  string `json:"caption"`   // 视频文案
+	// 视频热度数据
+	ViewCount    int64 `json:"view_count"`    // feed 观看数
+	LikeCount    int64 `json:"like_count"`    // feed 点赞数
+	CommentCount int64 `json:"comment_count"` // feed 评论数
+	CollectCount int64 `json:"collect_count"` // feed 收藏数
+	ForwardCount int64 `json:"forward_count"` // feed 转发数
+	ShareCount   int64 `json:"share_count"`   // feed 分享数
+	// 记录仿真决策信息
+	PlayDuration int64 `json:"play_duration"` // 播放时长(ms)
+}
+
+type LiveRoom struct {
+	// 视频基础数据
+	UserName string `json:"user_name"` // 主播名
+	LiveType string `json:"live_type"` // 直播间类型
+	// 直播热度数据
+	LiveUsers string `json:"live_users"` // 直播间人数
+	// 记录仿真决策信息
+	WatchDuration int64 `json:"watch_duration"` // 观看时长(ms)
+}