From bb52abf5795eb1ccc0dab003d5694e336edcaeb4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 30 Apr 2023 01:17:40 +0800 Subject: [PATCH] refactor: VideoCrawlerConfigs struct --- .../uitest/demo_android_video_crawler.json | 14 +++-- .../uitest/demo_android_video_crawler_test.go | 11 ++-- hrp/pkg/uixt/action.go | 3 +- hrp/pkg/uixt/video_crawler.go | 56 +++++++++++++------ hrp/pkg/uixt/video_crawler_test.go | 11 +++- 5 files changed, 65 insertions(+), 30 deletions(-) diff --git a/examples/uitest/demo_android_video_crawler.json b/examples/uitest/demo_android_video_crawler.json index 380a6377..94f060a2 100644 --- a/examples/uitest/demo_android_video_crawler.json +++ b/examples/uitest/demo_android_video_crawler.json @@ -19,7 +19,8 @@ "method": "video_crawler", "params": { "app_package_name": "com.ss.android.ugc.aweme", - "sleep_random": [ + "feed": { + "sleep_random": [ 0, 5, 0.7, @@ -27,9 +28,14 @@ 10, 0.3 ], - "target_count": { - "feed_count": 5, - "live_count": 3 + "target_count": 5 +}, + "live": { + "sleep_random": [ + 15, + 20 +], + "target_count": 3 } } } diff --git a/examples/uitest/demo_android_video_crawler_test.go b/examples/uitest/demo_android_video_crawler_test.go index c6f81bc0..9655808b 100644 --- a/examples/uitest/demo_android_video_crawler_test.go +++ b/examples/uitest/demo_android_video_crawler_test.go @@ -21,11 +21,14 @@ func TestAndroidVideoCrawlerTest(t *testing.T) { Android(). VideoCrawler(map[string]interface{}{ "app_package_name": "com.ss.android.ugc.aweme", - "target_count": map[string]interface{}{ - "feed_count": 5, - "live_count": 3, + "feed": map[string]interface{}{ + "target_count": 5, + "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, + }, + "live": map[string]interface{}{ + "target_count": 3, + "sleep_random": []float64{15, 20}, }, - "sleep_random": []float64{0, 5, 0.7, 5, 10, 0.3}, }), hrp.NewStep("exit"). Android(). diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index d80126f2..7518c327 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -417,7 +417,8 @@ func sleepRandom(params []interface{}) error { accProb += s.weight / totalProb if r < accProb { n := s.min + rand.Float64()*(s.max-s.min) - log.Info().Float64("duration", n).Msg("sleep random seconds") + log.Info().Float64("duration", n). + Interface("strategy_params", params).Msg("sleep random seconds") time.Sleep(time.Duration(n*1000) * time.Millisecond) return nil } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 8082f5a9..02e8324e 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -10,36 +10,49 @@ import ( ) type VideoStat struct { + configs *VideoCrawlerConfigs + FeedCount int `json:"feed_count"` LiveCount int `json:"live_count"` } -func (s *VideoStat) isFeedTargetAchieved(target *VideoStat) bool { +func (s *VideoStat) isFeedTargetAchieved() bool { log.Info(). Int("count", s.FeedCount). - Int("target", target.FeedCount). + Int("target", s.configs.Feed.TargetCount). Msg("current feed count") - return s.FeedCount >= target.FeedCount + return s.FeedCount >= s.configs.Feed.TargetCount } -func (s *VideoStat) isLiveTargetAchieved(target *VideoStat) bool { +func (s *VideoStat) isLiveTargetAchieved() bool { log.Info(). Int("count", s.LiveCount). - Int("target", target.LiveCount). + Int("target", s.configs.Live.TargetCount). Msg("current live count") - return s.LiveCount >= target.LiveCount + return s.LiveCount >= s.configs.Live.TargetCount } -func (s *VideoStat) isTargetAchieved(target *VideoStat) bool { - return s.isFeedTargetAchieved(target) && s.isLiveTargetAchieved(target) +func (s *VideoStat) isTargetAchieved() bool { + return s.isFeedTargetAchieved() && s.isLiveTargetAchieved() +} + +type FeedConfig struct { + TargetCount int `json:"target_count"` + SleepRandom []interface{} `json:"sleep_random"` +} + +type LiveConfig struct { + TargetCount int `json:"target_count"` + SleepRandom []interface{} `json:"sleep_random"` } type VideoCrawlerConfigs struct { AppPackageName string `json:"app_package_name"` - TargetCount VideoStat `json:"target_count"` + Feed FeedConfig `json:"feed"` + Live LiveConfig `json:"live"` } var androidActivities = map[string]map[string]string{ @@ -83,7 +96,7 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { } time.Sleep(5 * time.Second) - for !l.currentStat.isLiveTargetAchieved(&l.configs.TargetCount) { + for !l.currentStat.isLiveTargetAchieved() { // check if live room if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { return err @@ -100,14 +113,17 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { // swipe to next live video err = l.driver.SwipeUp() - // TODO: sleep custom random time - time.Sleep(15 * time.Second) if err != nil { log.Error().Err(err).Msg("swipe up failed") // TODO: retry maximum 3 times continue } + // sleep custom random time + if err := sleepRandom(l.configs.Live.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } + // TODO: check live type l.currentStat.LiveCount++ @@ -130,7 +146,7 @@ func (l *LiveCrawler) exitLiveRoom() error { } // exit live room failed, while video count achieved - if l.currentStat.isTargetAchieved(&l.configs.TargetCount) { + if l.currentStat.isTargetAchieved() { return nil } @@ -138,7 +154,9 @@ func (l *LiveCrawler) exitLiveRoom() error { } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { - currVideoStat := &VideoStat{} + currVideoStat := &VideoStat{ + configs: configs, + } defer func() { dExt.cacheStepData.VideoStat = currVideoStat }() @@ -179,7 +197,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // check if live video && run live crawler if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") - if !liveCrawler.currentStat.isLiveTargetAchieved(&configs.TargetCount) { + if !liveCrawler.currentStat.isLiveTargetAchieved() { if err := liveCrawler.Run(dExt, enterPoint); err != nil { log.Error().Err(err).Msg("run live crawler failed, continue") continue @@ -190,11 +208,13 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // TODO: check feed type currVideoStat.FeedCount++ - // TODO: sleep custom random time - time.Sleep(5 * time.Second) + // sleep custom random time + if err := sleepRandom(configs.Feed.SleepRandom); err != nil { + log.Error().Err(err).Msg("sleep random failed") + } // check if target count achieved - if currVideoStat.isTargetAchieved(&configs.TargetCount) { + if currVideoStat.isTargetAchieved() { log.Info().Msg("target count achieved, exit crawler") break } diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index 1a40e429..e6f78c18 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -9,9 +9,14 @@ func TestVideoCrawler(t *testing.T) { configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", - TargetCount: VideoStat{ - FeedCount: 5, - LiveCount: 3, + + Feed: FeedConfig{ + TargetCount: 5, + SleepRandom: []interface{}{0, 5, 0.7, 5, 10, 0.3}, + }, + Live: LiveConfig{ + TargetCount: 3, + SleepRandom: []interface{}{15, 20}, }, } err := driverExt.VideoCrawler(configs)