GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-07 00:17:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add timeout for video crawler

Browse Source
This commit is contained in:
lilong.129
2023-05-03 11:45:10 +08:00
parent d963885276
commit d27729edc6
5 changed files with 96 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/uitest/demo_android_video_crawler.json
+2 -1
View File
@@ -88,7 +88,8 @@
20 20
], ],
"target_count": 3 "target_count": 3
} },
"timeout": 600
} }
} }
] ]
examples/uitest/demo_android_video_crawler_test.go
+1
View File
@@ -21,6 +21,7 @@ func TestAndroidVideoCrawlerTest(t *testing.T) {
Android(). Android().
VideoCrawler(map[string]interface{}{ VideoCrawler(map[string]interface{}{
"app_package_name": "com.ss.android.ugc.aweme", "app_package_name": "com.ss.android.ugc.aweme",
"timeout": 600,
"feed": map[string]interface{}{ "feed": map[string]interface{}{
"target_count": 5, "target_count": 5,
"target_labels": []map[string]interface{}{ "target_labels": []map[string]interface{}{
hrp/internal/code/code.go
+2
View File
@@ -44,6 +44,7 @@ var (
InitPluginFailed = errors.New("init plugin failed") // 31 InitPluginFailed = errors.New("init plugin failed") // 31
BuildGoPluginFailed = errors.New("build go plugin failed") // 32 BuildGoPluginFailed = errors.New("build go plugin failed") // 32
BuildPyPluginFailed = errors.New("build py plugin failed") // 33 BuildPyPluginFailed = errors.New("build py plugin failed") // 33
TimeoutError = errors.New("timeout error") // 39
) )
// summary: [40, 50) // summary: [40, 50)
@@ -111,6 +112,7 @@ var errorsMap = map[error]int{
InitPluginFailed: 31, InitPluginFailed: 31,
BuildGoPluginFailed: 32, BuildGoPluginFailed: 32,
BuildPyPluginFailed: 33, BuildPyPluginFailed: 33,
TimeoutError: 39,
// ios related // ios related
IOSDeviceConnectionError: 50, IOSDeviceConnectionError: 50,
hrp/pkg/uixt/video_crawler.go
+90 -74
View File
@@ -7,10 +7,13 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/code"
) )
type VideoStat struct { type VideoStat struct {
configs *VideoCrawlerConfigs configs *VideoCrawlerConfigs
timer *time.Timer
FeedCount int `json:"feed_count"` FeedCount int `json:"feed_count"`
FeedStat map[string]int `json:"feed_stat"` // 分类统计 feed 数量：视频/图文/广告/特效/模板/购物 FeedStat map[string]int `json:"feed_stat"` // 分类统计 feed 数量：视频/图文/广告/特效/模板/购物
@@ -95,6 +98,7 @@ type LiveConfig struct {
type VideoCrawlerConfigs struct { type VideoCrawlerConfigs struct {
AppPackageName string `json:"app_package_name"` AppPackageName string `json:"app_package_name"`
Timeout int `json:"timeout"` // seconds
Feed FeedConfig `json:"feed"` Feed FeedConfig `json:"feed"`
Live LiveConfig `json:"live"` Live LiveConfig `json:"live"`
@@ -142,36 +146,41 @@ func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
for !l.currentStat.isLiveTargetAchieved() { for !l.currentStat.isLiveTargetAchieved() {
// check if live room select {
if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { case <-l.currentStat.timer.C:
return err return errors.Wrap(code.TimeoutError, "timeout in live crawler")
default:
// check if live room
if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil {
return err
}
// take screenshot and get screen texts by OCR
_, err := l.driver.GetScreenTextsByOCR()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
continue
}
// TODO: check live type
// swipe to next live video
err = l.driver.SwipeUp()
if err != nil {
log.Error().Err(err).Msg("swipe up failed")
// TODO: retry maximum 3 times
continue
}
// sleep custom random time
if err := sleepRandom(l.configs.Live.SleepRandom); err != nil {
log.Error().Err(err).Msg("sleep random failed")
}
// TODO: check live type
l.currentStat.LiveCount++
} }
// take screenshot and get screen texts by OCR
_, err := l.driver.GetScreenTextsByOCR()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
continue
}
// TODO: check live type
// swipe to next live video
err = l.driver.SwipeUp()
if err != nil {
log.Error().Err(err).Msg("swipe up failed")
// TODO: retry maximum 3 times
continue
}
// sleep custom random time
if err := sleepRandom(l.configs.Live.SleepRandom); err != nil {
log.Error().Err(err).Msg("sleep random failed")
}
// TODO: check live type
l.currentStat.LiveCount++
} }
log.Info().Msg("live count achieved, exit live room") log.Info().Msg("live count achieved, exit live room")
@@ -231,64 +240,71 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
currentStat: currVideoStat, currentStat: currVideoStat,
} }
// loop until target count achieved // loop until target count achieved or timeout
// the main loop is feed crawler // the main loop is feed crawler
currVideoStat.timer = time.NewTimer(time.Duration(configs.Timeout) * time.Second)
for { for {
// check if feed page select {
if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { case <-currVideoStat.timer.C:
return err return errors.Wrap(code.TimeoutError, "timeout in feed crawler")
} default:
// check if feed page
if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil {
return err
}
// take screenshot and get screen texts by OCR // take screenshot and get screen texts by OCR
texts, err := dExt.GetScreenTextsByOCR() texts, err := dExt.GetScreenTextsByOCR()
if err != nil { if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed") log.Error().Err(err).Msg("OCR GetTexts failed")
continue continue
} }
// automatic handling of pop-up windows // automatic handling of pop-up windows
if err := dExt.autoPopupHandler(texts); err != nil { if err := dExt.autoPopupHandler(texts); err != nil {
log.Error().Err(err).Msg("auto handle popup failed") log.Error().Err(err).Msg("auto handle popup failed")
return err return err
} }
// check if live video && run live crawler // check if live video && run live crawler
if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
log.Info().Msg("live video found") log.Info().Msg("live video found")
if !liveCrawler.currentStat.isLiveTargetAchieved() { if !liveCrawler.currentStat.isLiveTargetAchieved() {
if err := liveCrawler.Run(dExt, enterPoint); err != nil { if err := liveCrawler.Run(dExt, enterPoint); err != nil {
log.Error().Err(err).Msg("run live crawler failed, continue") if errors.Is(err, code.TimeoutError) {
continue return err
}
log.Error().Err(err).Msg("run live crawler failed, continue")
continue
}
} }
} }
}
// check feed type and incr feed count // check feed type and incr feed count
if err := currVideoStat.incrFeed(texts, dExt); err != nil { if err := currVideoStat.incrFeed(texts, dExt); err != nil {
log.Error().Err(err).Msg("incr feed failed") log.Error().Err(err).Msg("incr feed failed")
} }
// sleep custom random time // sleep custom random time
if err := sleepRandom(configs.Feed.SleepRandom); err != nil { if err := sleepRandom(configs.Feed.SleepRandom); err != nil {
log.Error().Err(err).Msg("sleep random failed") log.Error().Err(err).Msg("sleep random failed")
} }
// check if target count achieved // check if target count achieved
if currVideoStat.isTargetAchieved() { if currVideoStat.isTargetAchieved() {
log.Info().Msg("target count achieved, exit crawler") log.Info().Msg("target count achieved, exit crawler")
break return nil
} }
// swipe to next feed video // swipe to next feed video
log.Info().Msg("swipe to next feed video") log.Info().Msg("swipe to next feed video")
if err = dExt.SwipeUp(); err != nil { if err = dExt.SwipeUp(); err != nil {
log.Error().Err(err).Msg("swipe up failed") log.Error().Err(err).Msg("swipe up failed")
return err return err
}
time.Sleep(1 * time.Second)
} }
time.Sleep(1 * time.Second)
} }
return nil
} }
func (dExt *DriverExt) assertActivity(packageName, activityType string) error { func (dExt *DriverExt) assertActivity(packageName, activityType string) error {
hrp/pkg/uixt/video_crawler_test.go
+1
View File
@@ -9,6 +9,7 @@ func TestVideoCrawler(t *testing.T) {
configs := &VideoCrawlerConfigs{ configs := &VideoCrawlerConfigs{
AppPackageName: "com.ss.android.ugc.aweme", AppPackageName: "com.ss.android.ugc.aweme",
Timeout: 600,
Feed: FeedConfig{ Feed: FeedConfig{
TargetCount: 5, TargetCount: 5,