From 68dc545f35d371885b227206df12f50b3db0f139 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 28 Apr 2023 21:48:59 +0800 Subject: [PATCH] feat: implement video crawler --- hrp/pkg/uixt/android_adb_driver.go | 19 ++- hrp/pkg/uixt/android_test.go | 7 +- hrp/pkg/uixt/demo/main_test.go | 11 +- hrp/pkg/uixt/interface.go | 14 ++- hrp/pkg/uixt/ios_driver.go | 4 +- hrp/pkg/uixt/video_crawler.go | 191 ++++++++++++++++++++++++----- hrp/pkg/uixt/video_crawler_test.go | 8 +- 7 files changed, 208 insertions(+), 46 deletions(-) diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index d26abffe..988b3c76 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -367,22 +367,22 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error { return errors.New("package name is not given") } - foreApp, err := ad.GetForegroundApp() + app, err := ad.GetForegroundApp() if err != nil { return err } - if foreApp != packageName { + if app.BundleId != packageName { return errors.New("app is not in foreground") } return nil } -func (ad *adbDriver) GetForegroundApp() (packageName string, err error) { +func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) { // adb shell dumpsys activity activities | grep mResumedActivity output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities") if err != nil { log.Error().Err(err).Msg("failed to dumpsys activities") - return "", errors.Wrap(code.AndroidShellExecError, err.Error()) + return AppInfo{}, errors.Wrap(code.AndroidShellExecError, err.Error()) } lines := strings.Split(string(output), "\n") @@ -394,11 +394,18 @@ func (ad *adbDriver) GetForegroundApp() (packageName string, err error) { for _, str := range strs { if strings.Contains(str, "/") { // com.android.settings/.Settings - return strings.Split(str, "/")[0], nil + s := strings.Split(str, "/") + app := AppInfo{ + AppBaseInfo: AppBaseInfo{ + BundleId: s[0], + Activity: s[1], + }, + } + return app, nil } } } } - return "", errors.New("get foreground app failed") + return AppInfo{}, errors.New("get foreground app failed") } diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index 34ba6d10..a5061ee9 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -355,9 +355,12 @@ func TestDriver_IsAppInForeground(t *testing.T) { err := driverExt.Driver.AppLaunch("com.android.settings") checkErr(t, err) - foreApp, err := driverExt.Driver.GetForegroundApp() + app, err := driverExt.Driver.GetForegroundApp() checkErr(t, err) - if foreApp != "com.android.settings" { + if app.BundleId != "com.android.settings" { + t.FailNow() + } + if app.Activity != ".Settings" { t.FailNow() } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 1ff036ce..b2610e0e 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/rs/zerolog/log" + "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" ) @@ -32,7 +34,14 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) + // take screenshot and get screen texts by OCR + texts, err := driverExt.GetScreenTextsByOCR() + if err != nil { + log.Error().Err(err).Msg("OCR GetTexts failed") + t.Fatal(err) + } + + points, err := texts.FindTexts([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index e2e56b24..95d71611 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -251,8 +251,9 @@ type AppInfo struct { } type AppBaseInfo struct { - Pid int `json:"pid"` - BundleId string `json:"bundleId"` + Pid int `json:"pid,omitempty"` + BundleId string `json:"bundleId"` // package name for android + Activity string // view controller for ios } type AppState int @@ -579,6 +580,11 @@ type Device interface { StopPcap() string } +type ForegroundApp struct { + PackageName string + Activity string +} + // WebDriver defines methods supported by WebDriver drivers. type WebDriver interface { // NewSession starts a new session and returns the SessionInfo. @@ -623,8 +629,8 @@ type WebDriver interface { GetLastLaunchedApp() string // AssertAppForeground returns nil if the given package is in foreground AssertAppForeground(packageName string) error - // GetForegroundApp returns current foreground app package name - GetForegroundApp() (string, error) + // GetForegroundApp returns current foreground app package name and activity name + GetForegroundApp() (app AppInfo, err error) // StartCamera Starts a new camera for recording StartCamera() error diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index be8eecee..6acd6a99 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -373,8 +373,8 @@ func (wd *wdaDriver) AssertAppForeground(packageName string) error { return nil } -func (wd *wdaDriver) GetForegroundApp() (string, error) { - return "", nil +func (wd *wdaDriver) GetForegroundApp() (app AppInfo, err error) { + return AppInfo{}, nil } func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index edcd3090..8f86a809 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -1,36 +1,138 @@ package uixt import ( + "fmt" + "strings" "time" "github.com/pkg/errors" "github.com/rs/zerolog/log" - - "github.com/httprunner/httprunner/v4/hrp/internal/code" ) +type VideoStat struct { + FeedCount int `json:"feed_count"` + LiveCount int `json:"live_count"` +} + +func (s *VideoStat) IsTargetAchieved(target *VideoStat) bool { + log.Info(). + Interface("current", s). + Interface("target", target). + Msg("current video stat") + if s.FeedCount < target.FeedCount { + return false + } + if s.LiveCount < target.LiveCount { + return false + } + return true +} + type VideoCrawlerConfigs struct { AppPackageName string `json:"app_package_name"` - TargetFeedCount int `json:"target_feed_count"` - TargetLiveCount int `json:"target_live_count"` + TargetCount VideoStat `json:"target_count"` +} + +var androidActivities = map[string]map[string]string{ + // DY + "com.ss.android.ugc.aweme": { + "feed": ".splash.SplashActivity", + "live": ".live.LivePlayActivity", + }, + // KS + "com.smile.gifmaker": { + "feed": "com.yxcorp.gifshow.HomeActivity", + "live": "com.kuaishou.live.core.basic.activity.LiveSlideActivity", + }, +} + +type LiveCrawler struct { + driver *DriverExt + configs *VideoCrawlerConfigs // target video count + currentStat *VideoStat // current video stat +} + +func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) { + // 预览流入口 + points, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) + if err == nil { + return points[0], true + } + + // TODO: 头像入口 + + return PointF{}, false +} + +// run live video crawler +func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error { + log.Info().Msg("enter live room") + if err := driver.TapAbsXY(enterPoint.X, enterPoint.Y); err != nil { + log.Error().Err(err).Msg("tap live video failed") + return err + } + time.Sleep(5 * time.Second) + + for l.currentStat.LiveCount < l.configs.TargetCount.LiveCount { + // check if entered live room + if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil { + return err + } + + log.Info(). + Int("count", l.currentStat.LiveCount). + Int("target", l.configs.TargetCount.LiveCount). + Msg("current live count") + + // swipe to next live video + err := l.driver.SwipeUp() + if err != nil { + log.Error().Err(err).Msg("swipe up failed") + return err + } + time.Sleep(2 * time.Second) + l.currentStat.LiveCount++ + } + + log.Info().Msg("live count achieved, exit live room") + + return l.exitLiveRoom() +} + +func (l *LiveCrawler) exitLiveRoom() error { + // FIXME: exit live room + for i := 0; i < 5; i++ { + l.driver.SwipeRelative(0, 0.5, 0.5, 0.5) + time.Sleep(2 * time.Second) + + // check if back to feed page + if err := l.driver.assertActivity(l.configs.AppPackageName, "feed"); err == nil { + return nil + } + } + return errors.New("exit live room failed") } func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // launch app - if configs.AppPackageName != "" { - if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { - return err - } - time.Sleep(5 * time.Second) + if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil { + return err + } + time.Sleep(5 * time.Second) + + currVideoStat := &VideoStat{} + liveCrawler := LiveCrawler{ + driver: dExt, + configs: configs, + currentStat: currVideoStat, } // loop until target count achieved + // the main loop is feed crawler for { - // check if app in foreground - if err := dExt.Driver.AssertAppForeground(configs.AppPackageName); err != nil { - log.Error().Err(err).Str("packageName", configs.AppPackageName).Msg("app is not in foreground") - err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error()) + // check if feed page + if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil { return err } @@ -41,36 +143,69 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return err } - // check if text popup exists - if isTextPopup(texts) { - log.Info().Msg("text popup found") + // automatic handling of pop-up windows + if err := dExt.autoPopupHandler(texts); err != nil { + log.Error().Err(err).Msg("auto handle popup failed") + return err } - // check if live video - if isLiveVideo(texts) { + // check if live video && run live crawler + if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive { log.Info().Msg("live video found") + if liveCrawler.currentStat.LiveCount < configs.TargetCount.LiveCount { + if err := liveCrawler.Run(dExt, enterPoint); err != nil { + return err + } + } } - // assert feed video type + // check if target count achieved + if currVideoStat.IsTargetAchieved(&configs.TargetCount) { + log.Info().Msg("target count achieved, exit crawler") + break + } - // swipe to next video + // swipe to next feed video + log.Info().Msg("swipe to next feed video") if err = dExt.SwipeUp(); err != nil { log.Error().Err(err).Msg("swipe up failed") return err } - time.Sleep(5 * time.Second) + currVideoStat.FeedCount++ } - // return nil + return nil } -func isTextPopup(texts OCRTexts) bool { +func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error { + log.Debug().Str("pacakge_name", pacakgeName). + Str("activity_type", activityType).Msg("assert activity") + app, err := dExt.Driver.GetForegroundApp() + if err != nil { + log.Error().Err(err).Msg("get foreground app failed") + return err + } + + if app.BundleId != pacakgeName { + return fmt.Errorf("app %s is not in foreground", pacakgeName) + } + + if activities, ok := androidActivities[app.BundleId]; ok { + if activity, ok := activities[activityType]; ok { + if strings.HasSuffix(app.Activity, activity) { + return nil + } + } + } + + log.Error().Interface("app", app.AppBaseInfo).Msg("app activity not match") + return fmt.Errorf("%s activity is not in foreground", activityType) +} + +func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error { texts.FindTexts([]string{"确定", "取消"}) - return false -} -func isLiveVideo(texts OCRTexts) bool { - _, err := texts.FindTexts([]string{"点击进入直播间", "直播中"}) - return err == nil + // log.Warn().Msg("text popup found") + return nil } diff --git a/hrp/pkg/uixt/video_crawler_test.go b/hrp/pkg/uixt/video_crawler_test.go index d849395e..1a40e429 100644 --- a/hrp/pkg/uixt/video_crawler_test.go +++ b/hrp/pkg/uixt/video_crawler_test.go @@ -9,9 +9,11 @@ func TestVideoCrawler(t *testing.T) { configs := &VideoCrawlerConfigs{ AppPackageName: "com.ss.android.ugc.aweme", + TargetCount: VideoStat{ + FeedCount: 5, + LiveCount: 3, + }, } err := driverExt.VideoCrawler(configs) - if err != nil { - t.Fatal(err) - } + checkErr(t, err) }