feat: implement video crawler

hrp/pkg/uixt/android_adb_driver.go

@@ -367,22 +367,22 @@ func (ad *adbDriver) AssertAppForeground(packageName string) error {
 		return errors.New("package name is not given")
 	}
 
-	foreApp, err := ad.GetForegroundApp()
+	app, err := ad.GetForegroundApp()
 	if err != nil {
 		return err
 	}
-	if foreApp != packageName {
+	if app.BundleId != packageName {
 		return errors.New("app is not in foreground")
 	}
 	return nil
 }
 
-func (ad *adbDriver) GetForegroundApp() (packageName string, err error) {
+func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) {
 	// adb shell dumpsys activity activities | grep mResumedActivity
 	output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities")
 	if err != nil {
 		log.Error().Err(err).Msg("failed to dumpsys activities")
-		return "", errors.Wrap(code.AndroidShellExecError, err.Error())
+		return AppInfo{}, errors.Wrap(code.AndroidShellExecError, err.Error())
 	}
 
 	lines := strings.Split(string(output), "\n")
@@ -394,11 +394,18 @@ func (ad *adbDriver) GetForegroundApp() (packageName string, err error) {
 			for _, str := range strs {
 				if strings.Contains(str, "/") {
 					// com.android.settings/.Settings
-					return strings.Split(str, "/")[0], nil
+					s := strings.Split(str, "/")
+					app := AppInfo{
+						AppBaseInfo: AppBaseInfo{
+							BundleId: s[0],
+							Activity: s[1],
+						},
+					}
+					return app, nil
 				}
 			}
 		}
 	}
 
-	return "", errors.New("get foreground app failed")
+	return AppInfo{}, errors.New("get foreground app failed")
 }

hrp/pkg/uixt/android_test.go

@@ -355,9 +355,12 @@ func TestDriver_IsAppInForeground(t *testing.T) {
 	err := driverExt.Driver.AppLaunch("com.android.settings")
 	checkErr(t, err)
 
-	foreApp, err := driverExt.Driver.GetForegroundApp()
+	app, err := driverExt.Driver.GetForegroundApp()
 	checkErr(t, err)
-	if foreApp != "com.android.settings" {
+	if app.BundleId != "com.android.settings" {
+		t.FailNow()
+	}
+	if app.Activity != ".Settings" {
 		t.FailNow()
 	}
 

hrp/pkg/uixt/demo/main_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
 )
 
@@ -32,7 +34,14 @@ func TestIOSDemo(t *testing.T) {
 
 	// 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」
 	for {
-		points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"})
+		// take screenshot and get screen texts by OCR
+		texts, err := driverExt.GetScreenTextsByOCR()
+		if err != nil {
+			log.Error().Err(err).Msg("OCR GetTexts failed")
+			t.Fatal(err)
+		}
+
+		points, err := texts.FindTexts([]string{"青少年模式", "我知道了"})
 		if err != nil {
 			time.Sleep(1 * time.Second)
 			continue

hrp/pkg/uixt/interface.go

@@ -251,8 +251,9 @@ type AppInfo struct {
 }
 
 type AppBaseInfo struct {
-	Pid      int    `json:"pid"`
+	Pid      int    `json:"pid,omitempty"`
-	BundleId string `json:"bundleId"`
+	BundleId string `json:"bundleId"` // package name for android
+	Activity string // view controller for ios
 }
 
 type AppState int
@@ -579,6 +580,11 @@ type Device interface {
 	StopPcap() string
 }
 
+type ForegroundApp struct {
+	PackageName string
+	Activity    string
+}
+
 // WebDriver defines methods supported by WebDriver drivers.
 type WebDriver interface {
 	// NewSession starts a new session and returns the SessionInfo.
@@ -623,8 +629,8 @@ type WebDriver interface {
 	GetLastLaunchedApp() string
 	// AssertAppForeground returns nil if the given package is in foreground
 	AssertAppForeground(packageName string) error
-	// GetForegroundApp returns current foreground app package name
+	// GetForegroundApp returns current foreground app package name and activity name
-	GetForegroundApp() (string, error)
+	GetForegroundApp() (app AppInfo, err error)
 
 	// StartCamera Starts a new camera for recording
 	StartCamera() error

hrp/pkg/uixt/ios_driver.go

@@ -373,8 +373,8 @@ func (wd *wdaDriver) AssertAppForeground(packageName string) error {
 	return nil
 }
 
-func (wd *wdaDriver) GetForegroundApp() (string, error) {
+func (wd *wdaDriver) GetForegroundApp() (app AppInfo, err error) {
-	return "", nil
+	return AppInfo{}, nil
 }
 
 func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error {

hrp/pkg/uixt/video_crawler.go

@@ -1,36 +1,138 @@
 package uixt
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
-
-	"github.com/httprunner/httprunner/v4/hrp/internal/code"
 )
 
+type VideoStat struct {
+	FeedCount int `json:"feed_count"`
+	LiveCount int `json:"live_count"`
+}
+
+func (s *VideoStat) IsTargetAchieved(target *VideoStat) bool {
+	log.Info().
+		Interface("current", s).
+		Interface("target", target).
+		Msg("current video stat")
+	if s.FeedCount < target.FeedCount {
+		return false
+	}
+	if s.LiveCount < target.LiveCount {
+		return false
+	}
+	return true
+}
+
 type VideoCrawlerConfigs struct {
 	AppPackageName string `json:"app_package_name"`
 
-	TargetFeedCount int `json:"target_feed_count"`
+	TargetCount VideoStat `json:"target_count"`
-	TargetLiveCount int `json:"target_live_count"`
+}
+
+var androidActivities = map[string]map[string]string{
+	// DY
+	"com.ss.android.ugc.aweme": {
+		"feed": ".splash.SplashActivity",
+		"live": ".live.LivePlayActivity",
+	},
+	// KS
+	"com.smile.gifmaker": {
+		"feed": "com.yxcorp.gifshow.HomeActivity",
+		"live": "com.kuaishou.live.core.basic.activity.LiveSlideActivity",
+	},
+}
+
+type LiveCrawler struct {
+	driver      *DriverExt
+	configs     *VideoCrawlerConfigs // target video count
+	currentStat *VideoStat           // current video stat
+}
+
+func (l *LiveCrawler) checkLiveVideo(texts OCRTexts) (enterPoint PointF, yes bool) {
+	// 预览流入口
+	points, err := texts.FindTexts([]string{"点击进入直播间", "直播中"})
+	if err == nil {
+		return points[0], true
+	}
+
+	// TODO: 头像入口
+
+	return PointF{}, false
+}
+
+// run live video crawler
+func (l *LiveCrawler) Run(driver *DriverExt, enterPoint PointF) error {
+	log.Info().Msg("enter live room")
+	if err := driver.TapAbsXY(enterPoint.X, enterPoint.Y); err != nil {
+		log.Error().Err(err).Msg("tap live video failed")
+		return err
+	}
+	time.Sleep(5 * time.Second)
+
+	for l.currentStat.LiveCount < l.configs.TargetCount.LiveCount {
+		// check if entered live room
+		if err := l.driver.assertActivity(l.configs.AppPackageName, "live"); err != nil {
+			return err
+		}
+
+		log.Info().
+			Int("count", l.currentStat.LiveCount).
+			Int("target", l.configs.TargetCount.LiveCount).
+			Msg("current live count")
+
+		// swipe to next live video
+		err := l.driver.SwipeUp()
+		if err != nil {
+			log.Error().Err(err).Msg("swipe up failed")
+			return err
+		}
+		time.Sleep(2 * time.Second)
+		l.currentStat.LiveCount++
+	}
+
+	log.Info().Msg("live count achieved, exit live room")
+
+	return l.exitLiveRoom()
+}
+
+func (l *LiveCrawler) exitLiveRoom() error {
+	// FIXME: exit live room
+	for i := 0; i < 5; i++ {
+		l.driver.SwipeRelative(0, 0.5, 0.5, 0.5)
+		time.Sleep(2 * time.Second)
+
+		// check if back to feed page
+		if err := l.driver.assertActivity(l.configs.AppPackageName, "feed"); err == nil {
+			return nil
+		}
+	}
+	return errors.New("exit live room failed")
 }
 
 func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 	// launch app
-	if configs.AppPackageName != "" {
+	if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil {
-		if err = dExt.Driver.AppLaunch(configs.AppPackageName); err != nil {
+		return err
-			return err
+	}
-		}
+	time.Sleep(5 * time.Second)
-		time.Sleep(5 * time.Second)
+
+	currVideoStat := &VideoStat{}
+	liveCrawler := LiveCrawler{
+		driver:      dExt,
+		configs:     configs,
+		currentStat: currVideoStat,
 	}
 
 	// loop until target count achieved
+	// the main loop is feed crawler
 	for {
-		// check if app in foreground
+		// check if feed page
-		if err := dExt.Driver.AssertAppForeground(configs.AppPackageName); err != nil {
+		if err := dExt.assertActivity(configs.AppPackageName, "feed"); err != nil {
-			log.Error().Err(err).Str("packageName", configs.AppPackageName).Msg("app is not in foreground")
-			err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error())
 			return err
 		}
 
@@ -41,36 +143,69 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			return err
 		}
 
-		// check if text popup exists
+		// automatic handling of pop-up windows
-		if isTextPopup(texts) {
+		if err := dExt.autoPopupHandler(texts); err != nil {
-			log.Info().Msg("text popup found")
+			log.Error().Err(err).Msg("auto handle popup failed")
+			return err
 		}
 
-		// check if live video
+		// check if live video && run live crawler
-		if isLiveVideo(texts) {
+		if enterPoint, isLive := liveCrawler.checkLiveVideo(texts); isLive {
 			log.Info().Msg("live video found")
+			if liveCrawler.currentStat.LiveCount < configs.TargetCount.LiveCount {
+				if err := liveCrawler.Run(dExt, enterPoint); err != nil {
+					return err
+				}
+			}
 		}
 
-		// assert feed video type
+		// check if target count achieved
+		if currVideoStat.IsTargetAchieved(&configs.TargetCount) {
+			log.Info().Msg("target count achieved, exit crawler")
+			break
+		}
 
-		// swipe to next video
+		// swipe to next feed video
+		log.Info().Msg("swipe to next feed video")
 		if err = dExt.SwipeUp(); err != nil {
 			log.Error().Err(err).Msg("swipe up failed")
 			return err
 		}
-
 		time.Sleep(5 * time.Second)
+		currVideoStat.FeedCount++
 	}
 
-	// return nil
+	return nil
 }
 
-func isTextPopup(texts OCRTexts) bool {
+func (dExt *DriverExt) assertActivity(pacakgeName, activityType string) error {
+	log.Debug().Str("pacakge_name", pacakgeName).
+		Str("activity_type", activityType).Msg("assert activity")
+	app, err := dExt.Driver.GetForegroundApp()
+	if err != nil {
+		log.Error().Err(err).Msg("get foreground app failed")
+		return err
+	}
+
+	if app.BundleId != pacakgeName {
+		return fmt.Errorf("app %s is not in foreground", pacakgeName)
+	}
+
+	if activities, ok := androidActivities[app.BundleId]; ok {
+		if activity, ok := activities[activityType]; ok {
+			if strings.HasSuffix(app.Activity, activity) {
+				return nil
+			}
+		}
+	}
+
+	log.Error().Interface("app", app.AppBaseInfo).Msg("app activity not match")
+	return fmt.Errorf("%s activity is not in foreground", activityType)
+}
+
+func (dExt *DriverExt) autoPopupHandler(texts OCRTexts) error {
 	texts.FindTexts([]string{"确定", "取消"})
-	return false
-}
 
-func isLiveVideo(texts OCRTexts) bool {
+	// log.Warn().Msg("text popup found")
-	_, err := texts.FindTexts([]string{"点击进入直播间", "直播中"})
+	return nil
-	return err == nil
 }

hrp/pkg/uixt/video_crawler_test.go

@@ -9,9 +9,11 @@ func TestVideoCrawler(t *testing.T) {
 
 	configs := &VideoCrawlerConfigs{
 		AppPackageName: "com.ss.android.ugc.aweme",
+		TargetCount: VideoStat{
+			FeedCount: 5,
+			LiveCount: 3,
+		},
 	}
 	err := driverExt.VideoCrawler(configs)
-	if err != nil {
+	checkErr(t, err)
-		t.Fatal(err)
-	}
 }