Merge pull request #1685 from httprunner/video-crawler

video crawler

docs/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Release History
 
+## v4.3.7 (2023-09-19)
+
+**go version**
+
+- feat: add `WithSwipeOffset` to set offset for swipe start/end point
+- feat: set random offset for tap/swipe points with `WithOffsetRandomRange`
+- change: set `WithOffset` deprecated, replace with `WithTapOffset`
+
 ## v4.3.6 (2023-09-07)
 
 **go version**

examples/uitest/demo_douyin_follow_live_test.go

@@ -39,8 +39,8 @@ func TestIOSDouyinFollowLive(t *testing.T) {
 				TapByOCR("关注", uixt.WithIndex(1)).Sleep(10),
 			hrp.NewStep("向上滑动 2 次").
 				IOS().SwipeToTapTexts([]string{"理肤泉", "婉宝"}, uixt.WithCustomDirection(0.6, 0.2, 0.2, 0.2), uixt.WithIdentifier("click_live")).Sleep(10).
-				Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(). // 上划 1 次，等待 10s，截图保存
+				Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live"), uixt.WithOffsetRandomRange(-10, 10)).Sleep(10).ScreenShot(). // 上划 1 次，等待 10s，截图保存
-				Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(), // 再上划 1 次，等待 10s，截图保存
+				Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live"), uixt.WithOffsetRandomRange(-10, 10)).Sleep(10).ScreenShot(), // 再上划 1 次，等待 10s，截图保存
 		},
 	}
 

examples/uitest/expert_test.go

@@ -111,7 +111,7 @@ func TestAndroidExpertTest(t *testing.T) {
 			hrp.NewStep("home 以及 swipe_to_tap_app 自定义配置").
 				Android().
 				Home().
-				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)).
+				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithTapOffset(0, -50)).
 				Sleep(10),
 			hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言").
 				Android().
@@ -265,7 +265,7 @@ func TestIOSExpertTest(t *testing.T) {
 			hrp.NewStep("home 以及 swipe_to_tap_app 自定义配置").
 				IOS().
 				Home().
-				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)).
+				SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithTapOffset(0, -50)).
 				Sleep(10),
 			hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言").
 				IOS().

hrp/internal/builtin/utils.go

@@ -452,7 +452,7 @@ func GenNameWithTimestamp(tmpl string) string {
 }
 
 func IsZeroFloat64(f float64) bool {
-	threshold := 1e-3
+	threshold := 1e-9
 	return math.Abs(f) < threshold
 }
 

hrp/internal/code/code.go

@@ -78,16 +78,21 @@ var (
 	MobileUIAssertForegroundAppError      = errors.New("mobile UI assert foreground app error")      // 76
 	MobileUIAssertForegroundActivityError = errors.New("mobile UI assert foreground activity error") // 77
 	MobileUIPopupError                    = errors.New("mobile UI popup error")                      // 78
+	LoopActionNotFoundError               = errors.New("loop action not found error")                // 79
 )
 
 // CV related: [80, 90)
 var (
-	CVEnvMissedError         = errors.New("CV env missed error")         // 80
+	CVEnvMissedError         = errors.New("CV env missed error")      // 80
-	CVRequestError           = errors.New("CV prepare request error")    // 81
+	CVRequestError           = errors.New("CV prepare request error") // 81
-	CVServiceConnectionError = errors.New("CV service connect error")    // 82
+	CVServiceConnectionError = errors.New("CV service connect error") // 82
-	CVResponseError          = errors.New("CV parse response error")     // 83
+	CVResponseError          = errors.New("CV parse response error")  // 83
-	CVResultNotFoundError    = errors.New("CV result not found")         // 84
+	CVResultNotFoundError    = errors.New("CV result not found")      // 84
-	LoopActionNotFoundError  = errors.New("loop action not found error") // 85
+)
+
+// trackings related: [90, 100)
+var (
+	TrackingGetError = errors.New("get trackings failed") // 90
 )
 
 var errorsMap = map[error]int{
@@ -141,6 +146,7 @@ var errorsMap = map[error]int{
 	MobileUIAssertForegroundAppError:      76,
 	MobileUIAssertForegroundActivityError: 77,
 	MobileUIPopupError:                    78,
+	LoopActionNotFoundError:               79,
 
 	// OCR related
 	CVEnvMissedError:         80,
@@ -148,7 +154,9 @@ var errorsMap = map[error]int{
 	CVServiceConnectionError: 82,
 	CVResponseError:          83,
 	CVResultNotFoundError:    84,
-	LoopActionNotFoundError:  85,
+
+	// trackings related
+	TrackingGetError: 90,
 }
 
 func IsErrorPredefined(err error) bool {

hrp/internal/sdk/ga4.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -176,7 +176,7 @@ func (g *GA4Client) SendEvent(event Event) error {
 		return nil
 	}
 
-	bs, err = ioutil.ReadAll(res.Body)
+	bs, err = io.ReadAll(res.Body)
 	if err != nil {
 		return errors.Wrap(err, "read GA4 response body failed")
 	}

hrp/pkg/gadb/transport.go

@@ -3,7 +3,6 @@ package gadb
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"regexp"
 	"strconv"
@@ -97,7 +96,7 @@ func (t transport) ReadStringAll() (s string, err error) {
 }
 
 func (t transport) ReadBytesAll() (raw []byte, err error) {
-	raw, err = ioutil.ReadAll(t.sock)
+	raw, err = io.ReadAll(t.sock)
 	return
 }
 

hrp/pkg/gidevice/pkg/libimobiledevice/pcapd.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"time"
 
@@ -109,7 +108,7 @@ func (c *PcapdClient) GetPacket(buf []byte) ([]byte, error) {
 		}
 	}
 
-	packet, err := ioutil.ReadAll(preader)
+	packet, err := io.ReadAll(preader)
 	if err != nil {
 		return packet, err
 	}

hrp/pkg/uixt/action.go

@@ -106,10 +106,11 @@ type ActionOptions struct {
 	Scope    Scope    `json:"scope,omitempty" yaml:"scope,omitempty"`
 	AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"`
 
-	Regex    bool  `json:"regex,omitempty" yaml:"regex,omitempty"`         // use regex to match text
+	Regex             bool  `json:"regex,omitempty" yaml:"regex,omitempty"`                             // use regex to match text
-	Offset   []int `json:"offset,omitempty" yaml:"offset,omitempty"`       // used to tap offset of point
+	Offset            []int `json:"offset,omitempty" yaml:"offset,omitempty"`                           // used to tap offset of point
-	Index    int   `json:"index,omitempty" yaml:"index,omitempty"`         // index of the target element
+	OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points
-	MatchOne bool  `json:"match_one,omitempty" yaml:"match_one,omitempty"` // match one of the targets if existed
+	Index             int   `json:"index,omitempty" yaml:"index,omitempty"`                             // index of the target element
+	MatchOne          bool  `json:"match_one,omitempty" yaml:"match_one,omitempty"`                     // match one of the targets if existed
 
 	// set custiom options such as textview, id, description
 	Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"`
@@ -181,8 +182,18 @@ func (o *ActionOptions) Options() []ActionOption {
 			o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3]))
 	}
 	if len(o.Offset) == 2 {
-		options = append(options, WithOffset(o.Offset[0], o.Offset[1]))
+		// for tap [x,y] offset
+		options = append(options, WithTapOffset(o.Offset[0], o.Offset[1]))
+	} else if len(o.Offset) == 4 {
+		// for swipe [fromX, fromY, toX, toY] offset
+		options = append(options, WithSwipeOffset(
+			o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3]))
 	}
+	if len(o.OffsetRandomRange) == 2 {
+		options = append(options, WithOffsetRandomRange(
+			o.OffsetRandomRange[0], o.OffsetRandomRange[1]))
+	}
+
 	if o.Regex {
 		options = append(options, WithRegex(true))
 	}
@@ -238,52 +249,41 @@ func (o *ActionOptions) screenshotActions() []string {
 	return actions
 }
 
-func NewActionOptions(options ...ActionOption) *ActionOptions {
+func (o *ActionOptions) getRandomOffset() float64 {
-	actionOptions := &ActionOptions{}
+	if len(o.OffsetRandomRange) != 2 {
-	for _, option := range options {
+		// invalid offset random range, should be [min, max]
-		option(actionOptions)
+		return 0
 	}
-	return actionOptions
+
+	minOffset := o.OffsetRandomRange[0]
+	maxOffset := o.OffsetRandomRange[1]
+	return float64(builtin.GetRandomNumber(minOffset, maxOffset)) + rand.Float64()
 }
 
-func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} {
+func (o *ActionOptions) updateData(data map[string]interface{}) {
-	actionOptions := NewActionOptions(options...)
+	if o.Identifier != "" {
-
-	if actionOptions.Identifier != "" {
 		data["log"] = map[string]interface{}{
 			"enable": true,
-			"data":   actionOptions.Identifier,
+			"data":   o.Identifier,
 		}
 	}
 
-	// handle point offset
+	if o.Steps > 0 {
-	if len(actionOptions.Offset) == 2 {
+		data["steps"] = o.Steps
-		if x, ok := data["x"]; ok {
-			xf, _ := builtin.Interface2Float64(x)
-			data["x"] = xf + float64(actionOptions.Offset[0])
-		}
-		if y, ok := data["y"]; ok {
-			yf, _ := builtin.Interface2Float64(y)
-			data["y"] = yf + float64(actionOptions.Offset[1])
-		}
-	}
-
-	if actionOptions.Steps > 0 {
-		data["steps"] = actionOptions.Steps
 	}
 	if _, ok := data["steps"]; !ok {
 		data["steps"] = 12 // default steps
 	}
 
-	if actionOptions.PressDuration > 0 {
+	if o.PressDuration > 0 {
-		data["duration"] = actionOptions.PressDuration
+		data["duration"] = o.PressDuration
 	}
 	if _, ok := data["duration"]; !ok {
 		data["duration"] = 0 // default duration
 	}
 
-	if actionOptions.Frequency > 0 {
+	if o.Frequency > 0 {
-		data["frequency"] = actionOptions.Frequency
+		data["frequency"] = o.Frequency
 	}
 	if _, ok := data["frequency"]; !ok {
 		data["frequency"] = 60 // default frequency
@@ -294,13 +294,19 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption)
 	}
 
 	// custom options
-	if actionOptions.Custom != nil {
+	if o.Custom != nil {
-		for k, v := range actionOptions.Custom {
+		for k, v := range o.Custom {
 			data[k] = v
 		}
 	}
+}
 
-	return data
+func NewActionOptions(options ...ActionOption) *ActionOptions {
+	actionOptions := &ActionOptions{}
+	for _, option := range options {
+		option(actionOptions)
+	}
+	return actionOptions
 }
 
 type ActionOption func(o *ActionOptions)
@@ -377,12 +383,29 @@ func WithAbsScope(x1, y1, x2, y2 int) ActionOption {
 	}
 }
 
+// Deprecated: use WithTapOffset instead
 func WithOffset(offsetX, offsetY int) ActionOption {
 	return func(o *ActionOptions) {
 		o.Offset = []int{offsetX, offsetY}
 	}
 }
 
+// tap [x, y] with offset [offsetX, offsetY]
+var WithTapOffset = WithOffset
+
+// swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY]
+func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption {
+	return func(o *ActionOptions) {
+		o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY}
+	}
+}
+
+func WithOffsetRandomRange(min, max int) ActionOption {
+	return func(o *ActionOptions) {
+		o.OffsetRandomRange = []int{min, max}
+	}
+}
+
 func WithRegex(regex bool) ActionOption {
 	return func(o *ActionOptions) {
 		o.Regex = regex

hrp/pkg/uixt/android_adb_driver.go

@@ -228,6 +228,8 @@ func (ad *adbDriver) TapFloat(x, y float64, options ...ActionOption) (err error)
 		x += float64(actionOptions.Offset[0])
 		y += float64(actionOptions.Offset[1])
 	}
+	x += actionOptions.getRandomOffset()
+	y += actionOptions.getRandomOffset()
 
 	// adb shell input tap x y
 	xStr := fmt.Sprintf("%.1f", x)
@@ -272,6 +274,19 @@ func (ad *adbDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption)
 }
 
 func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error {
+	actionOptions := NewActionOptions(options...)
+
+	if len(actionOptions.Offset) == 4 {
+		fromX += float64(actionOptions.Offset[0])
+		fromY += float64(actionOptions.Offset[1])
+		toX += float64(actionOptions.Offset[2])
+		toY += float64(actionOptions.Offset[3])
+	}
+	fromX += actionOptions.getRandomOffset()
+	fromY += actionOptions.getRandomOffset()
+	toX += actionOptions.getRandomOffset()
+	toY += actionOptions.getRandomOffset()
+
 	// adb shell input swipe fromX fromY toX toY
 	_, err := ad.adbClient.RunShellCommand(
 		"input", "swipe",

hrp/pkg/uixt/android_device.go

@@ -95,11 +95,19 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er
 	}
 
 	dev := deviceList[0]
-	device.SerialNumber = dev.Serial()
+
+	if device.SerialNumber == "" {
+		selectSerial := dev.Serial()
+		device.SerialNumber = selectSerial
+		log.Warn().
+			Str("serial", device.SerialNumber).
+			Msg("android SerialNumber is not specified, select the first one")
+	}
+
 	device.d = dev
 	device.logcat = NewAdbLogcat(device.SerialNumber)
 
-	log.Info().Str("serial", device.SerialNumber).Msg("select android device")
+	log.Info().Str("serial", device.SerialNumber).Msg("init android device")
 	return device, nil
 }
 

hrp/pkg/uixt/android_uia2_driver.go

@@ -259,14 +259,23 @@ func (ud *uiaDriver) Tap(x, y int, options ...ActionOption) error {
 
 func (ud *uiaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) {
 	// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
+	actionOptions := NewActionOptions(options...)
+
+	if len(actionOptions.Offset) == 2 {
+		x += float64(actionOptions.Offset[0])
+		y += float64(actionOptions.Offset[1])
+	}
+	x += actionOptions.getRandomOffset()
+	y += actionOptions.getRandomOffset()
+
 	data := map[string]interface{}{
 		"x": x,
 		"y": y,
 	}
-	// new data options in post data for extra uiautomator configurations
+	// update data options in post data for extra uiautomator configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err = ud.httpPOST(newData, "/session", ud.sessionId, "appium/tap")
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap")
 	return
 }
 
@@ -299,6 +308,18 @@ func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e
 }
 
 func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) {
+	actionOptions := NewActionOptions(options...)
+	if len(actionOptions.Offset) == 4 {
+		fromX += float64(actionOptions.Offset[0])
+		fromY += float64(actionOptions.Offset[1])
+		toX += float64(actionOptions.Offset[2])
+		toY += float64(actionOptions.Offset[3])
+	}
+	fromX += actionOptions.getRandomOffset()
+	fromY += actionOptions.getRandomOffset()
+	toX += actionOptions.getRandomOffset()
+	toY += actionOptions.getRandomOffset()
+
 	data := map[string]interface{}{
 		"startX": fromX,
 		"startY": fromY,
@@ -306,11 +327,11 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...Action
 		"endY":   toY,
 	}
 
-	// new data options in post data for extra uiautomator configurations
+	// update data options in post data for extra uiautomator configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
 	// register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag"))
-	_, err = ud.httpPOST(newData, "/session", ud.sessionId, "touch/drag")
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/drag")
 	return
 }
 
@@ -325,6 +346,18 @@ func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption)
 
 func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error {
 	// register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform"))
+	actionOptions := NewActionOptions(options...)
+	if len(actionOptions.Offset) == 4 {
+		fromX += float64(actionOptions.Offset[0])
+		fromY += float64(actionOptions.Offset[1])
+		toX += float64(actionOptions.Offset[2])
+		toY += float64(actionOptions.Offset[3])
+	}
+	fromX += actionOptions.getRandomOffset()
+	fromY += actionOptions.getRandomOffset()
+	toX += actionOptions.getRandomOffset()
+	toY += actionOptions.getRandomOffset()
+
 	data := map[string]interface{}{
 		"startX": fromX,
 		"startY": fromY,
@@ -332,10 +365,10 @@ func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio
 		"endY":   toY,
 	}
 
-	// new data options in post data for extra uiautomator configurations
+	// update data options in post data for extra uiautomator configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err := ud.httpPOST(newData, "/session", ud.sessionId, "touch/perform")
+	_, err := ud.httpPOST(data, "/session", ud.sessionId, "touch/perform")
 	return err
 }
 
@@ -385,13 +418,14 @@ func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe
 func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) {
 	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
 	// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
+	actionOptions := NewActionOptions(options...)
 	data := map[string]interface{}{
 		"text": text,
 	}
 	// new data options in post data for extra uiautomator configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err = ud.httpPOST(newData, "/session", ud.sessionId, "keys")
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "keys")
 	return
 }
 

hrp/pkg/uixt/client.go

@@ -72,7 +72,7 @@ func (wd *Driver) httpRequest(method string, rawURL string, rawBody []byte) (raw
 		_ = resp.Body.Close()
 	}()
 
-	rawResp, err = ioutil.ReadAll(resp.Body)
+	rawResp, err = io.ReadAll(resp.Body)
 	logger := log.Debug().Int("statusCode", resp.StatusCode).Str("duration", time.Since(start).String())
 	if !strings.HasSuffix(rawURL, "screenshot") {
 		// avoid printing screenshot data

hrp/pkg/uixt/ext.go

@@ -2,7 +2,6 @@ package uixt
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"image"
 	"image/gif"
@@ -52,16 +51,16 @@ func WithThreshold(threshold float64) CVOption {
 }
 
 type ScreenResult struct {
-	bufSource *bytes.Buffer // raw image buffer bytes
+	bufSource   *bytes.Buffer // raw image buffer bytes
-	imagePath string        // image file path
+	imagePath   string        // image file path
+	imageResult *ImageResult  // image result
 
-	UploadedURL string      `json:"uploaded_url"`         // uploaded image url
+	Resolution  Size        `json:"resolution"`
-	Texts       OCRTexts    `json:"texts"`                // dumped raw OCRTexts
+	UploadedURL string      `json:"uploaded_url"` // uploaded image url
-	Icons       UIResultMap `json:"icons"`                // CV 识别的图标
+	Texts       OCRTexts    `json:"texts"`        // dumped raw OCRTexts
-	Tags        []string    `json:"tags"`                 // tags for image, e.g. ["feed", "ad", "live"]
+	Icons       UIResultMap `json:"icons"`        // CV 识别的图标
-	VideoType   string      `json:"video_type,omitempty"` // video type: feed, live-preview or live
+	Tags        []string    `json:"tags"`         // tags for image, e.g. ["feed", "ad", "live"]
-	Feed        *FeedVideo  `json:"feed,omitempty"`
+	Video       *Video      `json:"video,omitempty"`
-	Live        *LiveRoom   `json:"live,omitempty"`
 	Popup       *PopupInfo  `json:"popup,omitempty"`
 
 	SwipeStartTime  int64 `json:"swipe_start_time"`  // 滑动开始时间戳
@@ -74,16 +73,16 @@ type ScreenResult struct {
 	TotalElapsed int64 `json:"total_elapsed"` // current_swipe_finish -> next_swipe_start 整体耗时(ms)
 }
 
-type ScreenResultMap map[string]*ScreenResult
+type ScreenResultMap map[string]*ScreenResult // key is date time
 
 // getScreenShotUrls returns screenShotsUrls using imagePath as key and uploaded URL as value
 func (screenResults ScreenResultMap) getScreenShotUrls() map[string]string {
 	screenShotsUrls := make(map[string]string)
-	for imagePath, screenResult := range screenResults {
+	for _, screenResult := range screenResults {
 		if screenResult.UploadedURL == "" {
 			continue
 		}
-		screenShotsUrls[imagePath] = screenResult.UploadedURL
+		screenShotsUrls[screenResult.imagePath] = screenResult.UploadedURL
 	}
 	return screenShotsUrls
 }
@@ -176,7 +175,7 @@ func newDriverExt(device Device, driver WebDriver, plugin funplugin.IPlugin) (dE
 	// get device window size
 	dExt.windowSize, err = dExt.Driver.WindowSize()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "get screen resolution failed")
 	}
 
 	if dExt.ImageService, err = newVEDEMImageService(); err != nil {
@@ -275,37 +274,7 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} {
 
 	cacheData["screenshots_urls"] = dExt.cacheStepData.screenResults.getScreenShotUrls()
 	dExt.cacheStepData.screenResults.updatePopupCloseStatus()
-
+	cacheData["screen_results"] = dExt.cacheStepData.screenResults
-	screenSize, err := dExt.Driver.WindowSize()
-	if err != nil {
-		log.Warn().Err(err).Msg("get screen resolution failed")
-		screenSize = Size{}
-	}
-	screenResults := make(map[string]interface{})
-	for imagePath, screenResult := range dExt.cacheStepData.screenResults {
-		o, _ := json.Marshal(screenResult.Texts)
-		data := map[string]interface{}{
-			"tags":  screenResult.Tags,
-			"texts": string(o),
-			"resolution": map[string]int{
-				"width":  screenSize.Width,
-				"height": screenSize.Height,
-			},
-			"video_type":              screenResult.VideoType,
-			"feed":                    screenResult.Feed,
-			"live":                    screenResult.Live,
-			"swipe_start_time":        screenResult.SwipeStartTime,
-			"swipe_finish_time":       screenResult.SwipeFinishTime,
-			"screenshot_take_elapsed": screenResult.ScreenshotTakeElapsed,
-			"screenshot_cv_elapsed":   screenResult.ScreenshotCVElapsed,
-			"total_elapsed":           screenResult.TotalElapsed,
-			"icons":                   screenResult.Icons,
-			"popup":                   screenResult.Popup,
-		}
-
-		screenResults[imagePath] = data
-	}
-	cacheData["screen_results"] = screenResults
 
 	// clear cache
 	dExt.cacheStepData.reset()
@@ -370,14 +339,19 @@ func (dExt *DriverExt) AssertImage(imagePath, assert string) bool {
 }
 
 func (dExt *DriverExt) AssertForegroundApp(appName, assert string) bool {
-	var err error
+	app, err := dExt.Driver.GetForegroundApp()
+	if err != nil {
+		log.Warn().Err(err).Msg("get foreground app failed, skip app/activity assertion")
+		return true // Notice: ignore error when get foreground app failed
+	}
+	log.Debug().Interface("app", app).Msg("get foreground app")
+
+	// assert package name
 	switch assert {
 	case AssertionEqual:
-		err = dExt.Driver.AssertForegroundApp(appName)
+		return app.PackageName == appName
-		return err == nil
 	case AssertionNotEqual:
-		err = dExt.Driver.AssertForegroundApp(appName)
+		return app.PackageName != appName
-		return err != nil
 	default:
 		log.Warn().Str("assert method", assert).Msg("unexpected assert method")
 	}

hrp/pkg/uixt/ios_device.go

@@ -239,7 +239,14 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) {
 
 	dev := deviceList[0]
 	udid := dev.Properties().SerialNumber
-	device.UDID = udid
+
+	if device.UDID == "" {
+		device.UDID = udid
+		log.Warn().
+			Str("udid", udid).
+			Msg("ios UDID is not specified, select the first one")
+	}
+
 	device.d = dev
 
 	// run xctest if XCTestBundleID is set
@@ -251,7 +258,7 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) {
 		}
 	}
 
-	log.Info().Str("udid", device.UDID).Msg("select ios device")
+	log.Info().Str("udid", device.UDID).Msg("init ios device")
 	return device, nil
 }
 

hrp/pkg/uixt/ios_driver.go

@@ -461,14 +461,25 @@ func (wd *wdaDriver) Tap(x, y int, options ...ActionOption) error {
 
 func (wd *wdaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) {
 	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
-	data := map[string]interface{}{
+	actionOptions := NewActionOptions(options...)
-		"x": wd.toScale(x),
-		"y": wd.toScale(y),
-	}
-	// new data options in post data for extra WDA configurations
-	newData := mergeDataWithOptions(data, options...)
 
-	_, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/tap/0")
+	x = wd.toScale(x)
+	y = wd.toScale(y)
+	if len(actionOptions.Offset) == 2 {
+		x += float64(actionOptions.Offset[0])
+		y += float64(actionOptions.Offset[1])
+	}
+	x += actionOptions.getRandomOffset()
+	y += actionOptions.getRandomOffset()
+
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	// update data options in post data for extra WDA configurations
+	actionOptions.updateData(data)
+
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0")
 	return
 }
 
@@ -510,17 +521,34 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e
 
 func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) {
 	// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
+	actionOptions := NewActionOptions(options...)
+
+	fromX = wd.toScale(fromX)
+	fromY = wd.toScale(fromY)
+	toX = wd.toScale(toX)
+	toY = wd.toScale(toY)
+	if len(actionOptions.Offset) == 4 {
+		fromX += float64(actionOptions.Offset[0])
+		fromY += float64(actionOptions.Offset[1])
+		toX += float64(actionOptions.Offset[2])
+		toY += float64(actionOptions.Offset[3])
+	}
+	fromX += actionOptions.getRandomOffset()
+	fromY += actionOptions.getRandomOffset()
+	toX += actionOptions.getRandomOffset()
+	toY += actionOptions.getRandomOffset()
+
 	data := map[string]interface{}{
-		"fromX": wd.toScale(fromX),
+		"fromX": fromX,
-		"fromY": wd.toScale(fromY),
+		"fromY": fromY,
-		"toX":   wd.toScale(toX),
+		"toX":   toX,
-		"toY":   wd.toScale(toY),
+		"toY":   toY,
 	}
 
-	// new data options in post data for extra WDA configurations
+	// update data options in post data for extra WDA configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration")
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
 	return
 }
 
@@ -557,12 +585,13 @@ func (wd *wdaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe
 
 func (wd *wdaDriver) SendKeys(text string, options ...ActionOption) (err error) {
 	// [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)]
+	actionOptions := NewActionOptions(options...)
 	data := map[string]interface{}{"value": strings.Split(text, "")}
 
 	// new data options in post data for extra WDA configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/keys")
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys")
 	return
 }
 
@@ -572,22 +601,38 @@ func (wd *wdaDriver) Input(text string, options ...ActionOption) (err error) {
 
 // PressBack simulates a short press on the BACK button.
 func (wd *wdaDriver) PressBack(options ...ActionOption) (err error) {
+	actionOptions := NewActionOptions(options...)
+
 	windowSize, err := wd.WindowSize()
 	if err != nil {
 		return
 	}
+	fromX := wd.toScale(float64(windowSize.Width) * 0)
+	fromY := wd.toScale(float64(windowSize.Height) * 0.5)
+	toX := wd.toScale(float64(windowSize.Width) * 0.6)
+	toY := wd.toScale(float64(windowSize.Height) * 0.5)
+	if len(actionOptions.Offset) == 4 {
+		fromX += float64(actionOptions.Offset[0])
+		fromY += float64(actionOptions.Offset[1])
+		toX += float64(actionOptions.Offset[2])
+		toY += float64(actionOptions.Offset[3])
+	}
+	fromX += actionOptions.getRandomOffset()
+	fromY += actionOptions.getRandomOffset()
+	toX += actionOptions.getRandomOffset()
+	toY += actionOptions.getRandomOffset()
 
 	data := map[string]interface{}{
-		"fromX": wd.toScale(float64(windowSize.Width) * 0),
+		"fromX": fromX,
-		"fromY": wd.toScale(float64(windowSize.Height) * 0.5),
+		"fromY": fromY,
-		"toX":   wd.toScale(float64(windowSize.Width) * 0.6),
+		"toX":   toX,
-		"toY":   wd.toScale(float64(windowSize.Height) * 0.5),
+		"toY":   toY,
 	}
 
-	// new data options in post data for extra WDA configurations
+	// update data options in post data for extra WDA configurations
-	newData := mergeDataWithOptions(data, options...)
+	actionOptions.updateData(data)
 
-	_, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration")
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
 	return
 }
 

hrp/pkg/uixt/opencv.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"image"
 	"image/color"
-	"io/ioutil"
+	"io"
 	"math"
 	"os"
 
@@ -133,7 +133,7 @@ func getBufFromDisk(name string) (*bytes.Buffer, error) {
 		return nil, err
 	}
 	var all []byte
-	if all, err = ioutil.ReadAll(f); err != nil {
+	if all, err = io.ReadAll(f); err != nil {
 		return nil, err
 	}
 	return bytes.NewBuffer(all), nil
@@ -361,7 +361,7 @@ func getMatsFromDisk(nameImage, nameTpl string, flags gocv.IMReadFlag) (matImage
 // 			return nil, e
 // 		}
 // 		var all []byte
-// 		if all, e = ioutil.ReadAll(f); e != nil {
+// 		if all, e = io.ReadAll(f); e != nil {
 // 			return nil, e
 // 		}
 // 		return bytes.NewBuffer(all), nil

hrp/pkg/uixt/popups.go

@@ -114,6 +114,7 @@ func (dExt *DriverExt) ClosePopups(options ...ActionOption) error {
 }
 
 func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error {
+	log.Info().Msg("try to find and close popups")
 	actionOptions := NewActionOptions(options...)
 	maxRetryTimes := actionOptions.MaxRetryTimes
 	interval := actionOptions.Interval

hrp/pkg/uixt/service_vedem.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"image"
-	"io/ioutil"
+	"io"
 	"math"
 	"mime/multipart"
 	"net/http"
@@ -55,8 +55,8 @@ func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) {
 }
 
 type ImageResult struct {
-	URL       string     `json:"url"`       // image uploaded url
+	URL       string     `json:"url,omitempty"`       // image uploaded url
-	OCRResult OCRResults `json:"ocrResult"` // OCR texts
+	OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts
 	// NoLive（非直播间）
 	// Shop（电商）
 	// LifeService（生活服务）
@@ -67,9 +67,9 @@ type ImageResult struct {
 	// Media（媒体）
 	// Chat（语音）
 	// Event（赛事）
-	LiveType string            `json:"liveType"`    // 直播间类型
+	LiveType string             `json:"liveType,omitempty"`    // 直播间类型
-	UIResult UIResultMap       `json:"uiResult"`    // 图标检测
+	UIResult UIResultMap        `json:"uiResult,omitempty"`    // 图标检测
-	CPResult ClosePopupsResult `json:"closeResult"` // 弹窗按钮检测
+	CPResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
 }
 
 type APIResponseImage struct {
@@ -257,6 +257,10 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp
 			return
 		}
 
+		// ppe env
+		// req.Header.Add("x-tt-env", "ppe_vedem_algorithm")
+		// req.Header.Add("x-use-ppe", "1")
+
 		signToken := "UNSIGNED-PAYLOAD"
 		token := builtin.Sign("auth-v2", env.VEDEM_IMAGE_AK, env.VEDEM_IMAGE_SK, []byte(signToken))
 
@@ -267,23 +271,31 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp
 		start := time.Now()
 		resp, err = client.Do(req)
 		elapsed := time.Since(start)
-		var logID string
+		if err != nil {
-		if resp != nil {
+			log.Error().Err(err).
-			logID = getLogID(resp.Header)
+				Int("imageBufSize", size).
+				Msgf("request veDEM OCR service error, retry %d", i)
+			continue
 		}
-		if err == nil && resp.StatusCode == http.StatusOK {
+
-			log.Debug().
+		logID := getLogID(resp.Header)
+		statusCode := resp.StatusCode
+		if statusCode != http.StatusOK {
+			log.Error().
 				Str("X-TT-LOGID", logID).
-				Int("image_bytes", size).
+				Int("imageBufSize", size).
-				Int64("elapsed(ms)", elapsed.Milliseconds()).
+				Int("statusCode", statusCode).
-				Msg("request OCR service success")
+				Msgf("request veDEM OCR service failed, retry %d", i)
-			break
+			time.Sleep(1 * time.Second)
+			continue
 		}
-		log.Error().Err(err).
+
+		log.Debug().
 			Str("X-TT-LOGID", logID).
-			Int("imageBufSize", size).
+			Int("image_bytes", size).
-			Msgf("request veDEM OCR service failed, retry %d", i)
+			Int64("elapsed(ms)", elapsed.Milliseconds()).
-		time.Sleep(1 * time.Second)
+			Msg("request OCR service success")
+		break
 	}
 	if resp == nil {
 		err = code.CVServiceConnectionError
@@ -292,7 +304,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp
 
 	defer resp.Body.Close()
 
-	results, err := ioutil.ReadAll(resp.Body)
+	results, err := io.ReadAll(resp.Body)
 	if err != nil {
 		err = errors.Wrap(code.CVResponseError,
 			fmt.Sprintf("read response body error: %v", err))
@@ -379,6 +391,7 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
 		bufSource:             bufSource,
 		imagePath:             imagePath,
 		Tags:                  nil,
+		Resolution:            dExt.windowSize,
 		ScreenshotTakeElapsed: screenshotTakeElapsed,
 	}
 
@@ -388,16 +401,12 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
 		return nil, err
 	}
 	if imageResult != nil {
+		screenResult.imageResult = imageResult
 		screenResult.ScreenshotCVElapsed = time.Since(startTime).Milliseconds() - screenshotTakeElapsed
 		screenResult.Texts = imageResult.OCRResult.ToOCRTexts()
 		screenResult.UploadedURL = imageResult.URL
 		screenResult.Icons = imageResult.UIResult
 
-		if imageResult.LiveType != "" && imageResult.LiveType != "NoLive" {
-			screenResult.Live = &LiveRoom{
-				LiveType: imageResult.LiveType,
-			}
-		}
 		if actionOptions.ScreenShotWithClosePopups {
 			screenResult.Popup = &PopupInfo{
 				Type:      imageResult.CPResult.Type,
@@ -408,10 +417,9 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
 				CloseArea: imageResult.CPResult.CloseArea,
 			}
 		}
-
 	}
 
-	dExt.cacheStepData.screenResults[imagePath] = screenResult
+	dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
 
 	log.Debug().
 		Str("imagePath", imagePath).

hrp/pkg/uixt/service_vedem_test.go

@@ -81,7 +81,8 @@ func TestTapUIWithScreenshot(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = driver.TapByUIDetection(WithScreenShotUITypes("dyhouse", "shoppingbag"))
+	err = driver.TapByUIDetection(
+		WithScreenShotUITypes("dyhouse", "shoppingbag"))
 	if err != nil {
 		t.Fatal(err)
 	}

hrp/pkg/uixt/swipe.go

@@ -192,7 +192,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) er
 	}
 	// tap app icon above the text
 	if len(actionOptions.Offset) == 0 {
-		options = append(options, WithOffset(0, -25))
+		options = append(options, WithTapOffset(0, -25))
 	}
 	// set default swipe interval to 1 second
 	if builtin.IsZeroFloat64(actionOptions.Interval) {

hrp/pkg/uixt/video_crawler.go

@@ -1,6 +1,7 @@
 package uixt
 
 import (
+	"math"
 	"time"
 
 	"github.com/pkg/errors"
@@ -43,8 +44,6 @@ type VideoCrawler struct {
 
 	// used to help checking if swipe success
 	failedCount int64
-	lastFeed    *FeedVideo
-	lastLive    *LiveRoom
 
 	FeedCount int            `json:"feed_count"`
 	FeedStat  map[string]int `json:"feed_stat"` // 分类统计 feed 数量：视频/图文/广告/特效/模板/购物
@@ -112,143 +111,16 @@ func (vc *VideoCrawler) isTargetAchieved() bool {
 	return vc.isFeedTargetAchieved() && vc.isLiveTargetAchieved()
 }
 
-func (vc *VideoCrawler) checkLiveVideo(feedVideo *FeedVideo) (enterPoint PointF, yes bool) {
-	// TODO: check if preview-live from feedVideo
-	if feedVideo.Type != "live" {
-		return PointF{}, false
-	}
-
-	// take screenshot and get OCR texts via image service
-	texts, err := vc.driverExt.GetScreenTexts()
-	if err != nil {
-		return PointF{}, false
-	}
-
-	// 预览流入口：DY/KS
-	// 标签文案：点击进入直播间|进入直播间领金币
-	points, err := texts.FindTexts([]string{".*进入直播间.*"}, WithScope(0, 0.3, 1, 0.8), WithRegex(true))
-	if err == nil {
-		return points[0].Center(), true
-	}
-	// 标签文案：直播中|直播卖货|直播团购
-	points, err = texts.FindTexts([]string{"直播中|直播卖货|直播团购"},
-		WithScope(0, 0.7, 0.5, 1), WithRegex(true))
-	if err == nil {
-		return points[0].Center(), true
-	}
-
-	// 预览流入口：KS/KSLite
-	// 评论框文案：和主播聊聊天...|聊聊天...
-	points, err = texts.FindTexts([]string{".*聊聊天.*"}, WithRegex(true))
-	if err == nil {
-		point := points[0].Center()
-		enterPoint = PointF{
-			X: point.X,
-			Y: point.Y - 300,
-		}
-		return enterPoint, true
-	}
-
-	// TODO: 头像入口
-
-	return PointF{}, false
-}
-
-// run live video crawler
-func (vc *VideoCrawler) startLiveCrawler(enterPoint PointF) error {
-	log.Info().Msg("enter live room")
-	if err := vc.driverExt.TapAbsXY(enterPoint.X, enterPoint.Y); err != nil {
-		log.Error().Err(err).Msg("tap live video failed")
-		return err
-	}
-	time.Sleep(5 * time.Second)
-	for !vc.isLiveTargetAchieved() {
-		select {
-		case <-vc.timer.C:
-			log.Warn().Msg("timeout in live crawler")
-			return errors.Wrap(code.TimeoutError, "live crawler timeout")
-		case <-vc.driverExt.interruptSignal:
-			log.Warn().Msg("interrupted in live crawler")
-			return errors.Wrap(code.InterruptError, "live crawler interrupted")
-		default:
-			// swipe to next live video
-			swipeStartTime := time.Now()
-			if err := vc.driverExt.SwipeUp(); err != nil {
-				log.Error().Err(err).Msg("live swipe up failed")
-				return err
-			}
-			swipeFinishTime := time.Now()
-
-			// wait for live video loading
-			time.Sleep(5 * time.Second)
-
-			// TODO: get app event trackings
-			liveRoom, err := vc.getCurrentLiveRoom()
-			if err != nil {
-				return errors.Wrap(err, "get current live event trackings failed")
-			}
-
-			// take screenshot and get screen texts by OCR
-			screenResult, err := vc.driverExt.GetScreenResult(
-				WithScreenShotOCR(true), WithScreenShotUpload(true))
-			if err != nil {
-				log.Error().Err(err).Msg("OCR GetTexts failed")
-				time.Sleep(3 * time.Second)
-				continue
-			}
-			screenResult.Live = liveRoom
-
-			// TODO: check live type
-
-			// incr live count
-			screenResult.VideoType = "live"
-			vc.LiveCount++
-			log.Info().Strs("tags", screenResult.Tags).
-				Interface("live", screenResult.Live).
-				Msg("found live success")
-
-			// get simulation watch duration
-			if screenResult.Live.SimulationWatchDuration != 0 {
-				screenResult.Live.WatchDuration = screenResult.Live.SimulationWatchDuration
-			} else {
-				screenResult.Live.RandomWatchDuration = getSimulationDuration(vc.configs.Live.SleepRandom)
-				screenResult.Live.WatchDuration = screenResult.Live.RandomWatchDuration
-			}
-			// simulation watch live video
-			sleepStrict(swipeFinishTime, screenResult.Live.WatchDuration)
-
-			// log swipe timelines
-			screenResult.SwipeStartTime = swipeStartTime.UnixMilli()
-			screenResult.SwipeFinishTime = swipeFinishTime.UnixMilli()
-			screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
-		}
-	}
-
-	log.Info().Msg("live count achieved, exit live room")
-
-	return vc.exitLiveRoom()
-}
-
 func (vc *VideoCrawler) exitLiveRoom() error {
-	for i := 0; i < 3; i++ {
+	log.Info().Msg("press back to exit live room")
-		vc.driverExt.SwipeRelative(0.1, 0.5, 0.9, 0.5)
+	return vc.driverExt.Driver.PressBack()
-		time.Sleep(2 * time.Second)
-	}
-
-	// exit live room failed, while video count achieved
-	if vc.isTargetAchieved() {
-		return nil
-	}
-
-	// click X button on upper-right corner
-	if err := vc.driverExt.TapXY(0.95, 0.05); err == nil {
-		log.Info().Msg("tap X button on upper-right corner to exit live room")
-		time.Sleep(2 * time.Second)
-	}
-
-	return errors.New("exit live room failed")
 }
 
+const (
+	FOUND_FEED_SUCCESS = "found feed success"
+	FOUND_LIVE_SUCCESS = "found live success"
+)
+
 func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 	if dExt.plugin == nil {
 		return errors.New("miss plugin for video crawler")
@@ -267,13 +139,10 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 		configs:   configs,
 
 		failedCount: 0,
-		lastFeed:    &FeedVideo{},
+		FeedCount:   0,
-		lastLive:    &LiveRoom{},
+		FeedStat:    make(map[string]int),
-
+		LiveCount:   0,
-		FeedCount: 0,
+		LiveStat:    make(map[string]int),
-		FeedStat:  make(map[string]int),
-		LiveCount: 0,
-		LiveStat:  make(map[string]int),
 	}
 	defer func() {
 		dExt.cacheStepData.videoCrawler = crawler
@@ -294,76 +163,142 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 			// swipe to next feed video
 			log.Info().Msg("swipe to next feed video")
 			swipeStartTime := time.Now()
-			if err = dExt.SwipeUp(); err != nil {
+			if err = dExt.SwipeRelative(0.9, 0.8, 0.9, 0.1, WithOffsetRandomRange(-10, 10)); err != nil {
 				log.Error().Err(err).Msg("feed swipe up failed")
 				return err
 			}
 			swipeFinishTime := time.Now()
 
 			// get app event trackings
-			// retry 3 times if get feed failed, abort if fail 3 consecutive times
+			// retry 10 times if get feed failed, abort if fail 10 consecutive times
-			feedVideo, err := crawler.getCurrentFeedVideo()
+			feedVideo, err := crawler.getCurrentVideo()
-			if err != nil || feedVideo.VideoID == "" {
+			if err != nil || feedVideo.Type == "" {
-				if crawler.failedCount >= 3 {
+				if crawler.failedCount >= 10 {
-					// failed 3 consecutive times
+					// failed 10 consecutive times
-					return errors.New("get current feed video failed 3 consecutive times")
+					return errors.Wrap(code.TrackingGetError,
+						"get current feed video failed 10 consecutive times")
 				}
-				log.Warn().Interface("feedVideo", feedVideo).Msg("get current feed video failed")
+				log.Warn().Msg("get current feed video failed")
+
+				// check and handle popups
+				if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(3)); err != nil {
+					return err
+				}
+
 				// retry
 				crawler.failedCount++
 				continue
 			}
 
-			if feedVideo.VideoID == crawler.lastFeed.VideoID {
+			screenResult := &ScreenResult{
-				// app event tracking not changed
+				Resolution: dExt.windowSize,
-				// check and handle popups
+				Video:      feedVideo,
-				if err = crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(1)); err != nil {
+
-					return err
+				// log swipe timelines
-				}
+				SwipeStartTime:  swipeStartTime.UnixMilli(),
+				SwipeFinishTime: swipeFinishTime.UnixMilli(),
 			}
-			crawler.lastFeed = feedVideo
 
-			screenResult := &ScreenResult{}
+			switch feedVideo.Type {
-			dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
+			case VideoType_PreviewLive:
-
-			// check if live video && run live crawler
-			if enterPoint, isLive := crawler.checkLiveVideo(feedVideo); isLive {
 				// 直播预览流
-				screenResult.VideoType = "live-preview"
+				if crawler.isLiveTargetAchieved() {
-				// TODO
+					// 达标后不再进入直播间
-				// screenResult.Live = feedVideo
+					crawler.LiveCount++
-				log.Info().Msg("live video found")
+					dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
-				if !crawler.isLiveTargetAchieved() {
+					// 观播时长取随机时长与仿真时长的最小值
-					if err := crawler.startLiveCrawler(enterPoint); err != nil {
+					sleepTime := math.Min(float64(feedVideo.SimulationPlayDuration), float64(feedVideo.RandomPlayDuration))
+					feedVideo.PlayDuration = int64(sleepTime)
+					log.Info().
+						Strs("tags", screenResult.Tags).
+						Interface("video", feedVideo).
+						Msg(FOUND_LIVE_SUCCESS)
+					// simulation watch feed video
+					sleepStrict(swipeFinishTime, feedVideo.PlayDuration)
+					break
+				} else {
+					time.Sleep(1 * time.Second)
+					// live target not achieved, enter live
+					entryPoint := PointF{
+						X: float64(dExt.windowSize.Width / 2),
+						Y: float64(dExt.windowSize.Height / 2),
+					}
+
+					log.Info().Msg("tap screen center to enter live room")
+					if err := crawler.driverExt.TapAbsXY(entryPoint.X, entryPoint.Y,
+						WithOffsetRandomRange(-20, 20)); err != nil {
+						log.Error().Err(err).Msg("tap live video failed")
+						continue
+					}
+				}
+				fallthrough
+
+			case VideoType_Live:
+				// 直播
+				log.Info().
+					Strs("tags", screenResult.Tags).
+					Interface("video", feedVideo).
+					Msg(FOUND_LIVE_SUCCESS)
+
+				// take screenshot and get screen texts by OCR
+				screenResultFromOCR, err := crawler.driverExt.GetScreenResult(
+					WithScreenShotOCR(true),
+					WithScreenShotUpload(true),
+					WithScreenShotLiveType(true),
+					WithScreenShotClosePopups(true),
+				)
+				if err != nil {
+					log.Error().Err(err).Msg("get screen result failed")
+					time.Sleep(3 * time.Second)
+					continue
+				}
+				if e := crawler.driverExt.tapPopupHandler(screenResultFromOCR.Popup); e != nil {
+					log.Error().Err(e).Msg("auto handle popup failed")
+					continue
+				}
+
+				// add live type
+				if screenResultFromOCR.imageResult != nil &&
+					screenResultFromOCR.imageResult.LiveType != "" &&
+					screenResultFromOCR.imageResult.LiveType != "NoLive" {
+					screenResult.Video.LiveType = screenResultFromOCR.imageResult.LiveType
+				}
+
+				crawler.LiveCount++
+				// simulation watch feed video
+				sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration)
+
+				screenResultFromOCR.Video = screenResult.Video
+				screenResultFromOCR.Resolution = screenResult.Resolution
+				screenResultFromOCR.SwipeStartTime = screenResult.SwipeStartTime
+				screenResultFromOCR.SwipeFinishTime = screenResult.SwipeFinishTime
+				screenResultFromOCR.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
+
+				if crawler.isLiveTargetAchieved() {
+					log.Info().Interface("live", screenResult.Video).
+						Msg("live count achieved, exit live house")
+					err = crawler.exitLiveRoom()
+					if err != nil {
 						if errors.Is(err, code.TimeoutError) || errors.Is(err, code.InterruptError) {
 							return err
 						}
 						log.Error().Err(err).Msg("run live crawler failed, continue")
-						continue
 					}
 				}
-			} else {
+
-				// 点播
+			default:
-				// check feed type and incr feed count
+				// 点播 || 图文 || 广告 || etc.
-				screenResult.VideoType = "feed"
-				screenResult.Feed = feedVideo
 				crawler.FeedCount++
+				dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
 				log.Info().
 					Strs("tags", screenResult.Tags).
-					Interface("feed", screenResult.Feed).
+					Interface("video", feedVideo).
-					Msg("found feed success")
+					Msg(FOUND_FEED_SUCCESS)
-
-				// get simulation play duration
-				if screenResult.Feed.SimulationPlayDuration != 0 {
-					screenResult.Feed.PlayDuration = screenResult.Feed.SimulationPlayDuration
-				} else {
-					screenResult.Feed.RandomPlayDuration = getSimulationDuration(crawler.configs.Feed.SleepRandom)
-					screenResult.Feed.PlayDuration = screenResult.Feed.RandomPlayDuration
-				}
 
 				// simulation watch feed video
-				sleepStrict(swipeFinishTime, screenResult.Feed.PlayDuration)
+				sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration)
 			}
+			screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
 
 			// check if target count achieved
 			if crawler.isTargetAchieved() {
@@ -371,96 +306,104 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
 				return nil
 			}
 
-			// log swipe timelines
-			screenResult.SwipeStartTime = swipeStartTime.UnixMilli()
-			screenResult.SwipeFinishTime = swipeFinishTime.UnixMilli()
-			screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
-
 			// reset failed count
 			crawler.failedCount = 0
 		}
 	}
 }
 
-type FeedVideo struct {
+type VideoType string
-	// 视频基础数据
-	VideoID  string `json:"video_id"`  // 视频 video ID
-	UserName string `json:"user_name"` // 视频作者
-	Duration int64  `json:"duration"`  // 视频时长(ms)
-	Caption  string `json:"caption"`   // 视频文案
-	Type     string `json:"type"`      // 视频类型, feed/live // TODO: 区分视频、图文、广告
 
+const (
+	VideoType_Feed        VideoType = "FEED"
+	VideoType_PreviewLive VideoType = "PREVIEW-LIVE" // 直播预览流
+	VideoType_Live        VideoType = "LIVE"
+	VideoType_Image       VideoType = "IMAGE"
+)
+
+type Video struct {
+	Type     VideoType `json:"type" required:"true"` // 视频类型, feed/preview-live/live/image
+	DataType string    `json:"data_type"`            // 数据源对应的事件名称
+
+	// Feed 视频基础数据
+	CacheKey string `json:"cache_key,omitempty"` // cachekey
+	VideoID  string `json:"video_id,omitempty"`  // 视频 video ID
+	URL      string `json:"feed_url,omitempty"`  // 实际播放的视频 url
+	UserName string `json:"user_name"`           // 视频作者
+	Duration int64  `json:"duration,omitempty"`  // 视频时长(ms)
+	Caption  string `json:"caption,omitempty"`   // 视频文案
 	// 视频热度数据
-	ViewCount    int64 `json:"view_count"`    // feed 观看数
+	ViewCount    int64 `json:"view_count,omitempty"`    // feed 观看数
-	LikeCount    int64 `json:"like_count"`    // feed 点赞数
+	LikeCount    int64 `json:"like_count,omitempty"`    // feed 点赞数
-	CommentCount int64 `json:"comment_count"` // feed 评论数
+	CommentCount int64 `json:"comment_count,omitempty"` // feed 评论数
-	CollectCount int64 `json:"collect_count"` // feed 收藏数
+	CollectCount int64 `json:"collect_count,omitempty"` // feed 收藏数
-	ForwardCount int64 `json:"forward_count"` // feed 转发数
+	ForwardCount int64 `json:"forward_count,omitempty"` // feed 转发数
-	ShareCount   int64 `json:"share_count"`   // feed 分享数
+	ShareCount   int64 `json:"share_count,omitempty"`   // feed 分享数
+
+	// timelines
+	PublishTimestamp int64 `json:"publish_timestamp,omitempty"` // feed 发布时间戳
+	PreloadTimestamp int64 `json:"preload_timestamp,omitempty"` // feed 预加载时间戳
+
+	// Live 视频基础数据
+	LiveStreamID  string `json:"live_stream_id,omitempty"`  // 直播流 ID
+	LiveStreamURL string `json:"live_stream_url,omitempty"` // 直播流地址
+	LiveType      string `json:"live_type,omitempty"`       // 直播间类型
+	// 网络数据
+	ThroughputKbps int64 `json:"throughput_kbps,omitempty"` // 网速
+	// 视频热度数据
+	AudienceCount int64 `json:"audience_count,omitempty"` // 直播间人数
+
+	// 图文数据
+	ImageUrls []string `json:"image_urls,omitempty"` // 图片对应的 url 列表
 
 	// 记录仿真决策信息
 	PlayDuration           int64   `json:"play_duration"`            // 播放时长(ms)，取自 Simulation/Random
 	SimulationPlayProgress float64 `json:"simulation_play_progress"` // 仿真播放比例（完播率）
 	SimulationPlayDuration int64   `json:"simulation_play_duration"` // 仿真播放时长(ms)
 	RandomPlayDuration     int64   `json:"random_play_duration"`     // 随机播放时长(ms)
-
-	// timelines
-	PublishTimestamp int64 `json:"publish_timestamp"` // feed 发布时间戳
-	PreloadTimestamp int64 `json:"preload_timestamp"` // feed 预加载时间戳
 }
 
-type LiveRoom struct {
+func (vc *VideoCrawler) getCurrentVideo() (video *Video, err error) {
-	// 视频基础数据
+	if !vc.driverExt.plugin.Has("GetCurrentVideo") {
-	LiveStreamID string `json:"live_stream_id"` // 直播流 ID
+		return nil, errors.New("plugin missing GetCurrentVideo method")
-	UserName     string `json:"user_name"`      // 视频作者
-	Caption      string `json:"caption"`        // 视频文案
-	LiveType     string `json:"live_type"`      // 直播间类型, 基于算法服务获取
-
-	// 视频热度数据
-	AudienceCount string `json:"audience_count"` // 直播间人数
-	LikeCount     int64  `json:"like_count"`     // 点赞数
-
-	// 记录仿真决策信息
-	WatchDuration           int64 `json:"watch_duration"`            // 观播时长(ms)，取自 Simulation/Random
-	SimulationWatchDuration int64 `json:"simulation_watch_duration"` // 仿真观播时长(ms)
-	RandomWatchDuration     int64 `json:"random_watch_duration"`     // 随机观播时长(ms)
-
-	// timelines
-	PreloadTimestamp int64 `json:"preload_timestamp"` // feed 预加载时间戳
-}
-
-func (vc *VideoCrawler) getCurrentFeedVideo() (feedVideo *FeedVideo, err error) {
-	if !vc.driverExt.plugin.Has("GetCurrentFeedVideo") {
-		return nil, errors.New("plugin missing GetCurrentFeedVideo method")
 	}
 
-	resp, err := vc.driverExt.plugin.Call("GetCurrentFeedVideo")
+	resp, err := vc.driverExt.plugin.Call("GetCurrentVideo")
 	if err != nil {
-		return nil, errors.Wrap(err, "call plugin GetCurrentFeedVideo failed")
+		return nil, errors.Wrap(err, "call plugin GetCurrentVideo failed")
 	}
 
 	if resp == nil {
-		return nil, errors.New("feed not found")
+		return nil, errors.New("video not found")
 	}
 
 	feedBytes, err := json.Marshal(resp)
 	if err != nil {
-		return nil, errors.New("json marshal feed video info failed")
+		return nil, errors.New("json marshal video info failed")
 	}
 
-	feedVideo = &FeedVideo{}
+	video = &Video{}
-	err = json.Unmarshal(feedBytes, feedVideo)
+	err = json.Unmarshal(feedBytes, video)
 	if err != nil {
-		return nil, errors.Wrap(err, "json unmarshal feed video info failed")
+		return nil, errors.Wrap(err, "json unmarshal video info failed")
+	}
+
+	if video.Type == VideoType_Live || video.Type == VideoType_PreviewLive {
+		video.RandomPlayDuration = getSimulationDuration(vc.configs.Live.SleepRandom)
+	} else {
+		video.RandomPlayDuration = getSimulationDuration(vc.configs.Feed.SleepRandom)
+	}
+
+	// get simulation play duration
+	if video.SimulationPlayDuration != 0 {
+		video.PlayDuration = video.SimulationPlayDuration
+	} else {
+		video.PlayDuration = video.RandomPlayDuration
 	}
 
 	log.Info().
-		Interface("feedVideoCaption", feedVideo.Caption).
+		Str("type", string(video.Type)).
-		Msg("get current feed video success")
+		Str("dataType", video.DataType).
-	return feedVideo, nil
+		Msg("get current video success")
-}
+	return video, nil
-
-func (vc *VideoCrawler) getCurrentLiveRoom() (liveVideo *LiveRoom, err error) {
-	// TODO
-	return
 }

hrp/plugin.go

@@ -77,7 +77,7 @@ func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err er
 	// found plugin file
 	plugin, err = funplugin.Init(pluginPath, pluginOptions...)
 	if err != nil {
-		log.Error().Err(err).Msgf("init plugin failed: %s", pluginPath)
+		log.Error().Str("path", pluginPath).Msg("init plugin failed")
 		err = errors.Wrap(code.InitPluginFailed, err.Error())
 		return
 	}

hrp/server.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"strings"
@@ -42,7 +42,7 @@ func parseBody(r *http.Request) (data map[string]interface{}, err error) {
 
 	// Always set resp.Data to the incoming request body, in case we don't know
 	// how to handle the content type
-	body, err := ioutil.ReadAll(r.Body)
+	body, err := io.ReadAll(r.Body)
 	if err != nil {
 		r.Body.Close()
 		return nil, err