diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4083f633..ed37df31 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,13 @@ # Release History +## v4.3.7 (2023-09-19) + +**go version** + +- feat: add `WithSwipeOffset` to set offset for swipe start/end point +- feat: set random offset for tap/swipe points with `WithOffsetRandomRange` +- change: set `WithOffset` deprecated, replace with `WithTapOffset` + ## v4.3.6 (2023-09-07) **go version** diff --git a/examples/uitest/demo_douyin_follow_live_test.go b/examples/uitest/demo_douyin_follow_live_test.go index a749f709..abec51f2 100644 --- a/examples/uitest/demo_douyin_follow_live_test.go +++ b/examples/uitest/demo_douyin_follow_live_test.go @@ -39,8 +39,8 @@ func TestIOSDouyinFollowLive(t *testing.T) { TapByOCR("关注", uixt.WithIndex(1)).Sleep(10), hrp.NewStep("向上滑动 2 次"). IOS().SwipeToTapTexts([]string{"理肤泉", "婉宝"}, uixt.WithCustomDirection(0.6, 0.2, 0.2, 0.2), uixt.WithIdentifier("click_live")).Sleep(10). - Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(). // 上划 1 次,等待 10s,截图保存 - Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live")).Sleep(10).ScreenShot(), // 再上划 1 次,等待 10s,截图保存 + Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live"), uixt.WithOffsetRandomRange(-10, 10)).Sleep(10).ScreenShot(). // 上划 1 次,等待 10s,截图保存 + Swipe(0.9, 0.7, 0.9, 0.3, uixt.WithIdentifier("slide_in_live"), uixt.WithOffsetRandomRange(-10, 10)).Sleep(10).ScreenShot(), // 再上划 1 次,等待 10s,截图保存 }, } diff --git a/examples/uitest/expert_test.go b/examples/uitest/expert_test.go index 07871184..7ced56e3 100644 --- a/examples/uitest/expert_test.go +++ b/examples/uitest/expert_test.go @@ -111,7 +111,7 @@ func TestAndroidExpertTest(t *testing.T) { hrp.NewStep("home 以及 swipe_to_tap_app 自定义配置"). Android(). Home(). - SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)). + SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithTapOffset(0, -50)). Sleep(10), hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言"). Android(). @@ -265,7 +265,7 @@ func TestIOSExpertTest(t *testing.T) { hrp.NewStep("home 以及 swipe_to_tap_app 自定义配置"). IOS(). Home(). - SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithOffset(0, -50)). + SwipeToTapApp("$app_name", uixt.WithMaxRetryTimes(5), uixt.WithInterval(1), uixt.WithTapOffset(0, -50)). Sleep(10), hrp.NewStep("处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言"). IOS(). diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 2937018e..918530b3 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -452,7 +452,7 @@ func GenNameWithTimestamp(tmpl string) string { } func IsZeroFloat64(f float64) bool { - threshold := 1e-3 + threshold := 1e-9 return math.Abs(f) < threshold } diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 630f7fa9..887115e3 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -78,16 +78,21 @@ var ( MobileUIAssertForegroundAppError = errors.New("mobile UI assert foreground app error") // 76 MobileUIAssertForegroundActivityError = errors.New("mobile UI assert foreground activity error") // 77 MobileUIPopupError = errors.New("mobile UI popup error") // 78 + LoopActionNotFoundError = errors.New("loop action not found error") // 79 ) // CV related: [80, 90) var ( - CVEnvMissedError = errors.New("CV env missed error") // 80 - CVRequestError = errors.New("CV prepare request error") // 81 - CVServiceConnectionError = errors.New("CV service connect error") // 82 - CVResponseError = errors.New("CV parse response error") // 83 - CVResultNotFoundError = errors.New("CV result not found") // 84 - LoopActionNotFoundError = errors.New("loop action not found error") // 85 + CVEnvMissedError = errors.New("CV env missed error") // 80 + CVRequestError = errors.New("CV prepare request error") // 81 + CVServiceConnectionError = errors.New("CV service connect error") // 82 + CVResponseError = errors.New("CV parse response error") // 83 + CVResultNotFoundError = errors.New("CV result not found") // 84 +) + +// trackings related: [90, 100) +var ( + TrackingGetError = errors.New("get trackings failed") // 90 ) var errorsMap = map[error]int{ @@ -141,6 +146,7 @@ var errorsMap = map[error]int{ MobileUIAssertForegroundAppError: 76, MobileUIAssertForegroundActivityError: 77, MobileUIPopupError: 78, + LoopActionNotFoundError: 79, // OCR related CVEnvMissedError: 80, @@ -148,7 +154,9 @@ var errorsMap = map[error]int{ CVServiceConnectionError: 82, CVResponseError: 83, CVResultNotFoundError: 84, - LoopActionNotFoundError: 85, + + // trackings related + TrackingGetError: 90, } func IsErrorPredefined(err error) bool { diff --git a/hrp/internal/sdk/ga4.go b/hrp/internal/sdk/ga4.go index 1f75c366..5a736476 100644 --- a/hrp/internal/sdk/ga4.go +++ b/hrp/internal/sdk/ga4.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "math/rand" "net/http" "net/url" @@ -176,7 +176,7 @@ func (g *GA4Client) SendEvent(event Event) error { return nil } - bs, err = ioutil.ReadAll(res.Body) + bs, err = io.ReadAll(res.Body) if err != nil { return errors.Wrap(err, "read GA4 response body failed") } diff --git a/hrp/pkg/gadb/transport.go b/hrp/pkg/gadb/transport.go index f667dd4a..c55b32b7 100644 --- a/hrp/pkg/gadb/transport.go +++ b/hrp/pkg/gadb/transport.go @@ -3,7 +3,6 @@ package gadb import ( "fmt" "io" - "io/ioutil" "net" "regexp" "strconv" @@ -97,7 +96,7 @@ func (t transport) ReadStringAll() (s string, err error) { } func (t transport) ReadBytesAll() (raw []byte, err error) { - raw, err = ioutil.ReadAll(t.sock) + raw, err = io.ReadAll(t.sock) return } diff --git a/hrp/pkg/gidevice/pkg/libimobiledevice/pcapd.go b/hrp/pkg/gidevice/pkg/libimobiledevice/pcapd.go index 51ce59ff..292ccc21 100644 --- a/hrp/pkg/gidevice/pkg/libimobiledevice/pcapd.go +++ b/hrp/pkg/gidevice/pkg/libimobiledevice/pcapd.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" "io" - "io/ioutil" "strings" "time" @@ -109,7 +108,7 @@ func (c *PcapdClient) GetPacket(buf []byte) ([]byte, error) { } } - packet, err := ioutil.ReadAll(preader) + packet, err := io.ReadAll(preader) if err != nil { return packet, err } diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index 1a813f3b..242b59ea 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -106,10 +106,11 @@ type ActionOptions struct { Scope Scope `json:"scope,omitempty" yaml:"scope,omitempty"` AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"` - Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text - Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point - Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element - MatchOne bool `json:"match_one,omitempty" yaml:"match_one,omitempty"` // match one of the targets if existed + Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text + Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point + OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points + Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element + MatchOne bool `json:"match_one,omitempty" yaml:"match_one,omitempty"` // match one of the targets if existed // set custiom options such as textview, id, description Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"` @@ -181,8 +182,18 @@ func (o *ActionOptions) Options() []ActionOption { o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3])) } if len(o.Offset) == 2 { - options = append(options, WithOffset(o.Offset[0], o.Offset[1])) + // for tap [x,y] offset + options = append(options, WithTapOffset(o.Offset[0], o.Offset[1])) + } else if len(o.Offset) == 4 { + // for swipe [fromX, fromY, toX, toY] offset + options = append(options, WithSwipeOffset( + o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3])) } + if len(o.OffsetRandomRange) == 2 { + options = append(options, WithOffsetRandomRange( + o.OffsetRandomRange[0], o.OffsetRandomRange[1])) + } + if o.Regex { options = append(options, WithRegex(true)) } @@ -238,52 +249,41 @@ func (o *ActionOptions) screenshotActions() []string { return actions } -func NewActionOptions(options ...ActionOption) *ActionOptions { - actionOptions := &ActionOptions{} - for _, option := range options { - option(actionOptions) +func (o *ActionOptions) getRandomOffset() float64 { + if len(o.OffsetRandomRange) != 2 { + // invalid offset random range, should be [min, max] + return 0 } - return actionOptions + + minOffset := o.OffsetRandomRange[0] + maxOffset := o.OffsetRandomRange[1] + return float64(builtin.GetRandomNumber(minOffset, maxOffset)) + rand.Float64() } -func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) map[string]interface{} { - actionOptions := NewActionOptions(options...) - - if actionOptions.Identifier != "" { +func (o *ActionOptions) updateData(data map[string]interface{}) { + if o.Identifier != "" { data["log"] = map[string]interface{}{ "enable": true, - "data": actionOptions.Identifier, + "data": o.Identifier, } } - // handle point offset - if len(actionOptions.Offset) == 2 { - if x, ok := data["x"]; ok { - xf, _ := builtin.Interface2Float64(x) - data["x"] = xf + float64(actionOptions.Offset[0]) - } - if y, ok := data["y"]; ok { - yf, _ := builtin.Interface2Float64(y) - data["y"] = yf + float64(actionOptions.Offset[1]) - } - } - - if actionOptions.Steps > 0 { - data["steps"] = actionOptions.Steps + if o.Steps > 0 { + data["steps"] = o.Steps } if _, ok := data["steps"]; !ok { data["steps"] = 12 // default steps } - if actionOptions.PressDuration > 0 { - data["duration"] = actionOptions.PressDuration + if o.PressDuration > 0 { + data["duration"] = o.PressDuration } if _, ok := data["duration"]; !ok { data["duration"] = 0 // default duration } - if actionOptions.Frequency > 0 { - data["frequency"] = actionOptions.Frequency + if o.Frequency > 0 { + data["frequency"] = o.Frequency } if _, ok := data["frequency"]; !ok { data["frequency"] = 60 // default frequency @@ -294,13 +294,19 @@ func mergeDataWithOptions(data map[string]interface{}, options ...ActionOption) } // custom options - if actionOptions.Custom != nil { - for k, v := range actionOptions.Custom { + if o.Custom != nil { + for k, v := range o.Custom { data[k] = v } } +} - return data +func NewActionOptions(options ...ActionOption) *ActionOptions { + actionOptions := &ActionOptions{} + for _, option := range options { + option(actionOptions) + } + return actionOptions } type ActionOption func(o *ActionOptions) @@ -377,12 +383,29 @@ func WithAbsScope(x1, y1, x2, y2 int) ActionOption { } } +// Deprecated: use WithTapOffset instead func WithOffset(offsetX, offsetY int) ActionOption { return func(o *ActionOptions) { o.Offset = []int{offsetX, offsetY} } } +// tap [x, y] with offset [offsetX, offsetY] +var WithTapOffset = WithOffset + +// swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY] +func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption { + return func(o *ActionOptions) { + o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY} + } +} + +func WithOffsetRandomRange(min, max int) ActionOption { + return func(o *ActionOptions) { + o.OffsetRandomRange = []int{min, max} + } +} + func WithRegex(regex bool) ActionOption { return func(o *ActionOptions) { o.Regex = regex diff --git a/hrp/pkg/uixt/android_adb_driver.go b/hrp/pkg/uixt/android_adb_driver.go index 4577c104..b2d84652 100644 --- a/hrp/pkg/uixt/android_adb_driver.go +++ b/hrp/pkg/uixt/android_adb_driver.go @@ -228,6 +228,8 @@ func (ad *adbDriver) TapFloat(x, y float64, options ...ActionOption) (err error) x += float64(actionOptions.Offset[0]) y += float64(actionOptions.Offset[1]) } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() // adb shell input tap x y xStr := fmt.Sprintf("%.1f", x) @@ -272,6 +274,19 @@ func (ad *adbDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) } func (ad *adbDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { + actionOptions := NewActionOptions(options...) + + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + // adb shell input swipe fromX fromY toX toY _, err := ad.adbClient.RunShellCommand( "input", "swipe", diff --git a/hrp/pkg/uixt/android_device.go b/hrp/pkg/uixt/android_device.go index c1793880..42b4757e 100644 --- a/hrp/pkg/uixt/android_device.go +++ b/hrp/pkg/uixt/android_device.go @@ -95,11 +95,19 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er } dev := deviceList[0] - device.SerialNumber = dev.Serial() + + if device.SerialNumber == "" { + selectSerial := dev.Serial() + device.SerialNumber = selectSerial + log.Warn(). + Str("serial", device.SerialNumber). + Msg("android SerialNumber is not specified, select the first one") + } + device.d = dev device.logcat = NewAdbLogcat(device.SerialNumber) - log.Info().Str("serial", device.SerialNumber).Msg("select android device") + log.Info().Str("serial", device.SerialNumber).Msg("init android device") return device, nil } diff --git a/hrp/pkg/uixt/android_uia2_driver.go b/hrp/pkg/uixt/android_uia2_driver.go index 3a09e36d..674b70c3 100644 --- a/hrp/pkg/uixt/android_uia2_driver.go +++ b/hrp/pkg/uixt/android_uia2_driver.go @@ -259,14 +259,23 @@ func (ud *uiaDriver) Tap(x, y int, options ...ActionOption) error { func (ud *uiaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) + actionOptions := NewActionOptions(options...) + + if len(actionOptions.Offset) == 2 { + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) + } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() + data := map[string]interface{}{ "x": x, "y": y, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "appium/tap") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap") return } @@ -299,6 +308,18 @@ func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e } func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -306,11 +327,11 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...Action "endY": toY, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) // register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag")) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "touch/drag") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/drag") return } @@ -325,6 +346,18 @@ func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...ActionOption) func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...ActionOption) error { // register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform")) + actionOptions := NewActionOptions(options...) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ "startX": fromX, "startY": fromY, @@ -332,10 +365,10 @@ func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio "endY": toY, } - // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra uiautomator configurations + actionOptions.updateData(data) - _, err := ud.httpPOST(newData, "/session", ud.sessionId, "touch/perform") + _, err := ud.httpPOST(data, "/session", ud.sessionId, "touch/perform") return err } @@ -385,13 +418,14 @@ func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) { // register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys")) // https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85 + actionOptions := NewActionOptions(options...) data := map[string]interface{}{ "text": text, } // new data options in post data for extra uiautomator configurations - newData := mergeDataWithOptions(data, options...) + actionOptions.updateData(data) - _, err = ud.httpPOST(newData, "/session", ud.sessionId, "keys") + _, err = ud.httpPOST(data, "/session", ud.sessionId, "keys") return } diff --git a/hrp/pkg/uixt/client.go b/hrp/pkg/uixt/client.go index 1f7d7aca..e680f6f9 100644 --- a/hrp/pkg/uixt/client.go +++ b/hrp/pkg/uixt/client.go @@ -72,7 +72,7 @@ func (wd *Driver) httpRequest(method string, rawURL string, rawBody []byte) (raw _ = resp.Body.Close() }() - rawResp, err = ioutil.ReadAll(resp.Body) + rawResp, err = io.ReadAll(resp.Body) logger := log.Debug().Int("statusCode", resp.StatusCode).Str("duration", time.Since(start).String()) if !strings.HasSuffix(rawURL, "screenshot") { // avoid printing screenshot data diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 2f96b560..54009453 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -2,7 +2,6 @@ package uixt import ( "bytes" - "encoding/json" "fmt" "image" "image/gif" @@ -52,16 +51,16 @@ func WithThreshold(threshold float64) CVOption { } type ScreenResult struct { - bufSource *bytes.Buffer // raw image buffer bytes - imagePath string // image file path + bufSource *bytes.Buffer // raw image buffer bytes + imagePath string // image file path + imageResult *ImageResult // image result - UploadedURL string `json:"uploaded_url"` // uploaded image url - Texts OCRTexts `json:"texts"` // dumped raw OCRTexts - Icons UIResultMap `json:"icons"` // CV 识别的图标 - Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] - VideoType string `json:"video_type,omitempty"` // video type: feed, live-preview or live - Feed *FeedVideo `json:"feed,omitempty"` - Live *LiveRoom `json:"live,omitempty"` + Resolution Size `json:"resolution"` + UploadedURL string `json:"uploaded_url"` // uploaded image url + Texts OCRTexts `json:"texts"` // dumped raw OCRTexts + Icons UIResultMap `json:"icons"` // CV 识别的图标 + Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"] + Video *Video `json:"video,omitempty"` Popup *PopupInfo `json:"popup,omitempty"` SwipeStartTime int64 `json:"swipe_start_time"` // 滑动开始时间戳 @@ -74,16 +73,16 @@ type ScreenResult struct { TotalElapsed int64 `json:"total_elapsed"` // current_swipe_finish -> next_swipe_start 整体耗时(ms) } -type ScreenResultMap map[string]*ScreenResult +type ScreenResultMap map[string]*ScreenResult // key is date time // getScreenShotUrls returns screenShotsUrls using imagePath as key and uploaded URL as value func (screenResults ScreenResultMap) getScreenShotUrls() map[string]string { screenShotsUrls := make(map[string]string) - for imagePath, screenResult := range screenResults { + for _, screenResult := range screenResults { if screenResult.UploadedURL == "" { continue } - screenShotsUrls[imagePath] = screenResult.UploadedURL + screenShotsUrls[screenResult.imagePath] = screenResult.UploadedURL } return screenShotsUrls } @@ -176,7 +175,7 @@ func newDriverExt(device Device, driver WebDriver, plugin funplugin.IPlugin) (dE // get device window size dExt.windowSize, err = dExt.Driver.WindowSize() if err != nil { - return nil, err + return nil, errors.Wrap(err, "get screen resolution failed") } if dExt.ImageService, err = newVEDEMImageService(); err != nil { @@ -275,37 +274,7 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { cacheData["screenshots_urls"] = dExt.cacheStepData.screenResults.getScreenShotUrls() dExt.cacheStepData.screenResults.updatePopupCloseStatus() - - screenSize, err := dExt.Driver.WindowSize() - if err != nil { - log.Warn().Err(err).Msg("get screen resolution failed") - screenSize = Size{} - } - screenResults := make(map[string]interface{}) - for imagePath, screenResult := range dExt.cacheStepData.screenResults { - o, _ := json.Marshal(screenResult.Texts) - data := map[string]interface{}{ - "tags": screenResult.Tags, - "texts": string(o), - "resolution": map[string]int{ - "width": screenSize.Width, - "height": screenSize.Height, - }, - "video_type": screenResult.VideoType, - "feed": screenResult.Feed, - "live": screenResult.Live, - "swipe_start_time": screenResult.SwipeStartTime, - "swipe_finish_time": screenResult.SwipeFinishTime, - "screenshot_take_elapsed": screenResult.ScreenshotTakeElapsed, - "screenshot_cv_elapsed": screenResult.ScreenshotCVElapsed, - "total_elapsed": screenResult.TotalElapsed, - "icons": screenResult.Icons, - "popup": screenResult.Popup, - } - - screenResults[imagePath] = data - } - cacheData["screen_results"] = screenResults + cacheData["screen_results"] = dExt.cacheStepData.screenResults // clear cache dExt.cacheStepData.reset() @@ -370,14 +339,19 @@ func (dExt *DriverExt) AssertImage(imagePath, assert string) bool { } func (dExt *DriverExt) AssertForegroundApp(appName, assert string) bool { - var err error + app, err := dExt.Driver.GetForegroundApp() + if err != nil { + log.Warn().Err(err).Msg("get foreground app failed, skip app/activity assertion") + return true // Notice: ignore error when get foreground app failed + } + log.Debug().Interface("app", app).Msg("get foreground app") + + // assert package name switch assert { case AssertionEqual: - err = dExt.Driver.AssertForegroundApp(appName) - return err == nil + return app.PackageName == appName case AssertionNotEqual: - err = dExt.Driver.AssertForegroundApp(appName) - return err != nil + return app.PackageName != appName default: log.Warn().Str("assert method", assert).Msg("unexpected assert method") } diff --git a/hrp/pkg/uixt/ios_device.go b/hrp/pkg/uixt/ios_device.go index bc0842d9..843a34b3 100644 --- a/hrp/pkg/uixt/ios_device.go +++ b/hrp/pkg/uixt/ios_device.go @@ -239,7 +239,14 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) { dev := deviceList[0] udid := dev.Properties().SerialNumber - device.UDID = udid + + if device.UDID == "" { + device.UDID = udid + log.Warn(). + Str("udid", udid). + Msg("ios UDID is not specified, select the first one") + } + device.d = dev // run xctest if XCTestBundleID is set @@ -251,7 +258,7 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) { } } - log.Info().Str("udid", device.UDID).Msg("select ios device") + log.Info().Str("udid", device.UDID).Msg("init ios device") return device, nil } diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index aa2242e6..52390256 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -461,14 +461,25 @@ func (wd *wdaDriver) Tap(x, y int, options ...ActionOption) error { func (wd *wdaDriver) TapFloat(x, y float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] - data := map[string]interface{}{ - "x": wd.toScale(x), - "y": wd.toScale(y), - } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + actionOptions := NewActionOptions(options...) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/tap/0") + x = wd.toScale(x) + y = wd.toScale(y) + if len(actionOptions.Offset) == 2 { + x += float64(actionOptions.Offset[0]) + y += float64(actionOptions.Offset[1]) + } + x += actionOptions.getRandomOffset() + y += actionOptions.getRandomOffset() + + data := map[string]interface{}{ + "x": x, + "y": y, + } + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) + + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0") return } @@ -510,17 +521,34 @@ func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...ActionOption) e func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] + actionOptions := NewActionOptions(options...) + + fromX = wd.toScale(fromX) + fromY = wd.toScale(fromY) + toX = wd.toScale(toX) + toY = wd.toScale(toY) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() + data := map[string]interface{}{ - "fromX": wd.toScale(fromX), - "fromY": wd.toScale(fromY), - "toX": wd.toScale(toX), - "toY": wd.toScale(toY), + "fromX": fromX, + "fromY": fromY, + "toX": toX, + "toY": toY, } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } @@ -557,12 +585,13 @@ func (wd *wdaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe func (wd *wdaDriver) SendKeys(text string, options ...ActionOption) (err error) { // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] + actionOptions := NewActionOptions(options...) data := map[string]interface{}{"value": strings.Split(text, "")} // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/keys") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys") return } @@ -572,22 +601,38 @@ func (wd *wdaDriver) Input(text string, options ...ActionOption) (err error) { // PressBack simulates a short press on the BACK button. func (wd *wdaDriver) PressBack(options ...ActionOption) (err error) { + actionOptions := NewActionOptions(options...) + windowSize, err := wd.WindowSize() if err != nil { return } + fromX := wd.toScale(float64(windowSize.Width) * 0) + fromY := wd.toScale(float64(windowSize.Height) * 0.5) + toX := wd.toScale(float64(windowSize.Width) * 0.6) + toY := wd.toScale(float64(windowSize.Height) * 0.5) + if len(actionOptions.Offset) == 4 { + fromX += float64(actionOptions.Offset[0]) + fromY += float64(actionOptions.Offset[1]) + toX += float64(actionOptions.Offset[2]) + toY += float64(actionOptions.Offset[3]) + } + fromX += actionOptions.getRandomOffset() + fromY += actionOptions.getRandomOffset() + toX += actionOptions.getRandomOffset() + toY += actionOptions.getRandomOffset() data := map[string]interface{}{ - "fromX": wd.toScale(float64(windowSize.Width) * 0), - "fromY": wd.toScale(float64(windowSize.Height) * 0.5), - "toX": wd.toScale(float64(windowSize.Width) * 0.6), - "toY": wd.toScale(float64(windowSize.Height) * 0.5), + "fromX": fromX, + "fromY": fromY, + "toX": toX, + "toY": toY, } - // new data options in post data for extra WDA configurations - newData := mergeDataWithOptions(data, options...) + // update data options in post data for extra WDA configurations + actionOptions.updateData(data) - _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go index 2d22b20f..374e04eb 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/opencv.go @@ -7,7 +7,7 @@ import ( "fmt" "image" "image/color" - "io/ioutil" + "io" "math" "os" @@ -133,7 +133,7 @@ func getBufFromDisk(name string) (*bytes.Buffer, error) { return nil, err } var all []byte - if all, err = ioutil.ReadAll(f); err != nil { + if all, err = io.ReadAll(f); err != nil { return nil, err } return bytes.NewBuffer(all), nil @@ -361,7 +361,7 @@ func getMatsFromDisk(nameImage, nameTpl string, flags gocv.IMReadFlag) (matImage // return nil, e // } // var all []byte -// if all, e = ioutil.ReadAll(f); e != nil { +// if all, e = io.ReadAll(f); e != nil { // return nil, e // } // return bytes.NewBuffer(all), nil diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index efe9ad05..689efa5a 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -114,6 +114,7 @@ func (dExt *DriverExt) ClosePopups(options ...ActionOption) error { } func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error { + log.Info().Msg("try to find and close popups") actionOptions := NewActionOptions(options...) maxRetryTimes := actionOptions.MaxRetryTimes interval := actionOptions.Interval diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index c5f45395..7e32a9e3 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -4,7 +4,7 @@ import ( "bytes" "fmt" "image" - "io/ioutil" + "io" "math" "mime/multipart" "net/http" @@ -55,8 +55,8 @@ func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) { } type ImageResult struct { - URL string `json:"url"` // image uploaded url - OCRResult OCRResults `json:"ocrResult"` // OCR texts + URL string `json:"url,omitempty"` // image uploaded url + OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts // NoLive(非直播间) // Shop(电商) // LifeService(生活服务) @@ -67,9 +67,9 @@ type ImageResult struct { // Media(媒体) // Chat(语音) // Event(赛事) - LiveType string `json:"liveType"` // 直播间类型 - UIResult UIResultMap `json:"uiResult"` // 图标检测 - CPResult ClosePopupsResult `json:"closeResult"` // 弹窗按钮检测 + LiveType string `json:"liveType,omitempty"` // 直播间类型 + UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测 + CPResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测 } type APIResponseImage struct { @@ -257,6 +257,10 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp return } + // ppe env + // req.Header.Add("x-tt-env", "ppe_vedem_algorithm") + // req.Header.Add("x-use-ppe", "1") + signToken := "UNSIGNED-PAYLOAD" token := builtin.Sign("auth-v2", env.VEDEM_IMAGE_AK, env.VEDEM_IMAGE_SK, []byte(signToken)) @@ -267,23 +271,31 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp start := time.Now() resp, err = client.Do(req) elapsed := time.Since(start) - var logID string - if resp != nil { - logID = getLogID(resp.Header) + if err != nil { + log.Error().Err(err). + Int("imageBufSize", size). + Msgf("request veDEM OCR service error, retry %d", i) + continue } - if err == nil && resp.StatusCode == http.StatusOK { - log.Debug(). + + logID := getLogID(resp.Header) + statusCode := resp.StatusCode + if statusCode != http.StatusOK { + log.Error(). Str("X-TT-LOGID", logID). - Int("image_bytes", size). - Int64("elapsed(ms)", elapsed.Milliseconds()). - Msg("request OCR service success") - break + Int("imageBufSize", size). + Int("statusCode", statusCode). + Msgf("request veDEM OCR service failed, retry %d", i) + time.Sleep(1 * time.Second) + continue } - log.Error().Err(err). + + log.Debug(). Str("X-TT-LOGID", logID). - Int("imageBufSize", size). - Msgf("request veDEM OCR service failed, retry %d", i) - time.Sleep(1 * time.Second) + Int("image_bytes", size). + Int64("elapsed(ms)", elapsed.Milliseconds()). + Msg("request OCR service success") + break } if resp == nil { err = code.CVServiceConnectionError @@ -292,7 +304,7 @@ func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOp defer resp.Body.Close() - results, err := ioutil.ReadAll(resp.Body) + results, err := io.ReadAll(resp.Body) if err != nil { err = errors.Wrap(code.CVResponseError, fmt.Sprintf("read response body error: %v", err)) @@ -379,6 +391,7 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S bufSource: bufSource, imagePath: imagePath, Tags: nil, + Resolution: dExt.windowSize, ScreenshotTakeElapsed: screenshotTakeElapsed, } @@ -388,16 +401,12 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S return nil, err } if imageResult != nil { + screenResult.imageResult = imageResult screenResult.ScreenshotCVElapsed = time.Since(startTime).Milliseconds() - screenshotTakeElapsed screenResult.Texts = imageResult.OCRResult.ToOCRTexts() screenResult.UploadedURL = imageResult.URL screenResult.Icons = imageResult.UIResult - if imageResult.LiveType != "" && imageResult.LiveType != "NoLive" { - screenResult.Live = &LiveRoom{ - LiveType: imageResult.LiveType, - } - } if actionOptions.ScreenShotWithClosePopups { screenResult.Popup = &PopupInfo{ Type: imageResult.CPResult.Type, @@ -408,10 +417,9 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S CloseArea: imageResult.CPResult.CloseArea, } } - } - dExt.cacheStepData.screenResults[imagePath] = screenResult + dExt.cacheStepData.screenResults[time.Now().String()] = screenResult log.Debug(). Str("imagePath", imagePath). diff --git a/hrp/pkg/uixt/service_vedem_test.go b/hrp/pkg/uixt/service_vedem_test.go index 78ed8b4b..bc215bee 100644 --- a/hrp/pkg/uixt/service_vedem_test.go +++ b/hrp/pkg/uixt/service_vedem_test.go @@ -81,7 +81,8 @@ func TestTapUIWithScreenshot(t *testing.T) { t.Fatal(err) } - err = driver.TapByUIDetection(WithScreenShotUITypes("dyhouse", "shoppingbag")) + err = driver.TapByUIDetection( + WithScreenShotUITypes("dyhouse", "shoppingbag")) if err != nil { t.Fatal(err) } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 9ed9d926..0e34e23b 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -192,7 +192,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) er } // tap app icon above the text if len(actionOptions.Offset) == 0 { - options = append(options, WithOffset(0, -25)) + options = append(options, WithTapOffset(0, -25)) } // set default swipe interval to 1 second if builtin.IsZeroFloat64(actionOptions.Interval) { diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 0f8a1f75..f31693c5 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -1,6 +1,7 @@ package uixt import ( + "math" "time" "github.com/pkg/errors" @@ -43,8 +44,6 @@ type VideoCrawler struct { // used to help checking if swipe success failedCount int64 - lastFeed *FeedVideo - lastLive *LiveRoom FeedCount int `json:"feed_count"` FeedStat map[string]int `json:"feed_stat"` // 分类统计 feed 数量:视频/图文/广告/特效/模板/购物 @@ -112,143 +111,16 @@ func (vc *VideoCrawler) isTargetAchieved() bool { return vc.isFeedTargetAchieved() && vc.isLiveTargetAchieved() } -func (vc *VideoCrawler) checkLiveVideo(feedVideo *FeedVideo) (enterPoint PointF, yes bool) { - // TODO: check if preview-live from feedVideo - if feedVideo.Type != "live" { - return PointF{}, false - } - - // take screenshot and get OCR texts via image service - texts, err := vc.driverExt.GetScreenTexts() - if err != nil { - return PointF{}, false - } - - // 预览流入口:DY/KS - // 标签文案:点击进入直播间|进入直播间领金币 - points, err := texts.FindTexts([]string{".*进入直播间.*"}, WithScope(0, 0.3, 1, 0.8), WithRegex(true)) - if err == nil { - return points[0].Center(), true - } - // 标签文案:直播中|直播卖货|直播团购 - points, err = texts.FindTexts([]string{"直播中|直播卖货|直播团购"}, - WithScope(0, 0.7, 0.5, 1), WithRegex(true)) - if err == nil { - return points[0].Center(), true - } - - // 预览流入口:KS/KSLite - // 评论框文案:和主播聊聊天...|聊聊天... - points, err = texts.FindTexts([]string{".*聊聊天.*"}, WithRegex(true)) - if err == nil { - point := points[0].Center() - enterPoint = PointF{ - X: point.X, - Y: point.Y - 300, - } - return enterPoint, true - } - - // TODO: 头像入口 - - return PointF{}, false -} - -// run live video crawler -func (vc *VideoCrawler) startLiveCrawler(enterPoint PointF) error { - log.Info().Msg("enter live room") - if err := vc.driverExt.TapAbsXY(enterPoint.X, enterPoint.Y); err != nil { - log.Error().Err(err).Msg("tap live video failed") - return err - } - time.Sleep(5 * time.Second) - for !vc.isLiveTargetAchieved() { - select { - case <-vc.timer.C: - log.Warn().Msg("timeout in live crawler") - return errors.Wrap(code.TimeoutError, "live crawler timeout") - case <-vc.driverExt.interruptSignal: - log.Warn().Msg("interrupted in live crawler") - return errors.Wrap(code.InterruptError, "live crawler interrupted") - default: - // swipe to next live video - swipeStartTime := time.Now() - if err := vc.driverExt.SwipeUp(); err != nil { - log.Error().Err(err).Msg("live swipe up failed") - return err - } - swipeFinishTime := time.Now() - - // wait for live video loading - time.Sleep(5 * time.Second) - - // TODO: get app event trackings - liveRoom, err := vc.getCurrentLiveRoom() - if err != nil { - return errors.Wrap(err, "get current live event trackings failed") - } - - // take screenshot and get screen texts by OCR - screenResult, err := vc.driverExt.GetScreenResult( - WithScreenShotOCR(true), WithScreenShotUpload(true)) - if err != nil { - log.Error().Err(err).Msg("OCR GetTexts failed") - time.Sleep(3 * time.Second) - continue - } - screenResult.Live = liveRoom - - // TODO: check live type - - // incr live count - screenResult.VideoType = "live" - vc.LiveCount++ - log.Info().Strs("tags", screenResult.Tags). - Interface("live", screenResult.Live). - Msg("found live success") - - // get simulation watch duration - if screenResult.Live.SimulationWatchDuration != 0 { - screenResult.Live.WatchDuration = screenResult.Live.SimulationWatchDuration - } else { - screenResult.Live.RandomWatchDuration = getSimulationDuration(vc.configs.Live.SleepRandom) - screenResult.Live.WatchDuration = screenResult.Live.RandomWatchDuration - } - // simulation watch live video - sleepStrict(swipeFinishTime, screenResult.Live.WatchDuration) - - // log swipe timelines - screenResult.SwipeStartTime = swipeStartTime.UnixMilli() - screenResult.SwipeFinishTime = swipeFinishTime.UnixMilli() - screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds() - } - } - - log.Info().Msg("live count achieved, exit live room") - - return vc.exitLiveRoom() -} - func (vc *VideoCrawler) exitLiveRoom() error { - for i := 0; i < 3; i++ { - vc.driverExt.SwipeRelative(0.1, 0.5, 0.9, 0.5) - time.Sleep(2 * time.Second) - } - - // exit live room failed, while video count achieved - if vc.isTargetAchieved() { - return nil - } - - // click X button on upper-right corner - if err := vc.driverExt.TapXY(0.95, 0.05); err == nil { - log.Info().Msg("tap X button on upper-right corner to exit live room") - time.Sleep(2 * time.Second) - } - - return errors.New("exit live room failed") + log.Info().Msg("press back to exit live room") + return vc.driverExt.Driver.PressBack() } +const ( + FOUND_FEED_SUCCESS = "found feed success" + FOUND_LIVE_SUCCESS = "found live success" +) + func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { if dExt.plugin == nil { return errors.New("miss plugin for video crawler") @@ -267,13 +139,10 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { configs: configs, failedCount: 0, - lastFeed: &FeedVideo{}, - lastLive: &LiveRoom{}, - - FeedCount: 0, - FeedStat: make(map[string]int), - LiveCount: 0, - LiveStat: make(map[string]int), + FeedCount: 0, + FeedStat: make(map[string]int), + LiveCount: 0, + LiveStat: make(map[string]int), } defer func() { dExt.cacheStepData.videoCrawler = crawler @@ -294,76 +163,142 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // swipe to next feed video log.Info().Msg("swipe to next feed video") swipeStartTime := time.Now() - if err = dExt.SwipeUp(); err != nil { + if err = dExt.SwipeRelative(0.9, 0.8, 0.9, 0.1, WithOffsetRandomRange(-10, 10)); err != nil { log.Error().Err(err).Msg("feed swipe up failed") return err } swipeFinishTime := time.Now() // get app event trackings - // retry 3 times if get feed failed, abort if fail 3 consecutive times - feedVideo, err := crawler.getCurrentFeedVideo() - if err != nil || feedVideo.VideoID == "" { - if crawler.failedCount >= 3 { - // failed 3 consecutive times - return errors.New("get current feed video failed 3 consecutive times") + // retry 10 times if get feed failed, abort if fail 10 consecutive times + feedVideo, err := crawler.getCurrentVideo() + if err != nil || feedVideo.Type == "" { + if crawler.failedCount >= 10 { + // failed 10 consecutive times + return errors.Wrap(code.TrackingGetError, + "get current feed video failed 10 consecutive times") } - log.Warn().Interface("feedVideo", feedVideo).Msg("get current feed video failed") + log.Warn().Msg("get current feed video failed") + + // check and handle popups + if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(3)); err != nil { + return err + } + // retry crawler.failedCount++ continue } - if feedVideo.VideoID == crawler.lastFeed.VideoID { - // app event tracking not changed - // check and handle popups - if err = crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(1)); err != nil { - return err - } + screenResult := &ScreenResult{ + Resolution: dExt.windowSize, + Video: feedVideo, + + // log swipe timelines + SwipeStartTime: swipeStartTime.UnixMilli(), + SwipeFinishTime: swipeFinishTime.UnixMilli(), } - crawler.lastFeed = feedVideo - screenResult := &ScreenResult{} - dExt.cacheStepData.screenResults[time.Now().String()] = screenResult - - // check if live video && run live crawler - if enterPoint, isLive := crawler.checkLiveVideo(feedVideo); isLive { + switch feedVideo.Type { + case VideoType_PreviewLive: // 直播预览流 - screenResult.VideoType = "live-preview" - // TODO - // screenResult.Live = feedVideo - log.Info().Msg("live video found") - if !crawler.isLiveTargetAchieved() { - if err := crawler.startLiveCrawler(enterPoint); err != nil { + if crawler.isLiveTargetAchieved() { + // 达标后不再进入直播间 + crawler.LiveCount++ + dExt.cacheStepData.screenResults[time.Now().String()] = screenResult + // 观播时长取随机时长与仿真时长的最小值 + sleepTime := math.Min(float64(feedVideo.SimulationPlayDuration), float64(feedVideo.RandomPlayDuration)) + feedVideo.PlayDuration = int64(sleepTime) + log.Info(). + Strs("tags", screenResult.Tags). + Interface("video", feedVideo). + Msg(FOUND_LIVE_SUCCESS) + // simulation watch feed video + sleepStrict(swipeFinishTime, feedVideo.PlayDuration) + break + } else { + time.Sleep(1 * time.Second) + // live target not achieved, enter live + entryPoint := PointF{ + X: float64(dExt.windowSize.Width / 2), + Y: float64(dExt.windowSize.Height / 2), + } + + log.Info().Msg("tap screen center to enter live room") + if err := crawler.driverExt.TapAbsXY(entryPoint.X, entryPoint.Y, + WithOffsetRandomRange(-20, 20)); err != nil { + log.Error().Err(err).Msg("tap live video failed") + continue + } + } + fallthrough + + case VideoType_Live: + // 直播 + log.Info(). + Strs("tags", screenResult.Tags). + Interface("video", feedVideo). + Msg(FOUND_LIVE_SUCCESS) + + // take screenshot and get screen texts by OCR + screenResultFromOCR, err := crawler.driverExt.GetScreenResult( + WithScreenShotOCR(true), + WithScreenShotUpload(true), + WithScreenShotLiveType(true), + WithScreenShotClosePopups(true), + ) + if err != nil { + log.Error().Err(err).Msg("get screen result failed") + time.Sleep(3 * time.Second) + continue + } + if e := crawler.driverExt.tapPopupHandler(screenResultFromOCR.Popup); e != nil { + log.Error().Err(e).Msg("auto handle popup failed") + continue + } + + // add live type + if screenResultFromOCR.imageResult != nil && + screenResultFromOCR.imageResult.LiveType != "" && + screenResultFromOCR.imageResult.LiveType != "NoLive" { + screenResult.Video.LiveType = screenResultFromOCR.imageResult.LiveType + } + + crawler.LiveCount++ + // simulation watch feed video + sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration) + + screenResultFromOCR.Video = screenResult.Video + screenResultFromOCR.Resolution = screenResult.Resolution + screenResultFromOCR.SwipeStartTime = screenResult.SwipeStartTime + screenResultFromOCR.SwipeFinishTime = screenResult.SwipeFinishTime + screenResultFromOCR.TotalElapsed = time.Since(swipeFinishTime).Milliseconds() + + if crawler.isLiveTargetAchieved() { + log.Info().Interface("live", screenResult.Video). + Msg("live count achieved, exit live house") + err = crawler.exitLiveRoom() + if err != nil { if errors.Is(err, code.TimeoutError) || errors.Is(err, code.InterruptError) { return err } log.Error().Err(err).Msg("run live crawler failed, continue") - continue } } - } else { - // 点播 - // check feed type and incr feed count - screenResult.VideoType = "feed" - screenResult.Feed = feedVideo + + default: + // 点播 || 图文 || 广告 || etc. crawler.FeedCount++ + dExt.cacheStepData.screenResults[time.Now().String()] = screenResult log.Info(). Strs("tags", screenResult.Tags). - Interface("feed", screenResult.Feed). - Msg("found feed success") - - // get simulation play duration - if screenResult.Feed.SimulationPlayDuration != 0 { - screenResult.Feed.PlayDuration = screenResult.Feed.SimulationPlayDuration - } else { - screenResult.Feed.RandomPlayDuration = getSimulationDuration(crawler.configs.Feed.SleepRandom) - screenResult.Feed.PlayDuration = screenResult.Feed.RandomPlayDuration - } + Interface("video", feedVideo). + Msg(FOUND_FEED_SUCCESS) // simulation watch feed video - sleepStrict(swipeFinishTime, screenResult.Feed.PlayDuration) + sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration) } + screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds() // check if target count achieved if crawler.isTargetAchieved() { @@ -371,96 +306,104 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { return nil } - // log swipe timelines - screenResult.SwipeStartTime = swipeStartTime.UnixMilli() - screenResult.SwipeFinishTime = swipeFinishTime.UnixMilli() - screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds() - // reset failed count crawler.failedCount = 0 } } } -type FeedVideo struct { - // 视频基础数据 - VideoID string `json:"video_id"` // 视频 video ID - UserName string `json:"user_name"` // 视频作者 - Duration int64 `json:"duration"` // 视频时长(ms) - Caption string `json:"caption"` // 视频文案 - Type string `json:"type"` // 视频类型, feed/live // TODO: 区分视频、图文、广告 +type VideoType string +const ( + VideoType_Feed VideoType = "FEED" + VideoType_PreviewLive VideoType = "PREVIEW-LIVE" // 直播预览流 + VideoType_Live VideoType = "LIVE" + VideoType_Image VideoType = "IMAGE" +) + +type Video struct { + Type VideoType `json:"type" required:"true"` // 视频类型, feed/preview-live/live/image + DataType string `json:"data_type"` // 数据源对应的事件名称 + + // Feed 视频基础数据 + CacheKey string `json:"cache_key,omitempty"` // cachekey + VideoID string `json:"video_id,omitempty"` // 视频 video ID + URL string `json:"feed_url,omitempty"` // 实际播放的视频 url + UserName string `json:"user_name"` // 视频作者 + Duration int64 `json:"duration,omitempty"` // 视频时长(ms) + Caption string `json:"caption,omitempty"` // 视频文案 // 视频热度数据 - ViewCount int64 `json:"view_count"` // feed 观看数 - LikeCount int64 `json:"like_count"` // feed 点赞数 - CommentCount int64 `json:"comment_count"` // feed 评论数 - CollectCount int64 `json:"collect_count"` // feed 收藏数 - ForwardCount int64 `json:"forward_count"` // feed 转发数 - ShareCount int64 `json:"share_count"` // feed 分享数 + ViewCount int64 `json:"view_count,omitempty"` // feed 观看数 + LikeCount int64 `json:"like_count,omitempty"` // feed 点赞数 + CommentCount int64 `json:"comment_count,omitempty"` // feed 评论数 + CollectCount int64 `json:"collect_count,omitempty"` // feed 收藏数 + ForwardCount int64 `json:"forward_count,omitempty"` // feed 转发数 + ShareCount int64 `json:"share_count,omitempty"` // feed 分享数 + + // timelines + PublishTimestamp int64 `json:"publish_timestamp,omitempty"` // feed 发布时间戳 + PreloadTimestamp int64 `json:"preload_timestamp,omitempty"` // feed 预加载时间戳 + + // Live 视频基础数据 + LiveStreamID string `json:"live_stream_id,omitempty"` // 直播流 ID + LiveStreamURL string `json:"live_stream_url,omitempty"` // 直播流地址 + LiveType string `json:"live_type,omitempty"` // 直播间类型 + // 网络数据 + ThroughputKbps int64 `json:"throughput_kbps,omitempty"` // 网速 + // 视频热度数据 + AudienceCount int64 `json:"audience_count,omitempty"` // 直播间人数 + + // 图文数据 + ImageUrls []string `json:"image_urls,omitempty"` // 图片对应的 url 列表 // 记录仿真决策信息 PlayDuration int64 `json:"play_duration"` // 播放时长(ms),取自 Simulation/Random SimulationPlayProgress float64 `json:"simulation_play_progress"` // 仿真播放比例(完播率) SimulationPlayDuration int64 `json:"simulation_play_duration"` // 仿真播放时长(ms) RandomPlayDuration int64 `json:"random_play_duration"` // 随机播放时长(ms) - - // timelines - PublishTimestamp int64 `json:"publish_timestamp"` // feed 发布时间戳 - PreloadTimestamp int64 `json:"preload_timestamp"` // feed 预加载时间戳 } -type LiveRoom struct { - // 视频基础数据 - LiveStreamID string `json:"live_stream_id"` // 直播流 ID - UserName string `json:"user_name"` // 视频作者 - Caption string `json:"caption"` // 视频文案 - LiveType string `json:"live_type"` // 直播间类型, 基于算法服务获取 - - // 视频热度数据 - AudienceCount string `json:"audience_count"` // 直播间人数 - LikeCount int64 `json:"like_count"` // 点赞数 - - // 记录仿真决策信息 - WatchDuration int64 `json:"watch_duration"` // 观播时长(ms),取自 Simulation/Random - SimulationWatchDuration int64 `json:"simulation_watch_duration"` // 仿真观播时长(ms) - RandomWatchDuration int64 `json:"random_watch_duration"` // 随机观播时长(ms) - - // timelines - PreloadTimestamp int64 `json:"preload_timestamp"` // feed 预加载时间戳 -} - -func (vc *VideoCrawler) getCurrentFeedVideo() (feedVideo *FeedVideo, err error) { - if !vc.driverExt.plugin.Has("GetCurrentFeedVideo") { - return nil, errors.New("plugin missing GetCurrentFeedVideo method") +func (vc *VideoCrawler) getCurrentVideo() (video *Video, err error) { + if !vc.driverExt.plugin.Has("GetCurrentVideo") { + return nil, errors.New("plugin missing GetCurrentVideo method") } - resp, err := vc.driverExt.plugin.Call("GetCurrentFeedVideo") + resp, err := vc.driverExt.plugin.Call("GetCurrentVideo") if err != nil { - return nil, errors.Wrap(err, "call plugin GetCurrentFeedVideo failed") + return nil, errors.Wrap(err, "call plugin GetCurrentVideo failed") } if resp == nil { - return nil, errors.New("feed not found") + return nil, errors.New("video not found") } feedBytes, err := json.Marshal(resp) if err != nil { - return nil, errors.New("json marshal feed video info failed") + return nil, errors.New("json marshal video info failed") } - feedVideo = &FeedVideo{} - err = json.Unmarshal(feedBytes, feedVideo) + video = &Video{} + err = json.Unmarshal(feedBytes, video) if err != nil { - return nil, errors.Wrap(err, "json unmarshal feed video info failed") + return nil, errors.Wrap(err, "json unmarshal video info failed") + } + + if video.Type == VideoType_Live || video.Type == VideoType_PreviewLive { + video.RandomPlayDuration = getSimulationDuration(vc.configs.Live.SleepRandom) + } else { + video.RandomPlayDuration = getSimulationDuration(vc.configs.Feed.SleepRandom) + } + + // get simulation play duration + if video.SimulationPlayDuration != 0 { + video.PlayDuration = video.SimulationPlayDuration + } else { + video.PlayDuration = video.RandomPlayDuration } log.Info(). - Interface("feedVideoCaption", feedVideo.Caption). - Msg("get current feed video success") - return feedVideo, nil -} - -func (vc *VideoCrawler) getCurrentLiveRoom() (liveVideo *LiveRoom, err error) { - // TODO - return + Str("type", string(video.Type)). + Str("dataType", video.DataType). + Msg("get current video success") + return video, nil } diff --git a/hrp/plugin.go b/hrp/plugin.go index 5b91f7ea..ea9afad9 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -77,7 +77,7 @@ func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err er // found plugin file plugin, err = funplugin.Init(pluginPath, pluginOptions...) if err != nil { - log.Error().Err(err).Msgf("init plugin failed: %s", pluginPath) + log.Error().Str("path", pluginPath).Msg("init plugin failed") err = errors.Wrap(code.InitPluginFailed, err.Error()) return } diff --git a/hrp/server.go b/hrp/server.go index ab2417a6..482c0ffa 100644 --- a/hrp/server.go +++ b/hrp/server.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "io/ioutil" + "io" "log" "net/http" "strings" @@ -42,7 +42,7 @@ func parseBody(r *http.Request) (data map[string]interface{}, err error) { // Always set resp.Data to the incoming request body, in case we don't know // how to handle the content type - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { r.Body.Close() return nil, err