GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-05 07:26:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge master

Browse Source
This commit is contained in:
余泓铮
2024-08-27 16:56:26 +08:00
parent b88fcdd985 dd34fa4781
commit 4c18045f3b
30 changed files with 836 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hrp/cmd/adb/screencap.go
+2 -2
View File
@@ -2,7 +2,7 @@ package adb
import (
"fmt"
"io/ioutil"
"os"
"strings"
"time"
@@ -36,7 +36,7 @@ var screencapAndroidDevicesCmd = &cobra.Command{
}
filepath := fmt.Sprintf("%s.png", builtin.GenNameWithTimestamp("screencap_%d"))
if err = ioutil.WriteFile(filepath, res, 0o644); err != nil {
if err = os.WriteFile(filepath, res, 0o644); err != nil {
return err
}
fmt.Println("screencap saved to", filepath)
hrp/internal/version/VERSION
+1 -1
View File
@@ -1 +1 @@
v4.6.5
v4.6.5
hrp/pkg/gadb/client_test.go
+2 -2
View File
@@ -3,7 +3,7 @@
package gadb
import (
"io/ioutil"
"os"
"testing"
)
@@ -116,6 +116,6 @@ func TestScreenCap(t *testing.T) {
t.Error(err)
}
t.Log(len(res))
ioutil.WriteFile("/tmp/1.png", res, 0o644)
os.WriteFile("/tmp/1.png", res, 0o644)
}
}
hrp/pkg/gadb/device_test.go
+1 -2
View File
@@ -4,7 +4,6 @@ package gadb
import (
"bytes"
"io/ioutil"
"os"
"reflect"
"strings"
@@ -271,7 +270,7 @@ func TestDevice_Pull(t *testing.T) {
}
userHomeDir, _ := os.UserHomeDir()
if err = ioutil.WriteFile(userHomeDir+"/Desktop/hello.txt", buffer.Bytes(), DefaultFileMode); err != nil {
if err = os.WriteFile(userHomeDir+"/Desktop/hello.txt", buffer.Bytes(), DefaultFileMode); err != nil {
t.Fatal(err)
}
}
hrp/pkg/uixt/action.go
+1 -1
View File
@@ -752,7 +752,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) {
}
return dExt.VideoCrawler(configs)
case ACTION_ClosePopups:
return dExt.ClosePopups(action.GetOptions()...)
return dExt.ClosePopupsHandler()
case ACTION_EndToEndDelay:
dExt.CollectEndToEndDelay(action.GetOptions()...)
return nil
hrp/pkg/uixt/android_device.go
+4
View File
@@ -195,6 +195,10 @@ func (dev *AndroidDevice) Init() error {
return nil
}
func (dev *AndroidDevice) System() string {
return "android"
}
func (dev *AndroidDevice) UUID() string {
return dev.SerialNumber
}
hrp/pkg/uixt/android_test.go
+2 -2
View File
@@ -91,7 +91,7 @@ func TestDriver_Screenshot(t *testing.T) {
t.Fatal(err)
}
t.Log(ioutil.WriteFile("/Users/hero/Desktop/s1.png", screenshot.Bytes(), 0o600))
t.Log(os.WriteFile("/Users/hero/Desktop/s1.png", screenshot.Bytes(), 0o600))
}
func TestDriver_Rotation(t *testing.T) {
@@ -363,7 +363,7 @@ func TestDriver_AppLaunch(t *testing.T) {
t.Fatal(err)
}
t.Log(ioutil.WriteFile("s1.png", raw.Bytes(), 0o600))
t.Log(os.WriteFile("s1.png", raw.Bytes(), 0o600))
}
func TestDriver_IsAppInForeground(t *testing.T) {
hrp/pkg/uixt/ext.go
+8 -38
View File
@@ -14,7 +14,6 @@ import (
"os"
"os/signal"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
@@ -63,9 +62,12 @@ type ScreenResult struct {
Video *Video `json:"video,omitempty"`
Popup *PopupInfo `json:"popup,omitempty"`
SwipeStartTime int64 `json:"swipe_start_time"` // 滑动开始时间戳
SwipeFinishTime int64 `json:"swipe_finish_time"` // 滑动结束时间戳
SwipeStartTime int64 `json:"swipe_start_time"` // 滑动开始时间戳
SwipeFinishTime int64 `json:"swipe_finish_time"` // 滑动结束时间戳
FetchVideoStartTime int64 `json:"fetch_video_start_time"` // 抓取视频开始时间戳
FetchVideoFinishTime int64 `json:"fetch_video_finish_time"` // 抓取视频结束时间戳
FetchVideoElapsed int64 `json:"fetch_video_elapsed"` // 抓取视频耗时(ms)
ScreenshotTakeElapsed int64 `json:"screenshot_take_elapsed"` // 设备截图耗时(ms)
ScreenshotCVElapsed int64 `json:"screenshot_cv_elapsed"` // CV 识别耗时(ms)
@@ -87,40 +89,6 @@ func (screenResults ScreenResultMap) getScreenShotUrls() map[string]string {
return screenShotsUrls
}
// updatePopupCloseStatus checks if popup closed normally in every screenResult with close_popups on:
func (screenResults ScreenResultMap) updatePopupCloseStatus() {
var popupScreenResultList []*ScreenResult
for _, screenResult := range screenResults {
if screenResult.Popup == nil {
continue
}
popupScreenResultList = append(popupScreenResultList, screenResult)
}
if len(popupScreenResultList) == 0 {
return
}
sort.Slice(popupScreenResultList, func(i, j int) bool {
return popupScreenResultList[i].Popup.RetryCount < popupScreenResultList[j].Popup.RetryCount
})
for i := 0; i < len(popupScreenResultList)-1; i++ {
curPopup := popupScreenResultList[i].Popup
nextPopup := popupScreenResultList[i+1].Popup
// popup not existed, no need to close
if curPopup.CloseArea.IsEmpty() {
continue
}
// popup existed, but identical popups occurs during next retry
if nextPopup.CloseArea.IsIdentical(curPopup.CloseArea) {
popupScreenResultList[i].Popup.CloseStatus = CloseStatusFail
continue
}
// popup existed, but no popup or different popup occurs during next retry (IsClosed=true)
popupScreenResultList[i].Popup.CloseStatus = CloseStatusSuccess
}
}
type cacheStepData struct {
// cache step screenshot paths
screenShots []string
@@ -153,6 +121,9 @@ type DriverExt struct {
// funplugin
plugin funplugin.IPlugin
// cache last popup to check if popup handle result
lastPopup *PopupInfo
}
func newDriverExt(device Device, driver WebDriver, options ...DriverOption) (dExt *DriverExt, err error) {
@@ -364,7 +335,6 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} {
cacheData["screenshots"] = dExt.cacheStepData.screenShots
cacheData["screenshots_urls"] = dExt.cacheStepData.screenResults.getScreenShotUrls()
dExt.cacheStepData.screenResults.updatePopupCloseStatus()
cacheData["screen_results"] = dExt.cacheStepData.screenResults
cacheData["e2e_results"] = dExt.cacheStepData.e2eDelay
cacheData["driver_request_results"] = dExt.Driver.GetDriverResults()
hrp/pkg/uixt/interface.go
+2 -4
View File
@@ -7,8 +7,6 @@ import (
"time"
"github.com/httprunner/funplugin"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
)
var (
@@ -443,8 +441,8 @@ type PointF struct {
}
func (p PointF) IsIdentical(p2 PointF) bool {
return builtin.IsZeroFloat64(math.Abs(p.X-p2.X)) &&
builtin.IsZeroFloat64(math.Abs(p.Y-p2.Y))
// set the coordinate precision to 1 pixel
return math.Abs(p.X-p2.X) < 1 && math.Abs(p.Y-p2.Y) < 1
}
type Rect struct {
hrp/pkg/uixt/ios_test.go
+1 -1
View File
@@ -325,7 +325,7 @@ func Test_remoteWD_GetPasteboard(t *testing.T) {
// t.Fatal(err)
// }
// userHomeDir, _ := os.UserHomeDir()
// if err = ioutil.WriteFile(userHomeDir+"/Desktop/p1.png", buffer.Bytes(), 0600); err != nil {
// if err = os.WriteFile(userHomeDir+"/Desktop/p1.png", buffer.Bytes(), 0600); err != nil {
// t.Error(err)
// }
}
hrp/pkg/uixt/popups.go
+109 -67
View File
@@ -1,12 +1,11 @@
package uixt
import (
"time"
"math/rand"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/httprunner/httprunner/v4/hrp/internal/code"
)
@@ -27,12 +26,6 @@ var popups = [][]string{
{"管理使用时间", ".*忽略.*"},
}
const (
CloseStatusFound = "found"
CloseStatusSuccess = "success"
CloseStatusFail = "fail"
)
func findTextPopup(screenTexts OCRTexts) (closePoint *OCRText) {
for _, popup := range popups {
if len(popup) != 2 {
@@ -80,6 +73,12 @@ func (dExt *DriverExt) AutoPopupHandler() error {
return dExt.handleTextPopup(screenResult.Texts)
}
const (
CloseStatusFound = "found"
CloseStatusSuccess = "success"
CloseStatusFail = "fail"
)
// ClosePopupsResult represents the result of recognized popup to close
type ClosePopupsResult struct {
Type string `json:"type"`
@@ -89,80 +88,123 @@ type ClosePopupsResult struct {
}
type PopupInfo struct {
CloseStatus string `json:"close_status"`
Type string `json:"type"`
Text string `json:"text"`
RetryCount int `json:"retry_count"`
PicName string `json:"pic_name"`
PicURL string `json:"pic_url"`
PopupArea Box `json:"popup_area"`
CloseArea Box `json:"close_area"`
*ClosePopupsResult
CloseStatus string `json:"close_status"` // found/success/fail
ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个）
RetryCount int `json:"retry_count"`
PicName string `json:"pic_name"`
PicURL string `json:"pic_url"`
}
func (dExt *DriverExt) ClosePopups(options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
func (p *PopupInfo) getClosePoint(lastPopup *PopupInfo) (*PointF, error) {
closeResult := p.ClosePopupsResult
if closeResult == nil {
return nil, nil
}
// default to retry 5 times
if actionOptions.MaxRetryTimes == 0 {
options = append(options, WithMaxRetryTimes(5))
// 弹框不存在 && 关闭按钮不存在
if closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() {
if p.ClosePoints == nil {
// 关闭图标不存在 => 100% 确定不存在弹窗
return nil, nil
}
// 存在关闭按钮，结合上一次的 popup 进行判断
if lastPopup == nil || lastPopup.ClosePoints == nil {
// 当前关闭图标为首次出现，确定是弹窗关闭按钮的概率较小
log.Debug().Interface("closePoints", p.ClosePoints).
Msg("skip close points for the first time")
return nil, nil
}
// 连续两次都存在关闭图标
if p.ClosePoints[0].IsIdentical(lastPopup.ClosePoints[0]) {
// 连续两次图标位置相同 => 存在弹窗 => 点击关闭
log.Warn().
Interface("closePoint", p.ClosePoints[0]).
Interface("lastClosePoints", lastPopup.ClosePoints).
Msg("popup close point detected")
return getRandomClosePoint(p.ClosePoints), nil
}
// 连续两次图标位置不同 => 可能不是弹窗 => skip
log.Debug().Interface("closePoints", p.ClosePoints).
Interface("lastClosePoints", lastPopup.ClosePoints).
Msg("skip close points for not sure")
return nil, nil
}
// set default swipe interval to 1 second
if builtin.IsZeroFloat64(actionOptions.Interval) {
options = append(options, WithInterval(1))
// 弹窗存在 && 关闭按钮不存在
if !closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() {
if p.ClosePoints == nil {
// 关闭图标不存在 => 无法处理，抛异常
log.Error().Interface("popup", p).Msg("popup close area not found")
return nil, errors.Wrap(code.MobileUIPopupError, "popup close area not found")
}
// 使用关闭图标作为关闭按钮（随机选择一个）
return getRandomClosePoint(p.ClosePoints), nil
}
// 关闭按钮存在 && (弹框存在 || 不存在)
if closeResult.Type != "" || p.ClosePoints == nil {
// 弹窗类型存在 || 关闭图标不存在 => 基于关闭按钮关闭弹窗
closePoint := closeResult.CloseArea.Center()
return &closePoint, nil
} else {
// 弹窗类型不存在 && 关闭图标存在，使用关闭图标作为关闭按钮（随机选择一个）
return getRandomClosePoint(p.ClosePoints), nil
}
return dExt.ClosePopupsHandler(options...)
}
func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error {
func getRandomClosePoint(closePoints []PointF) *PointF {
if len(closePoints) == 1 {
return &closePoints[0]
}
return &closePoints[rand.Intn(len(closePoints))]
}
func (dExt *DriverExt) ClosePopupsHandler() (err error) {
log.Info().Msg("try to find and close popups")
actionOptions := NewActionOptions(options...)
maxRetryTimes := actionOptions.MaxRetryTimes
interval := actionOptions.Interval
for retryCount := 0; retryCount < maxRetryTimes; retryCount++ {
screenResult, err := dExt.GetScreenResult(
WithScreenShotClosePopups(true), WithScreenShotUpload(true))
if err != nil {
log.Error().Err(err).Msg("get screen result failed for popup handler")
continue
}
// 1. there are no popups here (fast return normally)
// 2. failed to close popup （maybe tap error, return error）
// 3. successful to close popup (sleep and wait for next retry if existed)
if screenResult.Popup == nil {
break
}
screenResult.Popup.RetryCount = retryCount
if !screenResult.Popup.PopupArea.IsEmpty() {
screenResult.Popup.CloseStatus = CloseStatusFound
}
if screenResult.Popup.CloseArea.IsEmpty() {
break
}
screenResult.Popup.CloseStatus = CloseStatusFound
if err = dExt.tapPopupHandler(screenResult.Popup); err != nil {
return err
}
// sleep for another popup (if existed) to pop
time.Sleep(time.Duration(1000*interval) * time.Millisecond)
screenResult, err := dExt.GetScreenResult(
WithScreenShotUpload(true),
WithScreenShotClosePopups(true), // get popup area and close area
WithScreenShotUITypes("close"), // get all close buttons
)
if err != nil {
log.Error().Err(err).Msg("get screen result failed for popup handler")
return err
}
return nil
}
func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error {
if popup == nil {
popup := screenResult.Popup
defer func() {
dExt.lastPopup = popup
}()
closePoint, err := popup.getClosePoint(dExt.lastPopup)
if err != nil {
return err
}
if closePoint == nil {
// close point not found
log.Debug().Msg("close point not found")
return nil
}
if popup.CloseArea.IsEmpty() {
return nil
}
log.Info().Str("type", popup.Type).Str("text", popup.Text).Msg("close popup")
popupCenter := popup.CloseArea.Center()
if err := dExt.TapAbsXY(popupCenter.X, popupCenter.Y); err != nil {
popup.CloseStatus = CloseStatusFound
log.Info().
Interface("closePoint", closePoint).
Interface("popup", popup).
Msg("tap to close popup")
if err := dExt.TapAbsXY(closePoint.X, closePoint.Y); err != nil {
log.Error().Err(err).Msg("tap popup failed")
return errors.Wrap(code.MobileUIPopupError, err.Error())
}
// tap popup success
// wait 1s and check if popup still exists
log.Info().Msg("tap close point success, check if popup still exists")
return nil
}
hrp/pkg/uixt/service_vedem.go
+15 -12
View File
@@ -67,10 +67,10 @@ type ImageResult struct {
// Media（媒体）
// Chat（语音）
// Event（赛事）
LiveType string `json:"liveType,omitempty"` // 直播间类型
LivePopularity int64 `json:"livePopularity,omitempty"` // 直播间热度
UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测
CPResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
LiveType string `json:"liveType,omitempty"` // 直播间类型
LivePopularity int64 `json:"livePopularity,omitempty"` // 直播间热度
UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测
ClosePopupsResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
}
type APIResponseImage struct {
@@ -410,7 +410,7 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
imageResult, err := dExt.ImageService.GetImage(bufSource, options...)
if err != nil {
log.Error().Err(err).Msg("GetImage from ImageService failed")
return nil, err
return screenResult, err
}
if imageResult != nil {
screenResult.imageResult = imageResult
@@ -420,14 +420,16 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
screenResult.Icons = imageResult.UIResult
screenResult.Video = &Video{LiveType: imageResult.LiveType, ViewCount: imageResult.LivePopularity}
if actionOptions.ScreenShotWithClosePopups && imageResult.CPResult != nil {
if actionOptions.ScreenShotWithClosePopups && imageResult.ClosePopupsResult != nil {
screenResult.Popup = &PopupInfo{
Type: imageResult.CPResult.Type,
Text: imageResult.CPResult.Text,
PicName: imagePath,
PicURL: imageResult.URL,
PopupArea: imageResult.CPResult.PopupArea,
CloseArea: imageResult.CPResult.CloseArea,
ClosePopupsResult: imageResult.ClosePopupsResult,
PicName: imagePath,
PicURL: imageResult.URL,
}
closeAreas, _ := imageResult.UIResult.FilterUIResults([]string{"close"})
for _, closeArea := range closeAreas {
screenResult.Popup.ClosePoints = append(screenResult.Popup.ClosePoints, closeArea.Center())
}
}
}
@@ -491,6 +493,7 @@ func (box Box) IsEmpty() bool {
}
func (box Box) IsIdentical(box2 Box) bool {
// set the coordinate precision to 1 pixel
return box.Point.IsIdentical(box2.Point) &&
builtin.IsZeroFloat64(math.Abs(box.Width-box2.Width)) &&
builtin.IsZeroFloat64(math.Abs(box.Height-box2.Height))
hrp/pkg/uixt/service_vedem_test.go
+1 -15
View File
@@ -102,21 +102,7 @@ func TestDriverExtOCR(t *testing.T) {
func TestClosePopup(t *testing.T) {
setupAndroid(t)
screenResult, err := driverExt.GetScreenResult(
WithScreenShotClosePopups(true), WithScreenShotUpload(true))
if err != nil {
t.Logf("get screen result failed for popup handler: %v", err)
return
}
t.Logf("screen result: %v", screenResult)
if screenResult.Popup == nil {
t.Log("there are no popups here")
return
}
t.Logf("popup info: %v", screenResult.Popup)
if err = driverExt.tapPopupHandler(screenResult.Popup); err != nil {
if err := driverExt.ClosePopupsHandler(); err != nil {
t.Fatal(err)
}
}
hrp/pkg/uixt/swipe.go
+24 -6
View File
@@ -11,10 +11,28 @@ import (
"github.com/httprunner/httprunner/v4/hrp/internal/code"
)
var directionSlice = [][]float64{
{0.85, 0.83, 0.85, 0.1},
{0.9, 0.75, 0.9, 0.1},
{0.6, 0.5, 0.6, 0.1},
}
func assertRelative(p float64) bool {
return p >= 0 && p <= 1
}
func (dExt *DriverExt) SwipeUpUtil(count int64, options ...ActionOption) error {
width := dExt.windowSize.Width
height := dExt.windowSize.Height
fromX := float64(width) * directionSlice[count%3][0]
fromY := float64(height) * directionSlice[count%3][1]
toX := float64(width) * directionSlice[count%3][2]
toY := float64(height) * directionSlice[count%3][3]
return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, options...)
}
// SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY]
func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...ActionOption) error {
width := dExt.windowSize.Width
@@ -156,17 +174,17 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption)
screenResult, err := d.GetScreenResult(
WithScreenShotOCR(true),
WithScreenShotUpload(true),
WithScreenShotClosePopups(true),
)
if err != nil {
return err
}
points, err := screenResult.Texts.FindTexts(texts, dExt.ParseActionOptions(optionsWithoutIdentifier...)...)
points, err := screenResult.Texts.FindTexts(texts,
dExt.ParseActionOptions(optionsWithoutIdentifier...)...)
if err != nil {
log.Error().Err(err).Msg("swipeToTapTexts failed")
log.Error().Err(err).Strs("texts", texts).Msg("find texts failed")
// target texts not found, try to auto handle popup
if e := dExt.tapPopupHandler(screenResult.Popup); e != nil {
log.Error().Err(e).Msg("auto handle popup failed")
if e := dExt.ClosePopupsHandler(); e != nil {
log.Error().Err(e).Msg("run popup handler failed")
}
return err
}
@@ -192,7 +210,7 @@ func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) er
}
// automatic handling popups before swipe
if err := dExt.ClosePopups(); err != nil {
if err := dExt.ClosePopupsHandler(); err != nil {
log.Error().Err(err).Msg("auto handle popup failed")
}
hrp/pkg/uixt/video_crawler.go
+124 -67
View File
@@ -2,6 +2,7 @@ package uixt
import (
"math"
"math/rand"
"time"
"github.com/pkg/errors"
@@ -113,7 +114,12 @@ func (vc *VideoCrawler) isTargetAchieved() bool {
func (vc *VideoCrawler) exitLiveRoom() error {
log.Info().Msg("press back to exit live room")
return vc.driverExt.Driver.PressBack()
err := vc.driverExt.Driver.PressBack()
time.Sleep(time.Duration(3) * time.Second)
if vc.driverExt.TapByOCR("退出直播间") == nil {
log.Info().Msg("clicked the button to exit the live room successfully")
}
return err
}
const (
@@ -148,6 +154,9 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
dExt.cacheStepData.videoCrawler = crawler
}()
// flag，仅当 flag 为 false 时，并处于内流时，才执行退出直播间逻辑
isFeed := true
// loop until target count achieved or timeout
// the main loop is feed crawler
crawler.timer = time.NewTimer(time.Duration(configs.Timeout) * time.Second)
@@ -160,65 +169,64 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
log.Warn().Msg("interrupted in feed crawler")
return errors.Wrap(code.InterruptError, "feed crawler interrupted")
default:
if err = crawler.clearCurrentVideo(); err != nil {
log.Error().Err(err).Msg("clear cache failed")
}
// swipe to next feed video
log.Info().Msg("swipe to next feed video")
swipeStartTime := time.Now()
if err = dExt.SwipeRelative(0.9, 0.8, 0.9, 0.1, WithOffsetRandomRange(-10, 10)); err != nil {
if err = dExt.SwipeUpUtil(crawler.failedCount, WithOffsetRandomRange(-10, 10)); err != nil {
log.Error().Err(err).Msg("feed swipe up failed")
return err
}
swipeFinishTime := time.Now()
// get app event trackings
// retry 10 times if get feed failed, abort if fail 10 consecutive times
feedVideo, err := crawler.getCurrentVideo()
if err != nil || feedVideo.Type == "" {
if crawler.failedCount >= 10 {
// failed 10 consecutive times
// retry 3 times if get feed failed, abort if fail 3 consecutive times
fetchVideoStartTime := time.Now()
currentVideo, err := crawler.getCurrentVideo()
if err != nil || currentVideo.Type == "" {
crawler.failedCount++
if crawler.failedCount >= 3 {
// failed 3 consecutive times
return errors.Wrap(code.TrackingGetError,
"get current feed video failed 10 consecutive times")
"get current feed video failed 3 consecutive times")
}
log.Warn().Msg("get current feed video failed")
log.Warn().
Int64("failedCount", crawler.failedCount).
Msg("get current feed video failed")
// check and handle popups
if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(3)); err != nil {
if err := crawler.driverExt.ClosePopupsHandler(); err != nil {
return err
}
// retry
crawler.failedCount++
continue
}
fetchVideoFinishTime := time.Now()
screenResult := &ScreenResult{
Resolution: dExt.windowSize,
Video: feedVideo,
// 直播预览流线上概率
livePreviewProb := crawler.getLivePreviewProb()
// log swipe timelines
SwipeStartTime: swipeStartTime.UnixMilli(),
SwipeFinishTime: swipeFinishTime.UnixMilli(),
}
switch feedVideo.Type {
switch currentVideo.Type {
case VideoType_PreviewLive:
isFeed = true
// 直播预览流
var skipEnterLive bool
if crawler.isLiveTargetAchieved() {
// 达标后不再进入直播间
crawler.LiveCount++
dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
// 观播时长取随机时长与仿真时长的最小值
sleepTime := math.Min(float64(feedVideo.SimulationPlayDuration), float64(feedVideo.RandomPlayDuration))
feedVideo.PlayDuration = int64(sleepTime)
log.Info().
Strs("tags", screenResult.Tags).
Interface("video", feedVideo).
Msg(FOUND_LIVE_SUCCESS)
// simulation watch feed video
sleepStrict(swipeFinishTime, feedVideo.PlayDuration)
break
} else {
log.Info().Interface("video", currentVideo).
Msg("live count achieved, skip entering live room")
skipEnterLive = true
} else if rand.Float64() <= livePreviewProb {
log.Info().Interface("livePreviewProb", livePreviewProb).Msg("skip entering preview")
skipEnterLive = true
}
if !skipEnterLive {
time.Sleep(1 * time.Second)
// live target not achieved, enter live
// enter live room
entryPoint := PointF{
X: float64(dExt.windowSize.Width / 2),
Y: float64(dExt.windowSize.Height / 2),
@@ -230,53 +238,64 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
log.Error().Err(err).Msg("tap live video failed")
continue
}
currentVideo.Type = VideoType_Live
} else {
// skip entering live room
// only mock simulation play duration
sleepTime := math.Min(float64(currentVideo.SimulationPlayDuration), float64(currentVideo.RandomPlayDuration))
currentVideo.PlayDuration = int64(sleepTime)
}
fallthrough
case VideoType_Live:
// 直播
log.Info().
Strs("tags", screenResult.Tags).
Interface("video", feedVideo).
Msg(FOUND_LIVE_SUCCESS)
crawler.LiveCount++
log.Info().Interface("video", currentVideo).Msg(FOUND_LIVE_SUCCESS)
// wait 3s for live loading
time.Sleep(3 * time.Second)
// take screenshot and get screen texts by OCR
screenResultFromOCR, err := crawler.driverExt.GetScreenResult(
screenResult, err := crawler.driverExt.GetScreenResult(
WithScreenShotOCR(true),
WithScreenShotUpload(true),
WithScreenShotLiveType(true),
WithScreenShotClosePopups(true),
)
if err != nil {
log.Error().Err(err).Msg("get screen result failed")
time.Sleep(3 * time.Second)
continue
}
if e := crawler.driverExt.tapPopupHandler(screenResultFromOCR.Popup); e != nil {
log.Error().Err(e).Msg("auto handle popup failed")
continue
}
// add live type
if screenResultFromOCR.imageResult != nil &&
screenResultFromOCR.imageResult.LiveType != "" &&
screenResultFromOCR.imageResult.LiveType != "NoLive" {
screenResult.Video.LiveType = screenResultFromOCR.imageResult.LiveType
if screenResult.imageResult != nil &&
screenResult.imageResult.LiveType != "" &&
screenResult.imageResult.LiveType != "NoLive" {
currentVideo.LiveType = screenResult.imageResult.LiveType
}
crawler.LiveCount++
// simulation watch feed video
sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration)
// simulation watch live video
simulationPlayDuration := math.Min(float64(currentVideo.PlayDuration), 300000)
sleepStrict(swipeFinishTime, int64(simulationPlayDuration))
screenResultFromOCR.Video = screenResult.Video
screenResultFromOCR.Resolution = screenResult.Resolution
screenResultFromOCR.SwipeStartTime = screenResult.SwipeStartTime
screenResultFromOCR.SwipeFinishTime = screenResult.SwipeFinishTime
screenResultFromOCR.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
screenResult.Video = currentVideo
screenResult.Resolution = dExt.windowSize
screenResult.SwipeStartTime = swipeStartTime.UnixMilli()
screenResult.SwipeFinishTime = swipeFinishTime.UnixMilli()
screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
screenResult.FetchVideoStartTime = fetchVideoStartTime.UnixMilli()
screenResult.FetchVideoFinishTime = fetchVideoFinishTime.UnixMilli()
screenResult.FetchVideoElapsed = fetchVideoFinishTime.Sub(fetchVideoStartTime).Milliseconds()
var exitLive bool
if crawler.isLiveTargetAchieved() {
log.Info().Interface("live", screenResult.Video).
Msg("live count achieved, exit live house")
log.Info().Interface("live", currentVideo).
Msg("live count achieved, exit live room")
exitLive = true
} else if rand.Float64() <= livePreviewProb {
log.Info().Interface("livePreviewProb", livePreviewProb).Msg("exit live room by preview live chance")
exitLive = true
}
// isFeed：通过预览流进入内流失败的情况下，防止使用退出直播间逻辑，影响：首次进入内流，至少会消费两个直播间才能退出
if !isFeed && exitLive && currentVideo.Type == VideoType_Live {
err = crawler.exitLiveRoom()
if err != nil {
if errors.Is(err, code.TimeoutError) || errors.Is(err, code.InterruptError) {
@@ -284,21 +303,34 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) {
}
log.Error().Err(err).Msg("run live crawler failed, continue")
}
} else {
isFeed = false
}
default:
isFeed = true
// 点播 || 图文 || 广告 || etc.
crawler.FeedCount++
log.Info().Interface("video", currentVideo).Msg(FOUND_FEED_SUCCESS)
screenResult := &ScreenResult{
Resolution: dExt.windowSize,
Video: currentVideo,
// log swipe timelines
SwipeStartTime: swipeStartTime.UnixMilli(),
SwipeFinishTime: swipeFinishTime.UnixMilli(),
FetchVideoStartTime: fetchVideoStartTime.UnixMilli(),
FetchVideoFinishTime: fetchVideoFinishTime.UnixMilli(),
FetchVideoElapsed: fetchVideoFinishTime.Sub(fetchVideoStartTime).Milliseconds(),
}
dExt.cacheStepData.screenResults[time.Now().String()] = screenResult
log.Info().
Strs("tags", screenResult.Tags).
Interface("video", feedVideo).
Msg(FOUND_FEED_SUCCESS)
// simulation watch feed video
sleepStrict(swipeFinishTime, screenResult.Video.PlayDuration)
simulationPlayDuration := math.Min(float64(currentVideo.PlayDuration), 600000)
sleepStrict(swipeFinishTime, int64(simulationPlayDuration))
screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
}
screenResult.TotalElapsed = time.Since(swipeFinishTime).Milliseconds()
// check if target count achieved
if crawler.isTargetAchieved() {
@@ -332,6 +364,9 @@ type Video struct {
UserName string `json:"user_name"` // 视频作者
Duration int64 `json:"duration,omitempty"` // 视频时长(ms)
Caption string `json:"caption,omitempty"` // 视频文案
// 作者信息
UserID string `json:"user_id"` // 作者用户名
FollowerCount int64 `json:"follower_count"` // 作者粉丝数
// 视频热度数据
ViewCount int64 `json:"view_count,omitempty"` // feed 观看数
LikeCount int64 `json:"like_count,omitempty"` // feed 点赞数
@@ -363,6 +398,19 @@ type Video struct {
RandomPlayDuration int64 `json:"random_play_duration"` // 随机播放时长(ms)
}
func (vc *VideoCrawler) clearCurrentVideo() error {
if !vc.driverExt.plugin.Has("ClearCurrentVideo") {
return errors.New("plugin missing ClearCurrentVideo method")
}
_, err := vc.driverExt.plugin.Call("ClearCurrentVideo")
if err != nil {
return errors.Wrap(err, "call plugin ClearCurrentVideo failed")
}
return nil
}
func (vc *VideoCrawler) getCurrentVideo() (video *Video, err error) {
if !vc.driverExt.plugin.Has("GetCurrentVideo") {
return nil, errors.New("plugin missing GetCurrentVideo method")
@@ -407,3 +455,12 @@ func (vc *VideoCrawler) getCurrentVideo() (video *Video, err error) {
Msg("get current video success")
return video, nil
}
func (vc *VideoCrawler) getLivePreviewProb() float64 {
if vc.driverExt.Device.System() == "ios" {
return 0.5326
} else if vc.driverExt.Device.System() == "android" {
return 0.3414
}
return -1
}
hrp/runner.go
+7
View File
@@ -240,6 +240,13 @@ func (r *HRPRunner) Run(testcases ...ITestCase) (err error) {
}
}()
if caseRunner.parsedConfig.PluginSetting != nil {
if p, ok := pluginMap.Load(caseRunner.parsedConfig.PluginSetting.Path); ok {
log.Info().Msg(fmt.Sprintf("starting to keep live, path: %v, address: %v", caseRunner.parsedConfig.PluginSetting.Path, p))
go p.(funplugin.IPlugin).StartHeartbeat()
}
}
for it := caseRunner.parametersIterator; it.HasNext(); {
// case runner can run multiple times with different parameters
// each run has its own session runner
hrp/runner_test.go
+23 -3
View File
@@ -1,7 +1,6 @@
package hrp
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
@@ -31,8 +30,8 @@ func removeHashicorpGoPlugin() {
func buildHashicorpPyPlugin() {
log.Info().Msg("[init] prepare hashicorp python plugin")
src, _ := ioutil.ReadFile(tmpl("plugin/debugtalk.py"))
err := ioutil.WriteFile(tmpl("debugtalk.py"), src, 0o644)
src, _ := os.ReadFile(tmpl("plugin/debugtalk.py"))
err := os.WriteFile(tmpl("debugtalk.py"), src, 0o644)
if err != nil {
log.Error().Err(err).Msg("copy hashicorp python plugin failed")
os.Exit(code.GetErrorCode(err))
@@ -156,6 +155,27 @@ func TestRunCaseWithThinkTime(t *testing.T) {
}
}
func TestRunCaseWithShell(t *testing.T) {
testcase1 := &TestCase{
Config: NewConfig("complex shell with env variables").
WithVariables(map[string]interface{}{
"SS": "12345",
"ABC": "$SS",
}),
TestSteps: []IStep{
NewStep("shell21").Shell("echo hello world"),
// NewStep("shell21").Shell("echo $ABC"),
// NewStep("shell21").Shell("which hrp"),
},
}
r := NewRunner(t)
err := r.Run(testcase1)
if err != nil {
t.Fatal()
}
}
func TestRunCaseWithPluginJSON(t *testing.T) {
buildHashicorpGoPlugin()
defer removeHashicorpGoPlugin()
hrp/step.go
+5
View File
@@ -12,6 +12,10 @@ const (
stepTypeWebSocket StepType = "websocket"
stepTypeAndroid StepType = "android"
stepTypeIOS StepType = "ios"
stepTypeShell StepType = "shell"
stepTypeSuffixExtraction StepType = "_extraction"
stepTypeSuffixValidation StepType = "_validation"
)
type StepResult struct {
@@ -41,6 +45,7 @@ type TStep struct {
WebSocket *WebSocketAction `json:"websocket,omitempty" yaml:"websocket,omitempty"`
Android *MobileStep `json:"android,omitempty" yaml:"android,omitempty"`
IOS *MobileStep `json:"ios,omitempty" yaml:"ios,omitempty"`
Shell *Shell `json:"shell,omitempty" yaml:"shell,omitempty"`
Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"`
SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"`
TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"`
hrp/step_mobile_ui.go
+4 -1
View File
@@ -531,7 +531,10 @@ func (s *StepMobileUIValidation) Name() string {
}
func (s *StepMobileUIValidation) Type() StepType {
return stepTypeIOS
if s.step.Android != nil {
return stepTypeAndroid + stepTypeSuffixValidation
}
return stepTypeIOS + stepTypeSuffixValidation
}
func (s *StepMobileUIValidation) Struct() *TStep {
hrp/step_request.go
+22 -10
View File
@@ -769,6 +769,18 @@ func (s *StepRequest) IOS() *StepMobile {
}
}
// Shell creates a new shell action
func (s *StepRequest) Shell(content string) *StepShell {
s.step.Shell = &Shell{
String: content,
ExpectExitCode: 0,
}
return &StepShell{
step: s.step,
}
}
// StepRequestWithOptionalArgs implements IStep interface.
type StepRequestWithOptionalArgs struct {
step *TStep
@@ -904,13 +916,13 @@ func (s *StepRequestExtraction) Name() string {
}
func (s *StepRequestExtraction) Type() StepType {
if s.step.Request != nil {
return StepType(fmt.Sprintf("request-%v", s.step.Request.Method))
}
var stepType StepType
if s.step.WebSocket != nil {
return StepType(fmt.Sprintf("websocket-%v", s.step.WebSocket.Type))
stepType = StepType(fmt.Sprintf("websocket-%v", s.step.WebSocket.Type))
} else {
stepType = StepType(fmt.Sprintf("request-%v", s.step.Request.Method))
}
return "extraction"
return stepType + stepTypeSuffixExtraction
}
func (s *StepRequestExtraction) Struct() *TStep {
@@ -940,13 +952,13 @@ func (s *StepRequestValidation) Name() string {
}
func (s *StepRequestValidation) Type() StepType {
if s.step.Request != nil {
return StepType(fmt.Sprintf("request-%v", s.step.Request.Method))
}
var stepType StepType
if s.step.WebSocket != nil {
return StepType(fmt.Sprintf("websocket-%v", s.step.WebSocket.Type))
stepType = StepType(fmt.Sprintf("websocket-%v", s.step.WebSocket.Type))
} else {
stepType = StepType(fmt.Sprintf("request-%v", s.step.Request.Method))
}
return "validation"
return stepType + stepTypeSuffixValidation
}
func (s *StepRequestValidation) Struct() *TStep {
hrp/step_shell.go
+119
View File
@@ -0,0 +1,119 @@
package hrp
import (
"fmt"
"os"
"time"
"github.com/httprunner/funplugin/myexec"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type Shell struct {
String string `json:"string" yaml:"string"`
ExpectExitCode int `json:"expect_exit_code" yaml:"expect_exit_code"`
}
// StepShell implements IStep interface.
type StepShell struct {
step *TStep
}
func (s *StepShell) Name() string {
return s.step.Name
}
func (s *StepShell) Type() StepType {
return stepTypeShell
}
func (s *StepShell) Struct() *TStep {
return s.step
}
func (s *StepShell) Run(r *SessionRunner) (*StepResult, error) {
return runStepShell(r, s.step)
}
// Validate switches to step validation.
func (s *StepShell) Validate() *StepShellValidation {
return &StepShellValidation{
step: s.step,
}
}
// StepShellValidation implements IStep interface.
type StepShellValidation struct {
step *TStep
}
func (s *StepShellValidation) Name() string {
return s.step.Name
}
func (s *StepShellValidation) Type() StepType {
return stepTypeShell + stepTypeSuffixValidation
}
func (s *StepShellValidation) Struct() *TStep {
return s.step
}
func (s *StepShellValidation) Run(r *SessionRunner) (*StepResult, error) {
return runStepShell(r, s.step)
}
func (s *StepShellValidation) AssertExitCode(expected int) *StepShellValidation {
s.step.Shell.ExpectExitCode = expected
return s
}
func runStepShell(r *SessionRunner, step *TStep) (stepResult *StepResult, err error) {
shell := step.Shell
log.Info().
Str("name", step.Name).
Str("type", string(stepTypeShell)).
Str("content", shell.String).
Msg("run shell string")
stepResult = &StepResult{
Name: step.Name,
StepType: stepTypeShell,
Success: false,
Elapsed: 0,
ContentSize: 0,
}
vars := r.caseRunner.parsedConfig.Variables
for key, value := range vars {
os.Setenv(key, fmt.Sprintf("%v", value))
}
start := time.Now()
exitCode, err := myexec.RunShell(shell.String)
stepResult.Elapsed = time.Since(start).Milliseconds()
if err != nil {
if exitCode == shell.ExpectExitCode {
// get expected error
log.Warn().Err(err).
Int("exitCode", exitCode).
Msg("get expected error, ignore")
stepResult.Success = true
return stepResult, nil
}
err = errors.Wrap(err, "exec shell string failed")
return
}
// validate response
if exitCode != shell.ExpectExitCode {
err = fmt.Errorf("unexpected exit code %d, expect %d",
exitCode, shell.ExpectExitCode)
return
}
stepResult.Success = true
return stepResult, nil
}
hrp/testcase.go
+4
View File
@@ -282,6 +282,10 @@ func (tc *TCase) toTestCase() (*TestCase, error) {
testCase.TestSteps = append(testCase.TestSteps, &StepMobile{
step: step,
})
} else if step.Shell != nil {
testCase.TestSteps = append(testCase.TestSteps, &StepShell{
step: step,
})
} else {
log.Warn().Interface("step", step).Msg("[convertTestCase] unexpected step")
}