GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-06-30 03:51:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: scenario detect of mobile ui automation

Browse Source
This commit is contained in:
xucong.053
2022-11-28 02:23:17 +08:00
committed by xucong.053
parent ef9eaa7e38
commit 3591f72913
7 changed files with 199 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hrp/pkg/uixt/algorithm.go
View File

@@ -325,3 +325,21 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption)
return
}
type SDService interface {
SceneDetection(detectImage []byte, detectType string) (bool, error)
}
func (dExt *DriverExt) ScenarioDetect(scenarioType string, options ...DataOption) (res bool, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
service, err := newVEDEMSDService()
if err != nil {
return
}
return service.SceneDetection(bufSource.Bytes(), scenarioType)
}

51
hrp/pkg/uixt/cp_vedem_test.go Normal file
View File

@@ -0,0 +1,51 @@
//go:build localtest
package uixt
import (
"fmt"
"os"
"testing"
)
func checkCP(buff []byte) error {
service, err := newVEDEMCPService()
if err != nil {
return err
}
sdResults, err := service.getCPResult(buff)
if err != nil {
return err
}
fmt.Println(sdResults)
return nil
}
func TestCPWithScreenshot(t *testing.T) {
device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
driver, err := device.NewUSBDriver(nil)
if err != nil {
t.Fatal(err)
}
raw, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
if err := checkCP(raw.Bytes()); err != nil {
t.Fatal(err)
}
}
func TestCPWithLocalFile(t *testing.T) {
imagePath := "~/Downloads/1669385239_validate_1669385367.png"
file, err := os.ReadFile(imagePath)
if err != nil {
t.Fatal(err)
}
if err := checkCP(file); err != nil {
t.Fatal(err)
}
}

9
hrp/pkg/uixt/ext.go
View File

@@ -47,6 +47,8 @@ const (
SelectorOCR string = "ui_ocr"
SelectorImage string = "ui_image"
SelectorForegroundApp string = "ui_foreground_app"
ScenarioType string = "scenario_type"
// assertions
AssertionEqual string = "equal"
AssertionNotEqual string = "not_equal"
@@ -390,6 +392,11 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool {
return true
}
func (dExt *DriverExt) IsScenarioType(scenarioType string) bool {
_, _, _, _, err := dExt.FindImageRectInUIKit(scenarioType)
return err == nil
}
var errActionNotImplemented = errors.New("UI action not implemented")
func convertToFloat64(val interface{}) (float64, error) {
@@ -749,6 +756,8 @@ func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...s
result = (dExt.IsImageExist(expected) == exp)
case SelectorForegroundApp:
result = (dExt.IsAppInForeground(expected) == exp)
case ScenarioType:
result, _ = dExt.ScenarioDetect(expected)
}
if !result {

132
hrp/pkg/uixt/sd_vedem.go
View File

@@ -3,7 +3,6 @@ package uixt
import (
"bytes"
"fmt"
"image"
"io/ioutil"
"mime/multipart"
"net/http"
@@ -17,15 +16,42 @@ import (
"github.com/rs/zerolog/log"
)
const (
CHECK_LIVE_TYPE_GAME = "checkLiveTypeGame" // 直播游戏
CHECK_LIVE_TYPE_SHOW = "checkLiveTypeShow" // 直播秀场
CHECK_LIVE_TYPE_PEOPLE = "checkLiveTypePeople" // 直播多人
CHECK_LIVE_TYPE_SHOP = "checkLiveTypeShop" // 直播电商
CHECK_BLACK_OR_WHITE = "checkBlackOrWhite" // 黑白屏
CHECK_PART_BLACK_OR_WHITE = "checkPartBlackOrWhite" // 部分黑白屏
CHECK_HALF_BLANK = "checkHalfBlank" // 半白屏检测
CHECK_PAGE_ERROR = "checkPageError" // 页面乱码
CHECK_SNOW = "checkSnow" // 雪花屏
CHECK_OVER_LAY = "checkOverlay" // 图像重叠
CHECK_LOAD_FAILED = "checkLoadFailed" // 图像加载失败检测
CHECK_DETECT_COLOR_BLOCK = "detectColorBlock" // 游戏色块检测
CHECK_PURPLE = "checkPurple" // 游戏紫块
CHECK_WHITE_RECT = "checkWhiteRect" // 游戏白块
CHECK_CORRUPT = "checkCorrupt" // 游戏花屏
CHECK_BLACK_EDGE = "checkBlackEdge" // 游戏黑边
CHECK_WHITE_RATIO = "checkWhiteRatio" // 白屏占比
CHECK_OVER_EXPOSURE = "checkOverExposure" // 图像过爆
CHECK_DIALOG = "checkDialog" // 弹窗检测
CHECK_TEXT_OVER_LAP = "checkTextOverlap" // 文字重叠
CHECK_TEXT_OVER_STEP = "checkTextOverstep" // 文字超框
CHECK_VIDEO_CORRUPT = "checkVideoCorrupt" // 视频花屏
CHECK_GREEN_VIDEO = "checkGreenVideo" // 视频绿屏
CHECK_DEFAULTCHECKED = "checkDefaultChecked" // 合规检测
)
type SDResult struct {
Image string `json:"image"`
Points []PointF `json:"points"`
}
type SDResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Result []SDResult `json:"result"`
Code int `json:"code"`
Message string `json:"message"`
Result bool `json:"result"`
}
type veDEMSDService struct{}
@@ -50,43 +76,33 @@ func checkSDEnv() error {
return nil
}
func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]SDResult, error) {
func (s *veDEMSDService) SceneDetection(detectImage []byte, detectType string) (bool, error) {
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
bodyWriter.WriteField("withDet", "true")
bodyWriter.WriteField("detectType", detectType)
// bodyWriter.WriteField("timestampOnly", "true")
formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png")
formWriter, err := bodyWriter.CreateFormFile("image", "image.png")
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
return false, errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
}
size, err := formWriter.Write(searchImage)
size, err := formWriter.Write(detectImage)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
}
formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png")
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
}
_, err = formWriter.Write(sourceImage)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
return false, errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
}
err = bodyWriter.Close()
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
return false, errors.Wrap(code.CVRequestError,
fmt.Sprintf("close body writer error: %v", err))
}
req, err := http.NewRequest("POST", env.VEDEM_SD_URL, bodyBuf)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
return false, errors.Wrap(code.CVRequestError,
fmt.Sprintf("construct request error: %v", err))
}
@@ -113,87 +129,29 @@ func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]
time.Sleep(1 * time.Second)
}
if resp == nil {
return nil, code.CVServiceConnectionError
return false, code.CVServiceConnectionError
}
defer resp.Body.Close()
results, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
return false, errors.Wrap(code.CVResponseError,
fmt.Sprintf("read response body error: %v", err))
}
if resp.StatusCode != http.StatusOK {
return nil, errors.Wrap(code.CVResponseError,
return false, errors.Wrap(code.CVResponseError,
fmt.Sprintf("unexpected response status code: %d, results: %v",
resp.StatusCode, string(results)))
}
var cvResult SDResponse
err = json.Unmarshal(results, &cvResult)
var sdResult SDResponse
err = json.Unmarshal(results, &sdResult)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
return false, errors.Wrap(code.CVResponseError,
fmt.Sprintf("json unmarshal response body error: %v", err))
}
return cvResult.Result, nil
}
func (s *veDEMSDService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
data := NewData(map[string]interface{}{}, options...)
cvResults, err := s.getSDResult(byteSearch, byteSource)
if err != nil {
log.Error().Err(err).Msg("getSDResult failed")
return
}
var rects []image.Rectangle
var cvImages []string
for _, cvResult := range cvResults {
rect = image.Rectangle{
// cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(cvResult.Points[0].X),
Y: int(cvResult.Points[0].Y),
},
Max: image.Point{
X: int(cvResult.Points[2].X),
Y: int(cvResult.Points[2].Y),
},
}
if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] {
cvImages = append(cvImages, cvResult.Image)
rects = append(rects, rect)
// match exactly, and not specify index, return the first one
if data.Index == 0 {
return rect, nil
}
}
}
if len(rects) == 0 {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("image not found"))
}
// get index
idx := data.Index
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
} else if idx < 0 {
idx = len(rects) + idx
}
// index out of range
if idx >= len(rects) {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("image found, index %d out of range", idx))
}
return rects[idx], nil
return sdResult.Result, nil
}

51
hrp/pkg/uixt/sd_vedem_test.go Normal file
View File

@@ -0,0 +1,51 @@
//go:build localtest
package uixt
import (
"fmt"
"os"
"testing"
)
func checkSD(buff []byte, detectType string) error {
service, err := newVEDEMSDService()
if err != nil {
return err
}
sdResults, err := service.SceneDetection(buff, detectType)
if err != nil {
return err
}
fmt.Println(sdResults)
return nil
}
func TestSDWithScreenshot(t *testing.T) {
device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
driver, err := device.NewUSBDriver(nil)
if err != nil {
t.Fatal(err)
}
raw, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
if err := checkSD(raw.Bytes(), "checkLiveTypeShop"); err != nil {
t.Fatal(err)
}
}
func TestSDWithLocalFile(t *testing.T) {
imagePath := "~/Downloads/s1.png"
file, err := os.ReadFile(imagePath)
if err != nil {
t.Fatal(err)
}
if err := checkSD(file, "checkLiveTypeShop"); err != nil {
t.Fatal(err)
}
}

12
hrp/response.go
View File

@@ -274,7 +274,7 @@ func (v *responseObject) searchRegexp(expr string) interface{} {
return expr
}
func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults []*ValidationResult, err error) {
func validateUI(ud *uixt.DriverExt, iValidators []interface{}, parser *Parser, variablesMapping map[string]interface{}) (validateResults []*ValidationResult, err error) {
for _, iValidator := range iValidators {
validator, ok := iValidator.(Validator)
if !ok {
@@ -287,14 +287,20 @@ func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults
}
// parse check value
if !strings.HasPrefix(validator.Check, "ui_") {
if !strings.HasPrefix(validator.Check, "ui_") && !strings.HasPrefix(validator.Check, "scenario") {
validataResult.CheckResult = "skip"
log.Warn().Interface("validator", validator).Msg("skip validator")
validateResults = append(validateResults, validataResult)
continue
}
expected, ok := validator.Expect.(string)
// parse expected value
expectValue, err := parser.Parse(validator.Expect, variablesMapping)
if err != nil {
return nil, errors.New("validator expect should be string")
}
expected, ok := expectValue.(string)
if !ok {
return nil, errors.New("validator expect should be string")
}

17
hrp/step_mobile_ui.go
View File

@@ -506,6 +506,21 @@ func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg .
return s
}
func (s *StepMobileUIValidation) AssertScenarioType(scenarioType string, msg ...string) *StepMobileUIValidation {
v := Validator{
Check: uixt.ScenarioType,
Assert: uixt.AssertionExists,
Expect: scenarioType,
}
if len(msg) > 0 {
v.Message = msg[0]
} else {
v.Message = fmt.Sprintf("cv image [%s] not found", scenarioType)
}
s.step.Validators = append(s.step.Validators, v)
return s
}
func (s *StepMobileUIValidation) AssertAppNotInForeground(packageName string, msg ...string) *StepMobileUIValidation {
v := Validator{
Check: uixt.SelectorForegroundApp,
@@ -666,7 +681,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
}
// validate
validateResults, err := validateUI(uiDriver, step.Validators)
validateResults, err := validateUI(uiDriver, step.Validators, s.caseRunner.parser, stepVariables)
if err != nil {
if !code.IsErrorPredefined(err) {
err = errors.Wrap(code.MobileUIValidationError, err.Error())