diff --git a/hrp/pkg/uixt/algorithm.go b/hrp/pkg/uixt/algorithm.go index 6671777c..d3617877 100644 --- a/hrp/pkg/uixt/algorithm.go +++ b/hrp/pkg/uixt/algorithm.go @@ -325,3 +325,21 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) return } + +type SDService interface { + SceneDetection(detectImage []byte, detectType string) (bool, error) +} + +func (dExt *DriverExt) ScenarioDetect(scenarioType string, options ...DataOption) (res bool, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + service, err := newVEDEMSDService() + if err != nil { + return + } + return service.SceneDetection(bufSource.Bytes(), scenarioType) +} diff --git a/hrp/pkg/uixt/cp_vedem_test.go b/hrp/pkg/uixt/cp_vedem_test.go new file mode 100644 index 00000000..5ec15546 --- /dev/null +++ b/hrp/pkg/uixt/cp_vedem_test.go @@ -0,0 +1,51 @@ +//go:build localtest + +package uixt + +import ( + "fmt" + "os" + "testing" +) + +func checkCP(buff []byte) error { + service, err := newVEDEMCPService() + if err != nil { + return err + } + sdResults, err := service.getCPResult(buff) + if err != nil { + return err + } + fmt.Println(sdResults) + return nil +} + +func TestCPWithScreenshot(t *testing.T) { + device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800)) + driver, err := device.NewUSBDriver(nil) + if err != nil { + t.Fatal(err) + } + + raw, err := driver.Screenshot() + if err != nil { + t.Fatal(err) + } + + if err := checkCP(raw.Bytes()); err != nil { + t.Fatal(err) + } +} + +func TestCPWithLocalFile(t *testing.T) { + imagePath := "~/Downloads/1669385239_validate_1669385367.png" + file, err := os.ReadFile(imagePath) + if err != nil { + t.Fatal(err) + } + + if err := checkCP(file); err != nil { + t.Fatal(err) + } +} diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index e49e13b2..ce40bb9e 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -47,6 +47,8 @@ const ( SelectorOCR string = "ui_ocr" SelectorImage string = "ui_image" SelectorForegroundApp string = "ui_foreground_app" + ScenarioType string = "scenario_type" + // assertions AssertionEqual string = "equal" AssertionNotEqual string = "not_equal" @@ -390,6 +392,11 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool { return true } +func (dExt *DriverExt) IsScenarioType(scenarioType string) bool { + _, _, _, _, err := dExt.FindImageRectInUIKit(scenarioType) + return err == nil +} + var errActionNotImplemented = errors.New("UI action not implemented") func convertToFloat64(val interface{}) (float64, error) { @@ -749,6 +756,8 @@ func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...s result = (dExt.IsImageExist(expected) == exp) case SelectorForegroundApp: result = (dExt.IsAppInForeground(expected) == exp) + case ScenarioType: + result, _ = dExt.ScenarioDetect(expected) } if !result { diff --git a/hrp/pkg/uixt/sd_vedem.go b/hrp/pkg/uixt/sd_vedem.go index 45884073..99c1a151 100644 --- a/hrp/pkg/uixt/sd_vedem.go +++ b/hrp/pkg/uixt/sd_vedem.go @@ -3,7 +3,6 @@ package uixt import ( "bytes" "fmt" - "image" "io/ioutil" "mime/multipart" "net/http" @@ -17,15 +16,42 @@ import ( "github.com/rs/zerolog/log" ) +const ( + CHECK_LIVE_TYPE_GAME = "checkLiveTypeGame" // 直播游戏 + CHECK_LIVE_TYPE_SHOW = "checkLiveTypeShow" // 直播秀场 + CHECK_LIVE_TYPE_PEOPLE = "checkLiveTypePeople" // 直播多人 + CHECK_LIVE_TYPE_SHOP = "checkLiveTypeShop" // 直播电商 + CHECK_BLACK_OR_WHITE = "checkBlackOrWhite" // 黑白屏 + CHECK_PART_BLACK_OR_WHITE = "checkPartBlackOrWhite" // 部分黑白屏 + CHECK_HALF_BLANK = "checkHalfBlank" // 半白屏检测 + CHECK_PAGE_ERROR = "checkPageError" // 页面乱码 + CHECK_SNOW = "checkSnow" // 雪花屏 + CHECK_OVER_LAY = "checkOverlay" // 图像重叠 + CHECK_LOAD_FAILED = "checkLoadFailed" // 图像加载失败检测 + CHECK_DETECT_COLOR_BLOCK = "detectColorBlock" // 游戏色块检测 + CHECK_PURPLE = "checkPurple" // 游戏紫块 + CHECK_WHITE_RECT = "checkWhiteRect" // 游戏白块 + CHECK_CORRUPT = "checkCorrupt" // 游戏花屏 + CHECK_BLACK_EDGE = "checkBlackEdge" // 游戏黑边 + CHECK_WHITE_RATIO = "checkWhiteRatio" // 白屏占比 + CHECK_OVER_EXPOSURE = "checkOverExposure" // 图像过爆 + CHECK_DIALOG = "checkDialog" // 弹窗检测 + CHECK_TEXT_OVER_LAP = "checkTextOverlap" // 文字重叠 + CHECK_TEXT_OVER_STEP = "checkTextOverstep" // 文字超框 + CHECK_VIDEO_CORRUPT = "checkVideoCorrupt" // 视频花屏 + CHECK_GREEN_VIDEO = "checkGreenVideo" // 视频绿屏 + CHECK_DEFAULTCHECKED = "checkDefaultChecked" // 合规检测 +) + type SDResult struct { Image string `json:"image"` Points []PointF `json:"points"` } type SDResponse struct { - Code int `json:"code"` - Message string `json:"message"` - Result []SDResult `json:"result"` + Code int `json:"code"` + Message string `json:"message"` + Result bool `json:"result"` } type veDEMSDService struct{} @@ -50,43 +76,33 @@ func checkSDEnv() error { return nil } -func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]SDResult, error) { +func (s *veDEMSDService) SceneDetection(detectImage []byte, detectType string) (bool, error) { bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) bodyWriter.WriteField("withDet", "true") + bodyWriter.WriteField("detectType", detectType) // bodyWriter.WriteField("timestampOnly", "true") - formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png") + formWriter, err := bodyWriter.CreateFormFile("image", "image.png") if err != nil { - return nil, errors.Wrap(code.CVRequestError, + return false, errors.Wrap(code.CVRequestError, fmt.Sprintf("create form file error: %v", err)) } - size, err := formWriter.Write(searchImage) + size, err := formWriter.Write(detectImage) if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("write form error: %v", err)) - } - - formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png") - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("create form file error: %v", err)) - } - _, err = formWriter.Write(sourceImage) - if err != nil { - return nil, errors.Wrap(code.CVRequestError, + return false, errors.Wrap(code.CVRequestError, fmt.Sprintf("write form error: %v", err)) } err = bodyWriter.Close() if err != nil { - return nil, errors.Wrap(code.CVRequestError, + return false, errors.Wrap(code.CVRequestError, fmt.Sprintf("close body writer error: %v", err)) } req, err := http.NewRequest("POST", env.VEDEM_SD_URL, bodyBuf) if err != nil { - return nil, errors.Wrap(code.CVRequestError, + return false, errors.Wrap(code.CVRequestError, fmt.Sprintf("construct request error: %v", err)) } @@ -113,87 +129,29 @@ func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([] time.Sleep(1 * time.Second) } if resp == nil { - return nil, code.CVServiceConnectionError + return false, code.CVServiceConnectionError } defer resp.Body.Close() results, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrap(code.CVResponseError, + return false, errors.Wrap(code.CVResponseError, fmt.Sprintf("read response body error: %v", err)) } if resp.StatusCode != http.StatusOK { - return nil, errors.Wrap(code.CVResponseError, + return false, errors.Wrap(code.CVResponseError, fmt.Sprintf("unexpected response status code: %d, results: %v", resp.StatusCode, string(results))) } - var cvResult SDResponse - err = json.Unmarshal(results, &cvResult) + var sdResult SDResponse + err = json.Unmarshal(results, &sdResult) if err != nil { - return nil, errors.Wrap(code.CVResponseError, + return false, errors.Wrap(code.CVResponseError, fmt.Sprintf("json unmarshal response body error: %v", err)) } - return cvResult.Result, nil -} - -func (s *veDEMSDService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { - data := NewData(map[string]interface{}{}, options...) - - cvResults, err := s.getSDResult(byteSearch, byteSource) - if err != nil { - log.Error().Err(err).Msg("getSDResult failed") - return - } - - var rects []image.Rectangle - var cvImages []string - for _, cvResult := range cvResults { - rect = image.Rectangle{ - // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 - Min: image.Point{ - X: int(cvResult.Points[0].X), - Y: int(cvResult.Points[0].Y), - }, - Max: image.Point{ - X: int(cvResult.Points[2].X), - Y: int(cvResult.Points[2].Y), - }, - } - if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { - cvImages = append(cvImages, cvResult.Image) - - rects = append(rects, rect) - - // match exactly, and not specify index, return the first one - if data.Index == 0 { - return rect, nil - } - } - } - - if len(rects) == 0 { - return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, - fmt.Sprintf("image not found")) - } - - // get index - idx := data.Index - if idx > 0 { - // NOTICE: index start from 1 - idx = idx - 1 - } else if idx < 0 { - idx = len(rects) + idx - } - - // index out of range - if idx >= len(rects) { - return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, - fmt.Sprintf("image found, index %d out of range", idx)) - } - - return rects[idx], nil + return sdResult.Result, nil } diff --git a/hrp/pkg/uixt/sd_vedem_test.go b/hrp/pkg/uixt/sd_vedem_test.go new file mode 100644 index 00000000..5522b268 --- /dev/null +++ b/hrp/pkg/uixt/sd_vedem_test.go @@ -0,0 +1,51 @@ +//go:build localtest + +package uixt + +import ( + "fmt" + "os" + "testing" +) + +func checkSD(buff []byte, detectType string) error { + service, err := newVEDEMSDService() + if err != nil { + return err + } + sdResults, err := service.SceneDetection(buff, detectType) + if err != nil { + return err + } + fmt.Println(sdResults) + return nil +} + +func TestSDWithScreenshot(t *testing.T) { + device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800)) + driver, err := device.NewUSBDriver(nil) + if err != nil { + t.Fatal(err) + } + + raw, err := driver.Screenshot() + if err != nil { + t.Fatal(err) + } + + if err := checkSD(raw.Bytes(), "checkLiveTypeShop"); err != nil { + t.Fatal(err) + } +} + +func TestSDWithLocalFile(t *testing.T) { + imagePath := "~/Downloads/s1.png" + file, err := os.ReadFile(imagePath) + if err != nil { + t.Fatal(err) + } + + if err := checkSD(file, "checkLiveTypeShop"); err != nil { + t.Fatal(err) + } +} diff --git a/hrp/response.go b/hrp/response.go index 5f28b269..713258b0 100644 --- a/hrp/response.go +++ b/hrp/response.go @@ -274,7 +274,7 @@ func (v *responseObject) searchRegexp(expr string) interface{} { return expr } -func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults []*ValidationResult, err error) { +func validateUI(ud *uixt.DriverExt, iValidators []interface{}, parser *Parser, variablesMapping map[string]interface{}) (validateResults []*ValidationResult, err error) { for _, iValidator := range iValidators { validator, ok := iValidator.(Validator) if !ok { @@ -287,14 +287,20 @@ func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults } // parse check value - if !strings.HasPrefix(validator.Check, "ui_") { + if !strings.HasPrefix(validator.Check, "ui_") && !strings.HasPrefix(validator.Check, "scenario") { validataResult.CheckResult = "skip" log.Warn().Interface("validator", validator).Msg("skip validator") validateResults = append(validateResults, validataResult) continue } - expected, ok := validator.Expect.(string) + // parse expected value + expectValue, err := parser.Parse(validator.Expect, variablesMapping) + if err != nil { + return nil, errors.New("validator expect should be string") + } + + expected, ok := expectValue.(string) if !ok { return nil, errors.New("validator expect should be string") } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 7fa7595b..971561e3 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -506,6 +506,21 @@ func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg . return s } +func (s *StepMobileUIValidation) AssertScenarioType(scenarioType string, msg ...string) *StepMobileUIValidation { + v := Validator{ + Check: uixt.ScenarioType, + Assert: uixt.AssertionExists, + Expect: scenarioType, + } + if len(msg) > 0 { + v.Message = msg[0] + } else { + v.Message = fmt.Sprintf("cv image [%s] not found", scenarioType) + } + s.step.Validators = append(s.step.Validators, v) + return s +} + func (s *StepMobileUIValidation) AssertAppNotInForeground(packageName string, msg ...string) *StepMobileUIValidation { v := Validator{ Check: uixt.SelectorForegroundApp, @@ -666,7 +681,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err } // validate - validateResults, err := validateUI(uiDriver, step.Validators) + validateResults, err := validateUI(uiDriver, step.Validators, s.caseRunner.parser, stepVariables) if err != nil { if !code.IsErrorPredefined(err) { err = errors.Wrap(code.MobileUIValidationError, err.Error())