feat: scenario detect of mobile ui automation

hrp/pkg/uixt/algorithm.go

@@ -325,3 +325,21 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption)
 
 	return
 }
+
+type SDService interface {
+	SceneDetection(detectImage []byte, detectType string) (bool, error)
+}
+
+func (dExt *DriverExt) ScenarioDetect(scenarioType string, options ...DataOption) (res bool, err error) {
+	var bufSource *bytes.Buffer
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		err = fmt.Errorf("takeScreenShot error: %v", err)
+		return
+	}
+
+	service, err := newVEDEMSDService()
+	if err != nil {
+		return
+	}
+	return service.SceneDetection(bufSource.Bytes(), scenarioType)
+}

hrp/pkg/uixt/cp_vedem_test.go

@@ -0,0 +1,51 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func checkCP(buff []byte) error {
+	service, err := newVEDEMCPService()
+	if err != nil {
+		return err
+	}
+	sdResults, err := service.getCPResult(buff)
+	if err != nil {
+		return err
+	}
+	fmt.Println(sdResults)
+	return nil
+}
+
+func TestCPWithScreenshot(t *testing.T) {
+	device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
+	driver, err := device.NewUSBDriver(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	raw, err := driver.Screenshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := checkCP(raw.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCPWithLocalFile(t *testing.T) {
+	imagePath := "~/Downloads/1669385239_validate_1669385367.png"
+	file, err := os.ReadFile(imagePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := checkCP(file); err != nil {
+		t.Fatal(err)
+	}
+}

hrp/pkg/uixt/ext.go

@@ -47,6 +47,8 @@ const (
 	SelectorOCR           string = "ui_ocr"
 	SelectorImage         string = "ui_image"
 	SelectorForegroundApp string = "ui_foreground_app"
+	ScenarioType          string = "scenario_type"
+
 	// assertions
 	AssertionEqual     string = "equal"
 	AssertionNotEqual  string = "not_equal"
@@ -390,6 +392,11 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool {
 	return true
 }
 
+func (dExt *DriverExt) IsScenarioType(scenarioType string) bool {
+	_, _, _, _, err := dExt.FindImageRectInUIKit(scenarioType)
+	return err == nil
+}
+
 var errActionNotImplemented = errors.New("UI action not implemented")
 
 func convertToFloat64(val interface{}) (float64, error) {
@@ -749,6 +756,8 @@ func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...s
 		result = (dExt.IsImageExist(expected) == exp)
 	case SelectorForegroundApp:
 		result = (dExt.IsAppInForeground(expected) == exp)
+	case ScenarioType:
+		result, _ = dExt.ScenarioDetect(expected)
 	}
 
 	if !result {

hrp/pkg/uixt/sd_vedem.go

@@ -3,7 +3,6 @@ package uixt
 import (
 	"bytes"
 	"fmt"
-	"image"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
@@ -17,15 +16,42 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+const (
+	CHECK_LIVE_TYPE_GAME      = "checkLiveTypeGame"     // 直播游戏
+	CHECK_LIVE_TYPE_SHOW      = "checkLiveTypeShow"     // 直播秀场
+	CHECK_LIVE_TYPE_PEOPLE    = "checkLiveTypePeople"   // 直播多人
+	CHECK_LIVE_TYPE_SHOP      = "checkLiveTypeShop"     // 直播电商
+	CHECK_BLACK_OR_WHITE      = "checkBlackOrWhite"     // 黑白屏
+	CHECK_PART_BLACK_OR_WHITE = "checkPartBlackOrWhite" // 部分黑白屏
+	CHECK_HALF_BLANK          = "checkHalfBlank"        // 半白屏检测
+	CHECK_PAGE_ERROR          = "checkPageError"        // 页面乱码
+	CHECK_SNOW                = "checkSnow"             // 雪花屏
+	CHECK_OVER_LAY            = "checkOverlay"          // 图像重叠
+	CHECK_LOAD_FAILED         = "checkLoadFailed"       // 图像加载失败检测
+	CHECK_DETECT_COLOR_BLOCK  = "detectColorBlock"      // 游戏色块检测
+	CHECK_PURPLE              = "checkPurple"           // 游戏紫块
+	CHECK_WHITE_RECT          = "checkWhiteRect"        // 游戏白块
+	CHECK_CORRUPT             = "checkCorrupt"          // 游戏花屏
+	CHECK_BLACK_EDGE          = "checkBlackEdge"        // 游戏黑边
+	CHECK_WHITE_RATIO         = "checkWhiteRatio"       // 白屏占比
+	CHECK_OVER_EXPOSURE       = "checkOverExposure"     // 图像过爆
+	CHECK_DIALOG              = "checkDialog"           // 弹窗检测
+	CHECK_TEXT_OVER_LAP       = "checkTextOverlap"      // 文字重叠
+	CHECK_TEXT_OVER_STEP      = "checkTextOverstep"     // 文字超框
+	CHECK_VIDEO_CORRUPT       = "checkVideoCorrupt"     // 视频花屏
+	CHECK_GREEN_VIDEO         = "checkGreenVideo"       // 视频绿屏
+	CHECK_DEFAULTCHECKED      = "checkDefaultChecked"   // 合规检测
+)
+
 type SDResult struct {
 	Image  string   `json:"image"`
 	Points []PointF `json:"points"`
 }
 
 type SDResponse struct {
-	Code    int        `json:"code"`
+	Code    int    `json:"code"`
-	Message string     `json:"message"`
+	Message string `json:"message"`
-	Result  []SDResult `json:"result"`
+	Result  bool   `json:"result"`
 }
 
 type veDEMSDService struct{}
@@ -50,43 +76,33 @@ func checkSDEnv() error {
 	return nil
 }
 
-func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]SDResult, error) {
+func (s *veDEMSDService) SceneDetection(detectImage []byte, detectType string) (bool, error) {
 	bodyBuf := &bytes.Buffer{}
 	bodyWriter := multipart.NewWriter(bodyBuf)
 	bodyWriter.WriteField("withDet", "true")
+	bodyWriter.WriteField("detectType", detectType)
 	// bodyWriter.WriteField("timestampOnly", "true")
 
-	formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png")
+	formWriter, err := bodyWriter.CreateFormFile("image", "image.png")
 	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
+		return false, errors.Wrap(code.CVRequestError,
 			fmt.Sprintf("create form file error: %v", err))
 	}
-	size, err := formWriter.Write(searchImage)
+	size, err := formWriter.Write(detectImage)
 	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
+		return false, errors.Wrap(code.CVRequestError,
-			fmt.Sprintf("write form error: %v", err))
-	}
-
-	formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png")
-	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
-			fmt.Sprintf("create form file error: %v", err))
-	}
-	_, err = formWriter.Write(sourceImage)
-	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
 			fmt.Sprintf("write form error: %v", err))
 	}
 
 	err = bodyWriter.Close()
 	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
+		return false, errors.Wrap(code.CVRequestError,
 			fmt.Sprintf("close body writer error: %v", err))
 	}
 
 	req, err := http.NewRequest("POST", env.VEDEM_SD_URL, bodyBuf)
 	if err != nil {
-		return nil, errors.Wrap(code.CVRequestError,
+		return false, errors.Wrap(code.CVRequestError,
 			fmt.Sprintf("construct request error: %v", err))
 	}
 
@@ -113,87 +129,29 @@ func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]
 		time.Sleep(1 * time.Second)
 	}
 	if resp == nil {
-		return nil, code.CVServiceConnectionError
+		return false, code.CVServiceConnectionError
 	}
 
 	defer resp.Body.Close()
 
 	results, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, errors.Wrap(code.CVResponseError,
+		return false, errors.Wrap(code.CVResponseError,
 			fmt.Sprintf("read response body error: %v", err))
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, errors.Wrap(code.CVResponseError,
+		return false, errors.Wrap(code.CVResponseError,
 			fmt.Sprintf("unexpected response status code: %d, results: %v",
 				resp.StatusCode, string(results)))
 	}
 
-	var cvResult SDResponse
+	var sdResult SDResponse
-	err = json.Unmarshal(results, &cvResult)
+	err = json.Unmarshal(results, &sdResult)
 	if err != nil {
-		return nil, errors.Wrap(code.CVResponseError,
+		return false, errors.Wrap(code.CVResponseError,
 			fmt.Sprintf("json unmarshal response body error: %v", err))
 	}
 
-	return cvResult.Result, nil
+	return sdResult.Result, nil
-}
-
-func (s *veDEMSDService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
-	data := NewData(map[string]interface{}{}, options...)
-
-	cvResults, err := s.getSDResult(byteSearch, byteSource)
-	if err != nil {
-		log.Error().Err(err).Msg("getSDResult failed")
-		return
-	}
-
-	var rects []image.Rectangle
-	var cvImages []string
-	for _, cvResult := range cvResults {
-		rect = image.Rectangle{
-			// cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
-			Min: image.Point{
-				X: int(cvResult.Points[0].X),
-				Y: int(cvResult.Points[0].Y),
-			},
-			Max: image.Point{
-				X: int(cvResult.Points[2].X),
-				Y: int(cvResult.Points[2].Y),
-			},
-		}
-		if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] {
-			cvImages = append(cvImages, cvResult.Image)
-
-			rects = append(rects, rect)
-
-			// match exactly, and not specify index, return the first one
-			if data.Index == 0 {
-				return rect, nil
-			}
-		}
-	}
-
-	if len(rects) == 0 {
-		return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
-			fmt.Sprintf("image not found"))
-	}
-
-	// get index
-	idx := data.Index
-	if idx > 0 {
-		// NOTICE: index start from 1
-		idx = idx - 1
-	} else if idx < 0 {
-		idx = len(rects) + idx
-	}
-
-	// index out of range
-	if idx >= len(rects) {
-		return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
-			fmt.Sprintf("image found, index %d out of range", idx))
-	}
-
-	return rects[idx], nil
 }

hrp/pkg/uixt/sd_vedem_test.go

@@ -0,0 +1,51 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func checkSD(buff []byte, detectType string) error {
+	service, err := newVEDEMSDService()
+	if err != nil {
+		return err
+	}
+	sdResults, err := service.SceneDetection(buff, detectType)
+	if err != nil {
+		return err
+	}
+	fmt.Println(sdResults)
+	return nil
+}
+
+func TestSDWithScreenshot(t *testing.T) {
+	device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
+	driver, err := device.NewUSBDriver(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	raw, err := driver.Screenshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := checkSD(raw.Bytes(), "checkLiveTypeShop"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSDWithLocalFile(t *testing.T) {
+	imagePath := "~/Downloads/s1.png"
+	file, err := os.ReadFile(imagePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := checkSD(file, "checkLiveTypeShop"); err != nil {
+		t.Fatal(err)
+	}
+}

hrp/response.go

@@ -274,7 +274,7 @@ func (v *responseObject) searchRegexp(expr string) interface{} {
 	return expr
 }
 
-func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults []*ValidationResult, err error) {
+func validateUI(ud *uixt.DriverExt, iValidators []interface{}, parser *Parser, variablesMapping map[string]interface{}) (validateResults []*ValidationResult, err error) {
 	for _, iValidator := range iValidators {
 		validator, ok := iValidator.(Validator)
 		if !ok {
@@ -287,14 +287,20 @@ func validateUI(ud *uixt.DriverExt, iValidators []interface{}) (validateResults
 		}
 
 		// parse check value
-		if !strings.HasPrefix(validator.Check, "ui_") {
+		if !strings.HasPrefix(validator.Check, "ui_") && !strings.HasPrefix(validator.Check, "scenario") {
 			validataResult.CheckResult = "skip"
 			log.Warn().Interface("validator", validator).Msg("skip validator")
 			validateResults = append(validateResults, validataResult)
 			continue
 		}
 
-		expected, ok := validator.Expect.(string)
+		// parse expected value
+		expectValue, err := parser.Parse(validator.Expect, variablesMapping)
+		if err != nil {
+			return nil, errors.New("validator expect should be string")
+		}
+
+		expected, ok := expectValue.(string)
 		if !ok {
 			return nil, errors.New("validator expect should be string")
 		}

hrp/step_mobile_ui.go

@@ -506,6 +506,21 @@ func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg .
 	return s
 }
 
+func (s *StepMobileUIValidation) AssertScenarioType(scenarioType string, msg ...string) *StepMobileUIValidation {
+	v := Validator{
+		Check:  uixt.ScenarioType,
+		Assert: uixt.AssertionExists,
+		Expect: scenarioType,
+	}
+	if len(msg) > 0 {
+		v.Message = msg[0]
+	} else {
+		v.Message = fmt.Sprintf("cv image [%s] not found", scenarioType)
+	}
+	s.step.Validators = append(s.step.Validators, v)
+	return s
+}
+
 func (s *StepMobileUIValidation) AssertAppNotInForeground(packageName string, msg ...string) *StepMobileUIValidation {
 	v := Validator{
 		Check:  uixt.SelectorForegroundApp,
@@ -666,7 +681,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
 	}
 
 	// validate
-	validateResults, err := validateUI(uiDriver, step.Validators)
+	validateResults, err := validateUI(uiDriver, step.Validators, s.caseRunner.parser, stepVariables)
 	if err != nil {
 		if !code.IsErrorPredefined(err) {
 			err = errors.Wrap(code.MobileUIValidationError, err.Error())