From 6f9946801d3e1341b07d6eff86ef858178b62b8c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 1 Sep 2024 15:45:54 +0800 Subject: [PATCH] refactor: relocate code --- hrp/pkg/uixt/{service_vedem.go => ai.go} | 272 +++--------------- hrp/pkg/uixt/ai_vedem.go | 223 ++++++++++++++ ...service_vedem_test.go => ai_vedem_test.go} | 32 +-- hrp/pkg/uixt/android_test.go | 27 +- hrp/pkg/uixt/interface.go | 16 -- hrp/pkg/uixt/popups.go | 21 +- hrp/pkg/uixt/popups_test.go | 47 +++ hrp/pkg/uixt/screenshot.go | 2 +- hrp/pkg/uixt/swipe_test.go | 6 +- 9 files changed, 336 insertions(+), 310 deletions(-) rename hrp/pkg/uixt/{service_vedem.go => ai.go} (54%) create mode 100644 hrp/pkg/uixt/ai_vedem.go rename hrp/pkg/uixt/{service_vedem_test.go => ai_vedem_test.go} (70%) create mode 100644 hrp/pkg/uixt/popups_test.go diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/ai.go similarity index 54% rename from hrp/pkg/uixt/service_vedem.go rename to hrp/pkg/uixt/ai.go index 52d9dd12..25487ef6 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/ai.go @@ -4,20 +4,13 @@ import ( "bytes" "fmt" "image" - "io" "math" - "mime/multipart" - "net/http" "regexp" - "time" "github.com/pkg/errors" - "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v4/hrp/code" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/env" - "github.com/httprunner/httprunner/v4/hrp/internal/json" ) type IImageService interface { @@ -25,8 +18,23 @@ type IImageService interface { GetImage(imageBuf *bytes.Buffer, options ...ActionOption) (imageResult *ImageResult, err error) } -var client = &http.Client{ - Timeout: time.Second * 10, +type ImageResult struct { + URL string `json:"url,omitempty"` // image uploaded url + OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts + // NoLive（非直播间） + // Shop（电商） + // LifeService（生活服务） + // Show（秀场） + // Game（游戏） + // People（多人） + // PK（PK） + // Media（媒体） + // Chat（语音） + // Event（赛事） + LiveType string `json:"liveType,omitempty"` // 直播间类型 + LivePopularity int64 `json:"livePopularity,omitempty"` // 直播间热度 + UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测 + ClosePopupsResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测 } type OCRResult struct { @@ -58,31 +66,6 @@ func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) { return } -type ImageResult struct { - URL string `json:"url,omitempty"` // image uploaded url - OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts - // NoLive（非直播间） - // Shop（电商） - // LifeService（生活服务） - // Show（秀场） - // Game（游戏） - // People（多人） - // PK（PK） - // Media（媒体） - // Chat（语音） - // Event（赛事） - LiveType string `json:"liveType,omitempty"` // 直播间类型 - LivePopularity int64 `json:"livePopularity,omitempty"` // 直播间热度 - UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测 - ClosePopupsResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测 -} - -type APIResponseImage struct { - Code int `json:"code"` - Message string `json:"message"` - Result ImageResult `json:"result"` -} - type OCRText struct { Text string Rect image.Rectangle @@ -198,199 +181,23 @@ func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (results OC fmt.Sprintf("texts %s not found in %v", texts, t.texts())) } -func newVEDEMImageService() (*veDEMImageService, error) { - if err := checkEnv(); err != nil { - return nil, err - } - return &veDEMImageService{}, nil -} +type UIResultMap map[string]UIResults -// veDEMImageService implements IImageService interface -// actions: -// -// ocr - get ocr texts -// upload - get image uploaded url -// liveType - get live type -// popup - get popup windows -// close - get close popup -// ui - get ui position by type(s) -type veDEMImageService struct{} - -func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOption) (imageResult *ImageResult, err error) { - actionOptions := NewActionOptions(options...) - screenshotActions := actionOptions.screenshotActions() - if len(screenshotActions) == 0 { - // skip - return nil, nil - } - - bodyBuf := &bytes.Buffer{} - bodyWriter := multipart.NewWriter(bodyBuf) - for _, action := range screenshotActions { - bodyWriter.WriteField("actions", action) - } - for _, uiType := range actionOptions.ScreenShotWithUITypes { - bodyWriter.WriteField("uiTypes", uiType) - } - - // 使用高精度集群 - bodyWriter.WriteField("ocrCluster", "highPrecision") - - if actionOptions.ScreenShotWithOCRCluster != "" { - bodyWriter.WriteField("ocrCluster", actionOptions.ScreenShotWithOCRCluster) - } - - if actionOptions.Timeout > 0 { - bodyWriter.WriteField("timeout", fmt.Sprintf("%v", actionOptions.Timeout)) - } else { - bodyWriter.WriteField("timeout", fmt.Sprintf("%v", 10)) - } - - formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") - if err != nil { - err = errors.Wrap(code.CVRequestError, - fmt.Sprintf("create form file error: %v", err)) - return - } - - size, err := formWriter.Write(imageBuf.Bytes()) - if err != nil { - err = errors.Wrap(code.CVRequestError, - fmt.Sprintf("write form error: %v", err)) - return - } - - err = bodyWriter.Close() - if err != nil { - err = errors.Wrap(code.CVRequestError, - fmt.Sprintf("close body writer error: %v", err)) - return - } - var req *http.Request - var resp *http.Response - // retry 3 times - for i := 1; i <= 3; i++ { - copiedBodyBuf := &bytes.Buffer{} - if _, err := copiedBodyBuf.Write(bodyBuf.Bytes()); err != nil { - log.Error().Err(err).Msg("copy screenshot buffer failed") - continue - } - - req, err = http.NewRequest("POST", env.VEDEM_IMAGE_URL, copiedBodyBuf) - if err != nil { - err = errors.Wrap(code.CVRequestError, - fmt.Sprintf("construct request error: %v", err)) +// FilterUIResults filters ui icons, the former the uiTypes, the higher the priority +func (u UIResultMap) FilterUIResults(uiTypes []string) (uiResults UIResults, err error) { + var ok bool + for _, uiType := range uiTypes { + uiResults, ok = u[uiType] + if ok && len(uiResults) != 0 { return } - - // ppe env - // req.Header.Add("x-tt-env", "ppe_vedem_algorithm") - // req.Header.Add("x-use-ppe", "1") - - signToken := "UNSIGNED-PAYLOAD" - token := builtin.Sign("auth-v2", env.VEDEM_IMAGE_AK, env.VEDEM_IMAGE_SK, []byte(signToken)) - - req.Header.Add("Agw-Auth", token) - req.Header.Add("Agw-Auth-Content", signToken) - req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) - - start := time.Now() - resp, err = client.Do(req) - elapsed := time.Since(start) - if err != nil { - log.Error().Err(err). - Int("imageBufSize", size). - Msgf("request veDEM OCR service error, retry %d", i) - continue - } - - logID := getLogID(resp.Header) - statusCode := resp.StatusCode - if statusCode != http.StatusOK { - log.Error(). - Str("X-TT-LOGID", logID). - Int("imageBufSize", size). - Int("statusCode", statusCode). - Msgf("request veDEM OCR service failed, retry %d", i) - time.Sleep(1 * time.Second) - continue - } - - log.Debug(). - Str("X-TT-LOGID", logID). - Int("image_bytes", size). - Int64("elapsed(ms)", elapsed.Milliseconds()). - Msg("request OCR service success") - break } - if resp == nil { - err = code.CVServiceConnectionError - return - } - - defer resp.Body.Close() - - results, err := io.ReadAll(resp.Body) - if err != nil { - err = errors.Wrap(code.CVResponseError, - fmt.Sprintf("read response body error: %v", err)) - return - } - - if resp.StatusCode != http.StatusOK { - err = errors.Wrap(code.CVResponseError, - fmt.Sprintf("unexpected response status code: %d, results: %v", - resp.StatusCode, string(results))) - return - } - - var imageResponse APIResponseImage - err = json.Unmarshal(results, &imageResponse) - if err != nil { - log.Error().Err(err). - Str("response", string(results)). - Msg("json unmarshal veDEM image response body failed") - err = errors.Wrap(code.CVResponseError, - "json unmarshal veDEM image response body error") - return - } - - if imageResponse.Code != 0 { - log.Error(). - Int("code", imageResponse.Code). - Str("message", imageResponse.Message). - Msg("request veDEM OCR service failed") - } - - imageResult = &imageResponse.Result - log.Debug().Interface("imageResult", imageResult).Msg("get image data by veDEM") - return imageResult, nil + err = errors.Wrap(code.CVResultNotFoundError, fmt.Sprintf("UI types %v not detected", uiTypes)) + return } -func checkEnv() error { - if env.VEDEM_IMAGE_URL == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_URL missed") - } - log.Info().Str("VEDEM_IMAGE_URL", env.VEDEM_IMAGE_URL).Msg("get env") - if env.VEDEM_IMAGE_AK == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_AK missed") - } - if env.VEDEM_IMAGE_SK == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_SK missed") - } - return nil -} - -func getLogID(header http.Header) string { - if len(header) == 0 { - return "" - } - - logID, ok := header["X-Tt-Logid"] - if !ok || len(logID) == 0 { - return "" - } - return logID[0] +type UIResult struct { + Box } type Box struct { @@ -417,10 +224,6 @@ func (box Box) Center() PointF { } } -type UIResult struct { - Box -} - type UIResults []UIResult func (u UIResults) FilterScope(scope AbsScope) (results UIResults) { @@ -473,17 +276,10 @@ func (u UIResults) GetUIResult(options ...ActionOption) (UIResult, error) { return uiResults[idx], nil } -type UIResultMap map[string]UIResults - -// FilterUIResults filters ui icons, the former the uiTypes, the higher the priority -func (u UIResultMap) FilterUIResults(uiTypes []string) (uiResults UIResults, err error) { - var ok bool - for _, uiType := range uiTypes { - uiResults, ok = u[uiType] - if ok && len(uiResults) != 0 { - return - } - } - err = errors.Wrap(code.CVResultNotFoundError, fmt.Sprintf("UI types %v not detected", uiTypes)) - return +// ClosePopupsResult represents the result of recognized popup to close +type ClosePopupsResult struct { + Type string `json:"type"` + PopupArea Box `json:"popupArea"` + CloseArea Box `json:"closeArea"` + Text string `json:"text"` } diff --git a/hrp/pkg/uixt/ai_vedem.go b/hrp/pkg/uixt/ai_vedem.go new file mode 100644 index 00000000..00118c35 --- /dev/null +++ b/hrp/pkg/uixt/ai_vedem.go @@ -0,0 +1,223 @@ +package uixt + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/code" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/env" + "github.com/httprunner/httprunner/v4/hrp/internal/json" +) + +var client = &http.Client{ + Timeout: time.Second * 10, +} + +type APIResponseImage struct { + Code int `json:"code"` + Message string `json:"message"` + Result ImageResult `json:"result"` +} + +func newVEDEMImageService() (*veDEMImageService, error) { + if err := checkEnv(); err != nil { + return nil, err + } + return &veDEMImageService{}, nil +} + +// veDEMImageService implements IImageService interface +// actions: +// +// ocr - get ocr texts +// upload - get image uploaded url +// liveType - get live type +// popup - get popup windows +// close - get close popup +// ui - get ui position by type(s) +type veDEMImageService struct{} + +func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOption) (imageResult *ImageResult, err error) { + actionOptions := NewActionOptions(options...) + screenshotActions := actionOptions.screenshotActions() + if len(screenshotActions) == 0 { + // skip + return nil, nil + } + + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + for _, action := range screenshotActions { + bodyWriter.WriteField("actions", action) + } + for _, uiType := range actionOptions.ScreenShotWithUITypes { + bodyWriter.WriteField("uiTypes", uiType) + } + + // 使用高精度集群 + bodyWriter.WriteField("ocrCluster", "highPrecision") + + if actionOptions.ScreenShotWithOCRCluster != "" { + bodyWriter.WriteField("ocrCluster", actionOptions.ScreenShotWithOCRCluster) + } + + if actionOptions.Timeout > 0 { + bodyWriter.WriteField("timeout", fmt.Sprintf("%v", actionOptions.Timeout)) + } else { + bodyWriter.WriteField("timeout", fmt.Sprintf("%v", 10)) + } + + formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") + if err != nil { + err = errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + return + } + + size, err := formWriter.Write(imageBuf.Bytes()) + if err != nil { + err = errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + return + } + + err = bodyWriter.Close() + if err != nil { + err = errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + return + } + var req *http.Request + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + copiedBodyBuf := &bytes.Buffer{} + if _, err := copiedBodyBuf.Write(bodyBuf.Bytes()); err != nil { + log.Error().Err(err).Msg("copy screenshot buffer failed") + continue + } + + req, err = http.NewRequest("POST", env.VEDEM_IMAGE_URL, copiedBodyBuf) + if err != nil { + err = errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + return + } + + // ppe env + // req.Header.Add("x-tt-env", "ppe_vedem_algorithm") + // req.Header.Add("x-use-ppe", "1") + + signToken := "UNSIGNED-PAYLOAD" + token := builtin.Sign("auth-v2", env.VEDEM_IMAGE_AK, env.VEDEM_IMAGE_SK, []byte(signToken)) + + req.Header.Add("Agw-Auth", token) + req.Header.Add("Agw-Auth-Content", signToken) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + + start := time.Now() + resp, err = client.Do(req) + elapsed := time.Since(start) + if err != nil { + log.Error().Err(err). + Int("imageBufSize", size). + Msgf("request veDEM OCR service error, retry %d", i) + continue + } + + logID := getLogID(resp.Header) + statusCode := resp.StatusCode + if statusCode != http.StatusOK { + log.Error(). + Str("X-TT-LOGID", logID). + Int("imageBufSize", size). + Int("statusCode", statusCode). + Msgf("request veDEM OCR service failed, retry %d", i) + time.Sleep(1 * time.Second) + continue + } + + log.Debug(). + Str("X-TT-LOGID", logID). + Int("image_bytes", size). + Int64("elapsed(ms)", elapsed.Milliseconds()). + Msg("request OCR service success") + break + } + if resp == nil { + err = code.CVServiceConnectionError + return + } + + defer resp.Body.Close() + + results, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + return + } + + if resp.StatusCode != http.StatusOK { + err = errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + return + } + + var imageResponse APIResponseImage + err = json.Unmarshal(results, &imageResponse) + if err != nil { + log.Error().Err(err). + Str("response", string(results)). + Msg("json unmarshal veDEM image response body failed") + err = errors.Wrap(code.CVResponseError, + "json unmarshal veDEM image response body error") + return + } + + if imageResponse.Code != 0 { + log.Error(). + Int("code", imageResponse.Code). + Str("message", imageResponse.Message). + Msg("request veDEM OCR service failed") + } + + imageResult = &imageResponse.Result + log.Debug().Interface("imageResult", imageResult).Msg("get image data by veDEM") + return imageResult, nil +} + +func checkEnv() error { + if env.VEDEM_IMAGE_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_URL missed") + } + log.Info().Str("VEDEM_IMAGE_URL", env.VEDEM_IMAGE_URL).Msg("get env") + if env.VEDEM_IMAGE_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_AK missed") + } + if env.VEDEM_IMAGE_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_SK missed") + } + return nil +} + +func getLogID(header http.Header) string { + if len(header) == 0 { + return "" + } + + logID, ok := header["X-Tt-Logid"] + if !ok || len(logID) == 0 { + return "" + } + return logID[0] +} diff --git a/hrp/pkg/uixt/service_vedem_test.go b/hrp/pkg/uixt/ai_vedem_test.go similarity index 70% rename from hrp/pkg/uixt/service_vedem_test.go rename to hrp/pkg/uixt/ai_vedem_test.go index f493773f..b0c0abf3 100644 --- a/hrp/pkg/uixt/service_vedem_test.go +++ b/hrp/pkg/uixt/ai_vedem_test.go @@ -6,7 +6,6 @@ import ( "bytes" "fmt" "os" - "regexp" "testing" ) @@ -24,7 +23,7 @@ func checkOCR(buff *bytes.Buffer) error { } func TestOCRWithScreenshot(t *testing.T) { - setupAndroid(t) + setupAndroidAdbDriver(t) raw, err := driverExt.Driver.Screenshot() if err != nil { @@ -52,27 +51,6 @@ func TestOCRWithLocalFile(t *testing.T) { } } -func matchPopup(text string) bool { - for _, popup := range popups { - if regexp.MustCompile(popup[1]).MatchString(text) { - return true - } - } - return false -} - -func TestMatchRegex(t *testing.T) { - testData := []string{ - "以后再说", "我知道了", "同意", "拒绝", "稍后", - "始终允许", "继续使用", "仅在使用中允许", - } - for _, text := range testData { - if !matchPopup(text) { - t.Fatal(text) - } - } -} - func TestTapUIWithScreenshot(t *testing.T) { serialNumber := os.Getenv("SERIAL_NUMBER") device, _ := NewAndroidDevice(WithSerialNumber(serialNumber)) @@ -98,11 +76,3 @@ func TestDriverExtOCR(t *testing.T) { t.Logf("point.X: %v, point.Y: %v", point.X, point.Y) driverExt.Driver.TapFloat(point.X, point.Y-20) } - -func TestClosePopup(t *testing.T) { - setupAndroid(t) - - if err := driverExt.ClosePopupsHandler(); err != nil { - t.Fatal(err) - } -} diff --git a/hrp/pkg/uixt/android_test.go b/hrp/pkg/uixt/android_test.go index e1be18b5..e643c42b 100644 --- a/hrp/pkg/uixt/android_test.go +++ b/hrp/pkg/uixt/android_test.go @@ -16,7 +16,16 @@ var ( driverExt *DriverExt ) -func setupAndroid(t *testing.T) { +func setupAndroidAdbDriver(t *testing.T) { + device, err := NewAndroidDevice() + checkErr(t, err) + device.UIA2 = false + device.LogOn = false + driverExt, err = device.NewDriver() + checkErr(t, err) +} + +func setupAndroidUIA2Driver(t *testing.T) { device, err := NewAndroidDevice() checkErr(t, err) device.UIA2 = true @@ -122,7 +131,7 @@ func TestDriver_DeviceSize(t *testing.T) { } func TestDriver_Source(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) source, err := driverExt.Driver.Source() if err != nil { @@ -192,7 +201,7 @@ func TestDriver_DeviceInfo(t *testing.T) { } func TestDriver_Tap(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) driverExt.Driver.StartCaptureLog("") err := driverExt.TapXY(0.5, 0.5, WithIdentifier("test"), WithPressDuration(4)) if err != nil { @@ -210,7 +219,7 @@ func TestDriver_Tap(t *testing.T) { } func TestDriver_Swipe(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) err := driverExt.Driver.Swipe(400, 1000, 400, 500, WithPressDuration(0.5)) if err != nil { t.Fatal(err) @@ -218,7 +227,7 @@ func TestDriver_Swipe(t *testing.T) { } func TestDriver_Swipe_Relative(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) err := driverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5) if err != nil { t.Fatal(err) @@ -245,7 +254,7 @@ func TestDriver_Drag(t *testing.T) { } func TestDriver_SendKeys(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) err := driverExt.Driver.SendKeys("辽宁省沈阳市新民市民族街36-4", WithIdentifier("test")) if err != nil { @@ -293,7 +302,7 @@ func TestDriver_SetRotation(t *testing.T) { } func TestDriver_GetOrientation(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) _, _ = driverExt.Driver.AppTerminate("com.quark.browser") _ = driverExt.Driver.AppLaunch("com.quark.browser") time.Sleep(2 * time.Second) @@ -366,7 +375,7 @@ func TestDriver_AppLaunch(t *testing.T) { } func TestDriver_IsAppInForeground(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) err := driverExt.Driver.AppLaunch("com.android.settings") checkErr(t, err) @@ -465,7 +474,7 @@ func TestDriver_ShellInputUnicode(t *testing.T) { } func TestTapTexts(t *testing.T) { - setupAndroid(t) + setupAndroidUIA2Driver(t) actions := []TapTextAction{ {Text: "^.*无视风险安装$", Options: []ActionOption{WithTapOffset(100, 0), WithRegex(true), WithIgnoreNotFoundError(true)}}, {Text: "已了解此应用未经检测.*", Options: []ActionOption{WithTapOffset(-450, 0), WithRegex(true), WithIgnoreNotFoundError(true)}}, diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 30d0598f..0c66acd6 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -323,22 +323,6 @@ const ( NotificationTypeDarwin NotificationType = "darwin" ) -// EventPageID The event page identifier -type EventPageID int - -const EventPageIDConsumer EventPageID = 0x0C - -// EventUsageID The event usage identifier (usages are defined per-page) -type EventUsageID int - -const ( - EventUsageIDCsmrVolumeUp EventUsageID = 0xE9 - EventUsageIDCsmrVolumeDown EventUsageID = 0xEA - EventUsageIDCsmrHome EventUsageID = 0x40 - EventUsageIDCsmrPower EventUsageID = 0x30 - EventUsageIDCsmrSnapshot EventUsageID = 0x65 // Power + Home -) - type Orientation string const ( diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index c4e88763..9ccf7e19 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -71,14 +71,6 @@ func (dExt *DriverExt) AutoPopupHandler() error { return dExt.handleTextPopup(screenResult.Texts) } -// ClosePopupsResult represents the result of recognized popup to close -type ClosePopupsResult struct { - Type string `json:"type"` - PopupArea Box `json:"popupArea"` - CloseArea Box `json:"closeArea"` - Text string `json:"text"` -} - type PopupInfo struct { *ClosePopupsResult ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个） @@ -101,8 +93,14 @@ func (p *PopupInfo) ClosePoint() *PointF { return &closePoint } -func (dExt *DriverExt) CheckPopup() (*PopupInfo, error) { - log.Info().Msg("check if popup exist") +func (dExt *DriverExt) CheckPopup() (popup *PopupInfo, err error) { + defer func() { + if popup == nil { + log.Info().Msg("check popup, no found") + } else { + log.Info().Interface("popup", popup).Msg("found popup") + } + }() screenResult, err := dExt.GetScreenResult( WithScreenShotUpload(true), WithScreenShotClosePopups(true), // get popup area and close area @@ -110,7 +108,7 @@ func (dExt *DriverExt) CheckPopup() (*PopupInfo, error) { if err != nil { return nil, errors.Wrap(err, "get screen result failed for popup handler") } - popup := screenResult.Popup + popup = screenResult.Popup if popup == nil { return nil, errors.New("popup not found") } @@ -119,7 +117,6 @@ func (dExt *DriverExt) CheckPopup() (*PopupInfo, error) { // close point not found return nil, errors.New("popup close point not found") } - log.Info().Interface("popup", popup).Msg("found popup") return popup, nil } diff --git a/hrp/pkg/uixt/popups_test.go b/hrp/pkg/uixt/popups_test.go new file mode 100644 index 00000000..d7ccbbdd --- /dev/null +++ b/hrp/pkg/uixt/popups_test.go @@ -0,0 +1,47 @@ +//go:build localtest + +package uixt + +import ( + "regexp" + "testing" +) + +func TestCheckPopup(t *testing.T) { + setupAndroidAdbDriver(t) + popup, err := driverExt.CheckPopup() + if err != nil { + t.Logf("error: %v", err) + } else { + t.Logf("popup: %+v", popup) + } +} + +func TestClosePopup(t *testing.T) { + setupAndroidAdbDriver(t) + + if err := driverExt.ClosePopupsHandler(); err != nil { + t.Fatal(err) + } +} + +func matchPopup(text string) bool { + for _, popup := range popups { + if regexp.MustCompile(popup[1]).MatchString(text) { + return true + } + } + return false +} + +func TestMatchRegex(t *testing.T) { + testData := []string{ + "以后再说", "我知道了", "同意", "拒绝", "稍后", + "始终允许", "继续使用", "仅在使用中允许", + } + for _, text := range testData { + if !matchPopup(text) { + t.Fatal(text) + } + } +} diff --git a/hrp/pkg/uixt/screenshot.go b/hrp/pkg/uixt/screenshot.go index eb3abf9a..d4970fcd 100644 --- a/hrp/pkg/uixt/screenshot.go +++ b/hrp/pkg/uixt/screenshot.go @@ -56,6 +56,7 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S if err != nil { return } + dExt.cacheStepData.screenShots = append(dExt.cacheStepData.screenShots, imagePath) screenResult = &ScreenResult{ bufSource: bufSource, @@ -171,7 +172,6 @@ func (dExt *DriverExt) GetScreenShot(fileName string) (raw *bytes.Buffer, path s log.Error().Err(err).Msg("save screenshot file failed") return nil, "", err } - dExt.cacheStepData.screenShots = append(dExt.cacheStepData.screenShots, path) return compressed, path, nil } diff --git a/hrp/pkg/uixt/swipe_test.go b/hrp/pkg/uixt/swipe_test.go index e40c41c5..fb2b2a45 100644 --- a/hrp/pkg/uixt/swipe_test.go +++ b/hrp/pkg/uixt/swipe_test.go @@ -7,7 +7,7 @@ import ( ) func TestAndroidSwipeAction(t *testing.T) { - setupAndroid(t) + setupAndroidAdbDriver(t) swipeAction := driverExt.prepareSwipeAction(WithDirection("up")) err := swipeAction(driverExt) @@ -19,14 +19,14 @@ func TestAndroidSwipeAction(t *testing.T) { } func TestAndroidSwipeToTapApp(t *testing.T) { - setupAndroid(t) + setupAndroidAdbDriver(t) err := driverExt.swipeToTapApp("抖音") checkErr(t, err) } func TestAndroidSwipeToTapTexts(t *testing.T) { - setupAndroid(t) + setupAndroidAdbDriver(t) err := driverExt.Driver.AppLaunch("com.ss.android.ugc.aweme") checkErr(t, err)