From fa34cd9a8fad85b846a26739da005aac39f42f7b Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 22 Sep 2023 11:11:54 +0800 Subject: [PATCH 1/3] change: handle live popup --- hrp/pkg/uixt/video_crawler.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 1f6a2905..0c51b8f8 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -242,17 +242,12 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { WithScreenShotOCR(true), WithScreenShotUpload(true), WithScreenShotLiveType(true), - WithScreenShotClosePopups(true), ) if err != nil { log.Error().Err(err).Msg("get screen result failed") time.Sleep(3 * time.Second) continue } - if e := crawler.driverExt.tapPopupHandler(screenResult.Popup); e != nil { - log.Error().Err(e).Msg("close live popup failed") - continue - } // add live type if screenResult.imageResult != nil && From 27297da8d366c8dbe631175544279af55a49509b Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 22 Sep 2023 14:15:55 +0800 Subject: [PATCH 2/3] refactor: close popup handler --- hrp/pkg/uixt/action.go | 14 +-- hrp/pkg/uixt/ext.go | 3 + hrp/pkg/uixt/popups.go | 134 +++++++++++++++++------------ hrp/pkg/uixt/service_vedem.go | 7 +- hrp/pkg/uixt/service_vedem_test.go | 16 +--- hrp/pkg/uixt/swipe.go | 5 +- hrp/pkg/uixt/video_crawler.go | 2 +- hrp/step_mobile_ui.go | 2 +- 8 files changed, 90 insertions(+), 93 deletions(-) diff --git a/hrp/pkg/uixt/action.go b/hrp/pkg/uixt/action.go index cd3d7c4a..bf5b48d7 100644 --- a/hrp/pkg/uixt/action.go +++ b/hrp/pkg/uixt/action.go @@ -668,19 +668,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) (err error) { } return dExt.VideoCrawler(configs) case ACTION_ClosePopups: - options := action.GetOptions() - actionOptions := NewActionOptions(options...) - - // default to retry 3 times - if actionOptions.MaxRetryTimes == 0 { - options = append(options, WithMaxRetryTimes(3)) - } - // set default swipe interval to 1 second - if builtin.IsZeroFloat64(actionOptions.Interval) { - options = append(options, WithInterval(1)) - } - - return dExt.ClosePopupsHandler(options...) + return dExt.ClosePopupsHandler() } return nil } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 7deaaa3e..28f09fc9 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -116,6 +116,9 @@ type DriverExt struct { // funplugin plugin funplugin.IPlugin + + // cache last popup to check if popup handle result + lastPopup *PopupInfo } func newDriverExt(device Device, driver WebDriver, plugin funplugin.IPlugin) (dExt *DriverExt, err error) { diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index 2fd032d8..b9158d15 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -88,12 +88,54 @@ type ClosePopupsResult struct { } type PopupInfo struct { + *ClosePopupsResult CloseStatus string `json:"close_status"` // found/success/fail RetryCount int `json:"retry_count"` CloseBox Box `json:"close_box"` // CV 识别的弹窗关闭按钮（弹窗存在 && 关闭按钮存在） ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个） } +func (p *PopupInfo) getClosePoint() (*PointF, error) { + closeResult := p.ClosePopupsResult + if closeResult == nil { + return nil, nil + } + + // 弹框不存在 & 关闭按钮不存在 + if closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() { + if p.ClosePoints == nil { + // 关闭图标不存在 => 100% 确定不存在弹窗 + return nil, nil + } + + // TODO: 结合连续两次关闭图标判断 + // 若连续两次都存在关闭图标，且位置相同 => 存在弹窗 => 点击关闭 + return nil, nil + } + + // 弹窗存在 & 关闭按钮不存在 + if !closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() { + if p.ClosePoints == nil { + // 关闭图标不存在 => 无法处理，抛异常 + log.Error().Interface("popup", p).Msg("popup close area not found") + return nil, errors.New("popup close area not found") + } + + // 使用关闭图标作为关闭按钮 + return &p.ClosePoints[0], nil + } + + closePoint := closeResult.CloseArea.Center() + + // 弹窗不存在 & 关闭按钮存在 => 可能是文字弹窗 => 基于关闭按钮关闭弹窗 + if closeResult.PopupArea.IsEmpty() && !closeResult.CloseArea.IsEmpty() { + return &closePoint, nil + } + + // 弹窗存在 & 关闭按钮存在 => 检测到弹窗存在 => 基于关闭按钮关闭弹窗 + return &closePoint, nil +} + func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool { if lastPopup == nil { return false @@ -111,65 +153,46 @@ func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool { return true } -func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error { - actionOptions := NewActionOptions(options...) - log.Info().Interface("actionOptions", actionOptions).Msg("try to find and close popups") - maxRetryTimes := actionOptions.MaxRetryTimes - interval := actionOptions.Interval +func (dExt *DriverExt) ClosePopupsHandler() (err error) { + log.Info().Msg("try to find and close popups") - var lastPopup *PopupInfo - for retryCount := 0; retryCount < maxRetryTimes; retryCount++ { - screenResult, err := dExt.GetScreenResult( - WithScreenShotUpload(true), - WithScreenShotClosePopups(true), - WithScreenShotUITypes("close"), // get all close buttons - ) - if err != nil { - log.Error().Err(err).Msg("get screen result failed for popup handler") - continue - } - - popup := screenResult.Popup - if popup == nil || popup.CloseBox.IsEmpty() { - log.Debug().Interface("popup", popup).Msg("no popup found") - break - } - popup.CloseStatus = CloseStatusFound - popup.RetryCount = retryCount - - // check if the current popup equals to the last popup - if popup.isIdentical(lastPopup) { - return errors.Wrap(code.MobileUIPopupError, "handle popup failed") - } - - if err = dExt.tapPopupHandler(popup); err != nil { - return err - } - - // sleep for another popup (if existed) to pop - time.Sleep(time.Duration(1000*interval) * time.Millisecond) - lastPopup = popup + screenResult, err := dExt.GetScreenResult( + WithScreenShotUpload(true), + WithScreenShotClosePopups(true), // get popup area and close area + WithScreenShotUITypes("close"), // get all close buttons + ) + if err != nil { + log.Error().Err(err).Msg("get screen result failed for popup handler") + return err } - return nil -} -func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error { - if popup == nil || popup.CloseBox.IsEmpty() { - log.Debug().Interface("popup", popup).Msg("no popup found") + popup := screenResult.Popup + + defer func() { + if err != nil { + dExt.lastPopup = popup + } else { + dExt.lastPopup = nil + } + }() + + // TODO: check if the current popup equals to the last popup + // if popup.isIdentical(dExt.lastPopup) { + // return errors.Wrap(code.MobileUIPopupError, "handle popup failed") + // } + + closePoint, err := popup.getClosePoint() + if err != nil { + return err + } + + if closePoint == nil { + // close point not found + log.Debug().Interface("", popup.ClosePoints).Msg("close point not found") return nil } + popup.CloseStatus = CloseStatusFound - - popupClose := popup.CloseBox - if popupClose.IsEmpty() { - log.Error(). - Interface("popup", popup). - Msg("popup close area not found") - return errors.Wrap(code.MobileUIPopupError, - "popup close area not found") - } - - closePoint := popupClose.Center() log.Info(). Interface("popup", popup). Interface("closePoint", closePoint). @@ -179,6 +202,7 @@ func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error { return errors.Wrap(code.MobileUIPopupError, err.Error()) } - // tap popup success - return nil + // wait 1s and check if popup still exists + time.Sleep(1 * time.Second) + return dExt.ClosePopupsHandler() } diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go index 1660431e..961dbc36 100644 --- a/hrp/pkg/uixt/service_vedem.go +++ b/hrp/pkg/uixt/service_vedem.go @@ -408,11 +408,8 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S screenResult.Icons = imageResult.UIResult if actionOptions.ScreenShotWithClosePopups { - popup := &PopupInfo{} - - closeResult := imageResult.ClosePopupsResult - if closeResult != nil && !closeResult.PopupArea.IsEmpty() && !closeResult.CloseArea.IsEmpty() { - popup.CloseBox = closeResult.CloseArea + popup := &PopupInfo{ + ClosePopupsResult: imageResult.ClosePopupsResult, } closeAreas, _ := imageResult.UIResult.FilterUIResults([]string{"close"}) diff --git a/hrp/pkg/uixt/service_vedem_test.go b/hrp/pkg/uixt/service_vedem_test.go index bc215bee..f493773f 100644 --- a/hrp/pkg/uixt/service_vedem_test.go +++ b/hrp/pkg/uixt/service_vedem_test.go @@ -102,21 +102,7 @@ func TestDriverExtOCR(t *testing.T) { func TestClosePopup(t *testing.T) { setupAndroid(t) - screenResult, err := driverExt.GetScreenResult( - WithScreenShotClosePopups(true), WithScreenShotUpload(true)) - if err != nil { - t.Logf("get screen result failed for popup handler: %v", err) - return - } - t.Logf("screen result: %v", screenResult) - - if screenResult.Popup == nil { - t.Log("there are no popups here") - return - } - t.Logf("popup info: %v", screenResult.Popup) - - if err = driverExt.tapPopupHandler(screenResult.Popup); err != nil { + if err := driverExt.ClosePopupsHandler(); err != nil { t.Fatal(err) } } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 4ad1141f..4388d9e8 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -143,7 +143,6 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) screenResult, err := d.GetScreenResult( WithScreenShotOCR(true), WithScreenShotUpload(true), - WithScreenShotClosePopups(true), ) if err != nil { return err @@ -153,8 +152,8 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) if err != nil { log.Error().Err(err).Strs("texts", texts).Msg("find texts failed") // target texts not found, try to auto handle popup - if e := dExt.tapPopupHandler(screenResult.Popup); e != nil { - log.Error().Err(e).Msg("tap popup handler failed") + if e := dExt.ClosePopupsHandler(); e != nil { + log.Error().Err(e).Msg("run popup handler failed") } return err } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 0c51b8f8..af805ca8 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -186,7 +186,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Warn().Msg("get current feed video failed") // check and handle popups - if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(1)); err != nil { + if err := crawler.driverExt.ClosePopupsHandler(); err != nil { return err } diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 85b8dfa4..f65a9d6a 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -624,7 +624,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err } // automatic handling of pop-up windows on each step finished - if err2 := uiDriver.ClosePopupsHandler(uixt.WithMaxRetryTimes(3), uixt.WithInterval(1)); err2 != nil { + if err2 := uiDriver.ClosePopupsHandler(); err2 != nil { log.Error().Err(err2).Str("step", step.Name).Msg("handle popup failed on step finished") } From fc605d3a588981d98c2f99246e53766181ac6ef1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 22 Sep 2023 15:00:28 +0800 Subject: [PATCH 3/3] change: check close points --- hrp/pkg/uixt/popups.go | 65 ++++++++++++++++------------------- hrp/pkg/uixt/video_crawler.go | 6 ++-- 2 files changed, 33 insertions(+), 38 deletions(-) diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index b9158d15..e64064de 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -89,13 +89,11 @@ type ClosePopupsResult struct { type PopupInfo struct { *ClosePopupsResult - CloseStatus string `json:"close_status"` // found/success/fail - RetryCount int `json:"retry_count"` - CloseBox Box `json:"close_box"` // CV 识别的弹窗关闭按钮（弹窗存在 && 关闭按钮存在） + CloseStatus string `json:"close_status"` // found/success/fail ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个） } -func (p *PopupInfo) getClosePoint() (*PointF, error) { +func (p *PopupInfo) getClosePoint(lastPopup *PopupInfo) (*PointF, error) { closeResult := p.ClosePopupsResult if closeResult == nil { return nil, nil @@ -108,8 +106,28 @@ func (p *PopupInfo) getClosePoint() (*PointF, error) { return nil, nil } - // TODO: 结合连续两次关闭图标判断 - // 若连续两次都存在关闭图标，且位置相同 => 存在弹窗 => 点击关闭 + // 存在关闭按钮，结合上一次的 popup 进行判断 + if lastPopup == nil || lastPopup.ClosePoints == nil { + // 当前关闭图标为首次出现，确定是弹窗关闭按钮的概率较小 + log.Debug().Interface("closePoints", p.ClosePoints). + Msg("skip close points for the first time") + return nil, nil + } + + // 连续两次都存在关闭图标 + if p.ClosePoints[0].IsIdentical(lastPopup.ClosePoints[0]) { + // 连续两次图标位置相同 => 存在弹窗 => 点击关闭 + log.Warn(). + Interface("closePoint", p.ClosePoints[0]). + Interface("lastClosePoints", lastPopup.ClosePoints). + Msg("popup close point detected") + return &p.ClosePoints[0], nil + } + + // 连续两次图标位置不同 => 可能不是弹窗 => skip + log.Debug().Interface("closePoints", p.ClosePoints). + Interface("lastClosePoints", lastPopup.ClosePoints). + Msg("skip close points for not sure") return nil, nil } @@ -136,23 +154,6 @@ func (p *PopupInfo) getClosePoint() (*PointF, error) { return &closePoint, nil } -func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool { - if lastPopup == nil { - return false - } - if lastPopup.CloseBox.IsEmpty() { - return false - } - if !p.CloseBox.IsIdentical(lastPopup.CloseBox) { - lastPopup.CloseStatus = CloseStatusSuccess - return false - } - - p.CloseStatus = CloseStatusFail - lastPopup.CloseStatus = CloseStatusFail - return true -} - func (dExt *DriverExt) ClosePopupsHandler() (err error) { log.Info().Msg("try to find and close popups") @@ -169,33 +170,24 @@ func (dExt *DriverExt) ClosePopupsHandler() (err error) { popup := screenResult.Popup defer func() { - if err != nil { - dExt.lastPopup = popup - } else { - dExt.lastPopup = nil - } + dExt.lastPopup = popup }() - // TODO: check if the current popup equals to the last popup - // if popup.isIdentical(dExt.lastPopup) { - // return errors.Wrap(code.MobileUIPopupError, "handle popup failed") - // } - - closePoint, err := popup.getClosePoint() + closePoint, err := popup.getClosePoint(dExt.lastPopup) if err != nil { return err } if closePoint == nil { // close point not found - log.Debug().Interface("", popup.ClosePoints).Msg("close point not found") + log.Debug().Msg("close point not found") return nil } popup.CloseStatus = CloseStatusFound log.Info(). - Interface("popup", popup). Interface("closePoint", closePoint). + Interface("popup", popup). Msg("tap to close popup") if err := dExt.TapAbsXY(closePoint.X, closePoint.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") @@ -203,6 +195,7 @@ func (dExt *DriverExt) ClosePopupsHandler() (err error) { } // wait 1s and check if popup still exists + log.Info().Msg("tap close point success, check if popup still exists") time.Sleep(1 * time.Second) return dExt.ClosePopupsHandler() } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index af805ca8..7c8e3dae 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -178,12 +178,15 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { // retry 10 times if get feed failed, abort if fail 10 consecutive times currentVideo, err := crawler.getCurrentVideo() if err != nil || currentVideo.Type == "" { + crawler.failedCount++ if crawler.failedCount >= 10 { // failed 10 consecutive times return errors.Wrap(code.TrackingGetError, "get current feed video failed 10 consecutive times") } - log.Warn().Msg("get current feed video failed") + log.Warn(). + Int64("failedCount", crawler.failedCount). + Msg("get current feed video failed") // check and handle popups if err := crawler.driverExt.ClosePopupsHandler(); err != nil { @@ -191,7 +194,6 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { } // retry - crawler.failedCount++ continue }