diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 54009453..7deaaa3e 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -14,7 +14,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strings" "syscall" "time" @@ -87,40 +86,6 @@ func (screenResults ScreenResultMap) getScreenShotUrls() map[string]string { return screenShotsUrls } -// updatePopupCloseStatus checks if popup closed normally in every screenResult with close_popups on: -func (screenResults ScreenResultMap) updatePopupCloseStatus() { - var popupScreenResultList []*ScreenResult - for _, screenResult := range screenResults { - if screenResult.Popup == nil { - continue - } - popupScreenResultList = append(popupScreenResultList, screenResult) - } - if len(popupScreenResultList) == 0 { - return - } - sort.Slice(popupScreenResultList, func(i, j int) bool { - return popupScreenResultList[i].Popup.RetryCount < popupScreenResultList[j].Popup.RetryCount - }) - - for i := 0; i < len(popupScreenResultList)-1; i++ { - curPopup := popupScreenResultList[i].Popup - nextPopup := popupScreenResultList[i+1].Popup - - // popup not existed, no need to close - if curPopup.CloseArea.IsEmpty() { - continue - } - // popup existed, but identical popups occurs during next retry - if nextPopup.CloseArea.IsIdentical(curPopup.CloseArea) { - popupScreenResultList[i].Popup.CloseStatus = CloseStatusFail - continue - } - // popup existed, but no popup or different popup occurs during next retry (IsClosed=true) - popupScreenResultList[i].Popup.CloseStatus = CloseStatusSuccess - } -} - type cacheStepData struct { // cache step screenshot paths screenShots []string @@ -273,7 +238,6 @@ func (dExt *DriverExt) GetStepCacheData() map[string]interface{} { cacheData["screenshots"] = dExt.cacheStepData.screenShots cacheData["screenshots_urls"] = dExt.cacheStepData.screenResults.getScreenShotUrls() - dExt.cacheStepData.screenResults.updatePopupCloseStatus() cacheData["screen_results"] = dExt.cacheStepData.screenResults // clear cache diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go index 689efa5a..a1d01e07 100644 --- a/hrp/pkg/uixt/popups.go +++ b/hrp/pkg/uixt/popups.go @@ -27,12 +27,6 @@ var popups = [][]string{ {"管理使用时间", ".*忽略.*"}, } -const ( - CloseStatusFound = "found" - CloseStatusSuccess = "success" - CloseStatusFail = "fail" -) - func findTextPopup(screenTexts OCRTexts) (closePoint *OCRText) { for _, popup := range popups { if len(popup) != 2 { @@ -80,6 +74,12 @@ func (dExt *DriverExt) AutoPopupHandler() error { return dExt.handleTextPopup(screenResult.Texts) } +const ( + CloseStatusFound = "found" + CloseStatusSuccess = "success" + CloseStatusFail = "fail" +) + // ClosePopupsResult represents the result of recognized popup to close type ClosePopupsResult struct { Type string `json:"type"` @@ -89,7 +89,7 @@ type ClosePopupsResult struct { } type PopupInfo struct { - CloseStatus string `json:"close_status"` + CloseStatus string `json:"close_status"` // found/success/fail Type string `json:"type"` Text string `json:"text"` RetryCount int `json:"retry_count"` @@ -99,6 +99,25 @@ type PopupInfo struct { CloseArea Box `json:"close_area"` } +func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool { + if lastPopup == nil || lastPopup.PopupArea.IsEmpty() { + return false + } + + if !p.CloseArea.IsIdentical(lastPopup.CloseArea) { + lastPopup.CloseStatus = CloseStatusSuccess + return false + } + + p.CloseStatus = CloseStatusFail + lastPopup.CloseStatus = CloseStatusFail + return true +} + +func (p *PopupInfo) exists() bool { + return p.PopupArea.IsEmpty() || p.CloseArea.IsEmpty() +} + func (dExt *DriverExt) ClosePopups(options ...ActionOption) error { actionOptions := NewActionOptions(options...) @@ -114,11 +133,12 @@ func (dExt *DriverExt) ClosePopups(options ...ActionOption) error { } func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error { - log.Info().Msg("try to find and close popups") actionOptions := NewActionOptions(options...) + log.Info().Interface("actionOptions", actionOptions).Msg("try to find and close popups") maxRetryTimes := actionOptions.MaxRetryTimes interval := actionOptions.Interval + var lastPopup *PopupInfo for retryCount := 0; retryCount < maxRetryTimes; retryCount++ { screenResult, err := dExt.GetScreenResult( WithScreenShotClosePopups(true), WithScreenShotUpload(true)) @@ -126,43 +146,57 @@ func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error { log.Error().Err(err).Msg("get screen result failed for popup handler") continue } - // 1. there are no popups here (fast return normally) - // 2. failed to close popup （maybe tap error, return error） - // 3. successful to close popup (sleep and wait for next retry if existed) - if screenResult.Popup == nil { - break - } - screenResult.Popup.RetryCount = retryCount - if !screenResult.Popup.PopupArea.IsEmpty() { - screenResult.Popup.CloseStatus = CloseStatusFound - } - if screenResult.Popup.CloseArea.IsEmpty() { - break - } - screenResult.Popup.CloseStatus = CloseStatusFound - if err = dExt.tapPopupHandler(screenResult.Popup); err != nil { + popup := screenResult.Popup + if popup == nil || !popup.exists() { + log.Debug().Msg("no popup found") + break + } + popup.CloseStatus = CloseStatusFound + popup.RetryCount = retryCount + + // check if the current popup equals to the last popup + if popup.isIdentical(lastPopup) { + return errors.Wrap(code.MobileUIPopupError, "handle popup failed") + } + + if err = dExt.tapPopupHandler(popup); err != nil { return err } + // sleep for another popup (if existed) to pop time.Sleep(time.Duration(1000*interval) * time.Millisecond) + lastPopup = popup } return nil } func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error { - if popup == nil { + if popup == nil || !popup.exists() { + log.Debug().Msg("no popup found") return nil } - if popup.CloseArea.IsEmpty() { - return nil + popup.CloseStatus = CloseStatusFound + + popupClose := popup.CloseArea + if popupClose.IsEmpty() { + log.Error(). + Interface("popup", popup). + Msg("popup close area not found") + return errors.Wrap(code.MobileUIPopupError, + "popup close area not found") } - log.Info().Str("type", popup.Type).Str("text", popup.Text).Msg("close popup") - popupCenter := popup.CloseArea.Center() - if err := dExt.TapAbsXY(popupCenter.X, popupCenter.Y); err != nil { + + closePoint := popupClose.Center() + log.Info(). + Interface("popup", popup). + Interface("closePoint", closePoint). + Msg("tap to close popup") + if err := dExt.TapAbsXY(closePoint.X, closePoint.Y); err != nil { log.Error().Err(err).Msg("tap popup failed") return errors.Wrap(code.MobileUIPopupError, err.Error()) } + // tap popup success return nil } diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 0e34e23b..4ad1141f 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -148,12 +148,13 @@ func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) if err != nil { return err } - points, err := screenResult.Texts.FindTexts(texts, dExt.ParseActionOptions(optionsWithoutIdentifier...)...) + points, err := screenResult.Texts.FindTexts(texts, + dExt.ParseActionOptions(optionsWithoutIdentifier...)...) if err != nil { - log.Error().Err(err).Msg("swipeToTapTexts failed") + log.Error().Err(err).Strs("texts", texts).Msg("find texts failed") // target texts not found, try to auto handle popup if e := dExt.tapPopupHandler(screenResult.Popup); e != nil { - log.Error().Err(e).Msg("auto handle popup failed") + log.Error().Err(e).Msg("tap popup handler failed") } return err } diff --git a/hrp/pkg/uixt/video_crawler.go b/hrp/pkg/uixt/video_crawler.go index 6df0cbe1..faf31c76 100644 --- a/hrp/pkg/uixt/video_crawler.go +++ b/hrp/pkg/uixt/video_crawler.go @@ -186,7 +186,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { log.Warn().Msg("get current feed video failed") // check and handle popups - if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(1)); err != nil { + if err := crawler.driverExt.ClosePopupsHandler(WithMaxRetryTimes(3)); err != nil { return err } @@ -234,9 +234,8 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { case VideoType_Live: // 直播 - log.Info(). - Interface("video", currentVideo). - Msg(FOUND_LIVE_SUCCESS) + crawler.LiveCount++ + log.Info().Interface("video", currentVideo).Msg(FOUND_LIVE_SUCCESS) // take screenshot and get screen texts by OCR screenResult, err := crawler.driverExt.GetScreenResult( @@ -251,7 +250,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { continue } if e := crawler.driverExt.tapPopupHandler(screenResult.Popup); e != nil { - log.Error().Err(e).Msg("auto handle popup failed") + log.Error().Err(e).Msg("close live popup failed") continue } @@ -262,7 +261,6 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { currentVideo.LiveType = screenResult.imageResult.LiveType } - crawler.LiveCount++ // simulation watch feed video sleepStrict(swipeFinishTime, currentVideo.PlayDuration) @@ -295,6 +293,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { default: // 点播 || 图文 || 广告 || etc. crawler.FeedCount++ + log.Info().Interface("video", currentVideo).Msg(FOUND_FEED_SUCCESS) screenResult := &ScreenResult{ Resolution: dExt.windowSize, @@ -304,11 +303,7 @@ func (dExt *DriverExt) VideoCrawler(configs *VideoCrawlerConfigs) (err error) { SwipeStartTime: swipeStartTime.UnixMilli(), SwipeFinishTime: swipeFinishTime.UnixMilli(), } - dExt.cacheStepData.screenResults[time.Now().String()] = screenResult - log.Info(). - Interface("video", currentVideo). - Msg(FOUND_FEED_SUCCESS) // simulation watch feed video sleepStrict(swipeFinishTime, currentVideo.PlayDuration) diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index 09a3f194..66014878 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -625,7 +625,7 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err // automatic handling of pop-up windows on each step finished if err2 := uiDriver.ClosePopups(); err2 != nil { - log.Error().Err(err2).Str("step", step.Name).Msg("auto handle popup failed") + log.Error().Err(err2).Str("step", step.Name).Msg("handle popup failed on step finished") } // save attachments