From b5612dbb2a1332b625a63929326eebc71d39a4d2 Mon Sep 17 00:00:00 2001
From: "lilong.129" <lilong.129@bytedance.com>
Date: Thu, 21 Sep 2023 21:31:10 +0800
Subject: [PATCH] feat: get all close buttons from CV

---
 hrp/pkg/uixt/popups.go        | 32 +++++++++++++++-----------------
 hrp/pkg/uixt/service_vedem.go | 27 ++++++++++++++++-----------
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/hrp/pkg/uixt/popups.go b/hrp/pkg/uixt/popups.go
index cccd00d9..7b197e8b 100644
--- a/hrp/pkg/uixt/popups.go
+++ b/hrp/pkg/uixt/popups.go
@@ -89,22 +89,20 @@ type ClosePopupsResult struct {
 }
 
 type PopupInfo struct {
-	CloseStatus string `json:"close_status"` // found/success/fail
-	Type        string `json:"type"`
-	Text        string `json:"text"`
-	RetryCount  int    `json:"retry_count"`
-	PicName     string `json:"pic_name"`
-	PicURL      string `json:"pic_url"`
-	PopupArea   Box    `json:"popup_area"`
-	CloseArea   Box    `json:"close_area"`
+	CloseStatus string   `json:"close_status"` // found/success/fail
+	RetryCount  int      `json:"retry_count"`
+	CloseBox    Box      `json:"close_box"`              // CV 识别的弹窗关闭按钮（弹窗存在 && 关闭按钮存在）
+	ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个）
 }
 
 func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool {
-	if lastPopup == nil || lastPopup.PopupArea.IsEmpty() {
+	if lastPopup == nil {
 		return false
 	}
-
-	if !p.CloseArea.IsIdentical(lastPopup.CloseArea) {
+	if lastPopup.CloseBox.IsEmpty() {
+		return false
+	}
+	if !p.CloseBox.IsIdentical(lastPopup.CloseBox) {
 		lastPopup.CloseStatus = CloseStatusSuccess
 		return false
 	}
@@ -115,10 +113,7 @@ func (p *PopupInfo) isIdentical(lastPopup *PopupInfo) bool {
 }
 
 func (p *PopupInfo) exists() bool {
-	if p.PopupArea.IsEmpty() || p.CloseArea.IsEmpty() {
-		return false
-	}
-	return true
+	return !p.CloseBox.IsEmpty()
 }
 
 func (dExt *DriverExt) ClosePopups(options ...ActionOption) error {
@@ -144,7 +139,10 @@ func (dExt *DriverExt) ClosePopupsHandler(options ...ActionOption) error {
 	var lastPopup *PopupInfo
 	for retryCount := 0; retryCount < maxRetryTimes; retryCount++ {
 		screenResult, err := dExt.GetScreenResult(
-			WithScreenShotClosePopups(true), WithScreenShotUpload(true))
+			WithScreenShotUpload(true),
+			WithScreenShotClosePopups(true),
+			WithScreenShotUITypes("close"), // get all close buttons
+		)
 		if err != nil {
 			log.Error().Err(err).Msg("get screen result failed for popup handler")
 			continue
@@ -181,7 +179,7 @@ func (dExt *DriverExt) tapPopupHandler(popup *PopupInfo) error {
 	}
 	popup.CloseStatus = CloseStatusFound
 
-	popupClose := popup.CloseArea
+	popupClose := popup.CloseBox
 	if popupClose.IsEmpty() {
 		log.Error().
 			Interface("popup", popup).
diff --git a/hrp/pkg/uixt/service_vedem.go b/hrp/pkg/uixt/service_vedem.go
index 7723a731..beb18e80 100644
--- a/hrp/pkg/uixt/service_vedem.go
+++ b/hrp/pkg/uixt/service_vedem.go
@@ -67,9 +67,9 @@ type ImageResult struct {
 	// Media（媒体）
 	// Chat（语音）
 	// Event（赛事）
-	LiveType string             `json:"liveType,omitempty"`    // 直播间类型
-	UIResult UIResultMap        `json:"uiResult,omitempty"`    // 图标检测
-	CPResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
+	LiveType          string             `json:"liveType,omitempty"`    // 直播间类型
+	UIResult          UIResultMap        `json:"uiResult,omitempty"`    // 图标检测
+	ClosePopupsResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
 }
 
 type APIResponseImage struct {
@@ -407,15 +407,20 @@ func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *S
 		screenResult.UploadedURL = imageResult.URL
 		screenResult.Icons = imageResult.UIResult
 
-		if actionOptions.ScreenShotWithClosePopups && imageResult.CPResult != nil {
-			screenResult.Popup = &PopupInfo{
-				Type:      imageResult.CPResult.Type,
-				Text:      imageResult.CPResult.Text,
-				PicName:   imagePath,
-				PicURL:    imageResult.URL,
-				PopupArea: imageResult.CPResult.PopupArea,
-				CloseArea: imageResult.CPResult.CloseArea,
+		if actionOptions.ScreenShotWithClosePopups {
+			popup := &PopupInfo{}
+
+			closeResult := imageResult.ClosePopupsResult
+			if closeResult != nil && !closeResult.PopupArea.IsEmpty() && !closeResult.CloseArea.IsEmpty() {
+				popup.CloseBox = closeResult.CloseArea
 			}
+
+			closeAreas, _ := imageResult.UIResult.FilterUIResults([]string{"close"})
+			for _, closeArea := range closeAreas {
+				popup.ClosePoints = append(popup.ClosePoints, closeArea.Center())
+			}
+
+			screenResult.Popup = popup
 		}
 	}