httprunner/hrp/pkg/uixt/popups.go

package uixt

import (
	"math/rand"

	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"

	"github.com/httprunner/httprunner/v4/hrp/internal/code"
)

// TODO: add more popup texts
var popups = [][]string{
	{".*青少年.*", "我知道了"}, // 青少年弹窗
	{".*个人信息保护.*", "同意"},
	{".*通讯录.*", "拒绝"},
	{".*更新.*", "以后再说|稍后|取消"},
	{".*升级.*", "以后再说|稍后|取消"},
	{".*定位.*", "仅.*允许"},
	{".*拍照.*", "仅.*允许"},
	{".*录音.*", "仅.*允许"},
	{".*位置.*", "仅.*允许"},
	{".*权限.*", "仅.*允许|始终允许"},
	{".*允许.*", "仅.*允许|始终允许"},
	{".*风险.*", "继续使用"},
	{"管理使用时间", ".*忽略.*"},
}

func findTextPopup(screenTexts OCRTexts) (closePoint *OCRText) {
	for _, popup := range popups {
		if len(popup) != 2 {
			continue
		}

		points, err := screenTexts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
		if err == nil {
			log.Warn().Interface("popup", popup).
				Interface("texts", screenTexts).Msg("text popup found")
			closePoint = &points[1]
			break
		}
	}
	return
}

func (dExt *DriverExt) handleTextPopup(screenTexts OCRTexts) error {
	closePoint := findTextPopup(screenTexts)
	if closePoint == nil {
		// no popup found
		return nil
	}

	log.Info().Str("text", closePoint.Text).Msg("close popup")
	pointCenter := closePoint.Center()
	if err := dExt.TapAbsXY(pointCenter.X, pointCenter.Y); err != nil {
		log.Error().Err(err).Msg("tap popup failed")
		return errors.Wrap(code.MobileUIPopupError, err.Error())
	}
	// tap popup success
	return nil
}

func (dExt *DriverExt) AutoPopupHandler() error {
	// TODO: check popup by activity type

	// check popup by screenshot
	screenResult, err := dExt.GetScreenResult(
		WithScreenShotOCR(true), WithScreenShotUpload(true))
	if err != nil {
		return errors.Wrap(err, "get screen result failed for popup handler")
	}

	return dExt.handleTextPopup(screenResult.Texts)
}

const (
	CloseStatusFound   = "found"
	CloseStatusSuccess = "success"
	CloseStatusFail    = "fail"
)

// ClosePopupsResult represents the result of recognized popup to close
type ClosePopupsResult struct {
	Type      string `json:"type"`
	PopupArea Box    `json:"popupArea"`
	CloseArea Box    `json:"closeArea"`
	Text      string `json:"text"`
}

type PopupInfo struct {
	*ClosePopupsResult
	CloseStatus string   `json:"close_status"`           // found/success/fail
	ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个）
}

func (p *PopupInfo) getClosePoint(lastPopup *PopupInfo) (*PointF, error) {
	closeResult := p.ClosePopupsResult
	if closeResult == nil {
		return nil, nil
	}

	// 弹框不存在 && 关闭按钮不存在
	if closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() {
		if p.ClosePoints == nil {
			// 关闭图标不存在 => 100% 确定不存在弹窗
			return nil, nil
		}

		// 存在关闭按钮，结合上一次的 popup 进行判断
		if lastPopup == nil || lastPopup.ClosePoints == nil {
			// 当前关闭图标为首次出现，确定是弹窗关闭按钮的概率较小
			log.Debug().Interface("closePoints", p.ClosePoints).
				Msg("skip close points for the first time")
			return nil, nil
		}

		// 连续两次都存在关闭图标
		if p.ClosePoints[0].IsIdentical(lastPopup.ClosePoints[0]) {
			// 连续两次图标位置相同 => 存在弹窗 => 点击关闭
			log.Warn().
				Interface("closePoint", p.ClosePoints[0]).
				Interface("lastClosePoints", lastPopup.ClosePoints).
				Msg("popup close point detected")
			return getRandomClosePoint(p.ClosePoints), nil
		}

		// 连续两次图标位置不同 => 可能不是弹窗 => skip
		log.Debug().Interface("closePoints", p.ClosePoints).
			Interface("lastClosePoints", lastPopup.ClosePoints).
			Msg("skip close points for not sure")
		return nil, nil
	}

	// 弹窗存在 && 关闭按钮不存在
	if !closeResult.PopupArea.IsEmpty() && closeResult.CloseArea.IsEmpty() {
		if p.ClosePoints == nil {
			// 关闭图标不存在 => 无法处理，抛异常
			log.Error().Interface("popup", p).Msg("popup close area not found")
			return nil, errors.Wrap(code.MobileUIPopupError, "popup close area not found")
		}

		// 使用关闭图标作为关闭按钮（随机选择一个）
		return getRandomClosePoint(p.ClosePoints), nil
	}

	// 关闭按钮存在 && (弹框存在 || 不存在)
	if closeResult.Type != "" || p.ClosePoints == nil {
		// 弹窗类型存在 || 关闭图标不存在 => 基于关闭按钮关闭弹窗
		closePoint := closeResult.CloseArea.Center()
		return &closePoint, nil
	} else {
		// 弹窗类型不存在 && 关闭图标存在，使用关闭图标作为关闭按钮（随机选择一个）
		return getRandomClosePoint(p.ClosePoints), nil
	}
}

func getRandomClosePoint(closePoints []PointF) *PointF {
	if len(closePoints) == 1 {
		return &closePoints[0]
	}
	return &closePoints[rand.Intn(len(closePoints))]
}

func (dExt *DriverExt) ClosePopupsHandler() (err error) {
	log.Info().Msg("try to find and close popups")

	screenResult, err := dExt.GetScreenResult(
		WithScreenShotUpload(true),
		WithScreenShotClosePopups(true), // get popup area and close area
		WithScreenShotUITypes("close"),  // get all close buttons
	)
	if err != nil {
		log.Error().Err(err).Msg("get screen result failed for popup handler")
		return err
	}

	popup := screenResult.Popup

	defer func() {
		dExt.lastPopup = popup
	}()

	closePoint, err := popup.getClosePoint(dExt.lastPopup)
	if err != nil {
		return err
	}

	if closePoint == nil {
		// close point not found
		log.Debug().Msg("close point not found")
		return nil
	}

	popup.CloseStatus = CloseStatusFound
	log.Info().
		Interface("closePoint", closePoint).
		Interface("popup", popup).
		Msg("tap to close popup")
	if err := dExt.TapAbsXY(closePoint.X, closePoint.Y); err != nil {
		log.Error().Err(err).Msg("tap popup failed")
		return errors.Wrap(code.MobileUIPopupError, err.Error())
	}

	// wait 1s and check if popup still exists
	log.Info().Msg("tap close point success, check if popup still exists")
	return nil
}