GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-08-13 01:24:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support closing app popup window by cv service

Browse Source
This commit is contained in:
xucong.053
2022-11-23 22:51:21 +08:00
committed by xucong.053
parent b08411f3d4
commit f6ed75474d
14 changed files with 875 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hrp/internal/code/code.go
View File

@@ -82,6 +82,13 @@ var (
)
// CV related: [90, 100)
var (
CVEnvMissedError = errors.New("CV env missed error") // 90
CVRequestError = errors.New("CV prepare request error") // 91
CVServiceConnectionError = errors.New("CV service connect error") // 92
CVResponseError = errors.New("CV parse response error") // 93
CVImageNotFoundError = errors.New("CV image not found") // 94
)
var errorsMap = map[error]int{
// environment

6
hrp/internal/env/env.go vendored
View File

@@ -13,6 +13,12 @@ var (
VEDEM_OCR_URL = os.Getenv("VEDEM_OCR_URL")
VEDEM_OCR_AK = os.Getenv("VEDEM_OCR_AK")
VEDEM_OCR_SK = os.Getenv("VEDEM_OCR_SK")
VEDEM_IM_URL = os.Getenv("VEDEM_IM_URL")
VEDEM_IM_AK = os.Getenv("VEDEM_IM_AK")
VEDEM_IM_SK = os.Getenv("VEDEM_IM_SK")
VEDEM_CP_URL = os.Getenv("VEDEM_CP_URL")
VEDEM_CP_AK = os.Getenv("VEDEM_CP_AK")
VEDEM_CP_SK = os.Getenv("VEDEM_CP_SK")
DISABLE_GA = os.Getenv("DISABLE_GA")
DISABLE_SENTRY = os.Getenv("DISABLE_SENTRY")
PYPI_INDEX_URL = os.Getenv("PYPI_INDEX_URL")

327
hrp/pkg/uixt/algorithm.go Normal file
View File

@@ -0,0 +1,327 @@
package uixt
import (
"bytes"
"fmt"
"image"
"io/ioutil"
"net/http"
"os"
"strings"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
const (
IconMatching MatchMethod = iota
TemplateMatching
MultiScaleTemplateMatchingPre
MultiScaleTemplateMatching
KAZEMatching
BRISKMatching
AKAZEMatching
ORBMatching
SIFTMatching
SURFMatching
BRIEFMatching
)
func getBufFromDisk(name string) (*bytes.Buffer, error) {
var f *os.File
var err error
if f, err = os.Open(name); err != nil {
return nil, err
}
var all []byte
if all, err = ioutil.ReadAll(f); err != nil {
return nil, err
}
return bytes.NewBuffer(all), nil
}
func getBufFromNetwork(name string) (*bytes.Buffer, error) {
res, err := http.Get(name)
if err != nil {
return nil, err
}
defer func() {
_ = res.Body.Close()
}()
imageBytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
return bytes.NewBuffer(imageBytes), nil
}
type ImageTemplate struct {
FilePath string `json:"filepath"`
rawBuffer *bytes.Buffer
}
func NewImageTemplate(filepath string) ImageTemplate {
return ImageTemplate{FilePath: filepath}
}
func (i *ImageTemplate) read() (imageBuffer *bytes.Buffer, err error) {
if i.rawBuffer.Len() != 0 {
return i.rawBuffer, nil
}
if strings.HasPrefix(i.FilePath, "http://") || strings.HasPrefix(i.FilePath, "https://") {
imageBuffer, err = getBufFromNetwork(i.FilePath)
if err != nil {
return nil, errors.Wrap(err, "failed to request image")
}
} else if builtin.IsFilePathExists(i.FilePath) {
imageBuffer, err = getBufFromDisk(i.FilePath)
if err != nil {
return nil, errors.Wrap(err, "failed to read image")
}
} else {
return nil, errors.New(fmt.Sprintf("not found image: %s", i.FilePath))
}
i.rawBuffer = imageBuffer
return imageBuffer, nil
}
// extendCV 获得扩展后的 Driver，
// 并指定匹配阀值，
// 获取当前设备的 Scale，
func extendCV(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) {
for _, option := range options {
option(&dExt.CVArgs)
}
if dExt.threshold == 0 {
dExt.threshold = 0.95 // default threshold
}
return
}
func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = dExt.matchMode
newExt.threshold = threshold
return
}
func (dExt *DriverExt) OnlyOnceMatchMode(matchMode MatchMode) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = matchMode
newExt.threshold = dExt.threshold
return
}
// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
// pathSource := filepath.Join(sExt.pathname, GenFilename())
// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
// return image.Rectangle{}, err
// }
//
// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), MatchMode(sExt.MatchMode)); err != nil {
// return image.Rectangle{}, err
// }
// return
// }
func (dExt *DriverExt) FindAllImageRect(search string, options ...DataOption) (rects []image.Rectangle, err error) {
var bufSource, bufSearch *bytes.Buffer
im := NewImageTemplate(search)
if bufSearch, err = im.read(); err != nil {
return nil, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return nil, err
}
switch dExt.matchMethod {
case IconMatching:
fallthrough
case TemplateMatching:
rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), MatchMode(dExt.matchMode))
}
return rects, err
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
im := NewImageTemplate(imagePath)
if bufSearch, err = im.read(); err != nil {
return 0, 0, 0, 0, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return 0, 0, 0, 0, err
}
var rect image.Rectangle
switch dExt.matchMethod {
case IconMatching:
service, err := newVEDEMIMService()
if err != nil {
return 0, 0, 0, 0, err
}
rect, err = service.FindImage(bufSearch.Bytes(), bufSource.Bytes(), options...)
case TemplateMatching:
rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), MatchMode(dExt.matchMode))
default:
return 0, 0, 0, 0, errors.New("method not supported")
}
if err != nil {
return 0, 0, 0, 0, err
}
// if rect, err = dExt.findImgRect(search); err != nil {
// return 0, 0, 0, 0, err
// }
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
type CVService interface {
FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error)
}
func (dExt *DriverExt) FindImageByCV(cvImage string, options ...DataOption) (rect image.Rectangle, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
service, err := newVEDEMIMService()
if err != nil {
return
}
im := NewImageTemplate(cvImage)
bufSearch, err := im.read()
if err != nil {
log.Error().Err(err).Msg("failed to get image")
return
}
rect, err = service.FindImage(bufSearch.Bytes(), bufSource.Bytes(), options...)
if err != nil {
log.Warn().Msgf("FindImage failed: %s", err.Error())
return
}
log.Info().Str("cvImage", cvImage).
Interface("rect", rect).Msgf("FindImageByCV success")
return
}
func (dExt *DriverExt) ClosePopupHandler() {
retryCount := 3 // 重试次数
for retryCount > 0 {
rect, err := dExt.FindPopupCloseButton()
if err != nil {
break
}
x, y, width, height := dExt.MappingToRectInUIKit(rect)
pointX := x + width*0.5
pointY := y + height*0.5
err = dExt.TapAbsXY(pointX, pointY)
if err != nil {
break
}
retryCount--
}
}
func (dExt *DriverExt) FindPopupCloseButton(options ...DataOption) (rect image.Rectangle, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
service, err := newVEDEMCPService()
if err != nil {
return
}
rect, err = service.FindPopupCloseButton(bufSource.Bytes(), options...)
if err != nil {
log.Warn().Msgf("FindPopupCloseButton failed: %s", err.Error())
return
}
log.Info().Interface("rect", rect).Msgf("FindPopupCloseButton success")
return
}
type OCRService interface {
GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error)
FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) (rect image.Rectangle, err error)
FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) (rects []image.Rectangle, err error)
}
func (dExt *DriverExt) GetTextsByOCR(options ...DataOption) (texts OCRTexts, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
ocrTexts, err := dExt.ocrService.GetTexts(bufSource, options...)
if err != nil {
log.Error().Err(err).Msg("GetTexts failed")
return
}
return ocrTexts, nil
}
func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
rect, err := dExt.ocrService.FindText(ocrText, bufSource, options...)
if err != nil {
log.Warn().Msgf("FindText failed: %s", err.Error())
return
}
log.Info().Str("ocrText", ocrText).
Interface("rect", rect).Msgf("FindTextByOCR success")
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource, options...)
if err != nil {
log.Warn().Msgf("FindTexts failed: %s", err.Error())
return
}
log.Info().Interface("ocrTexts", ocrTexts).
Interface("rects", rects).Msgf("FindTextsByOCR success")
for _, rect := range rects {
x, y, width, height := dExt.MappingToRectInUIKit(rect)
points = append(points, []float64{x, y, width, height})
}
return
}

12
hrp/pkg/uixt/android_device.go
View File

@@ -59,6 +59,12 @@ func WithAdbLogOn(logOn bool) AndroidDeviceOption {
}
}
func WithAdbClosePopup(isTrue bool) AndroidDeviceOption {
return func(device *AndroidDevice) {
device.ClosePopup = isTrue
}
}
func GetAndroidDeviceOptions(dev *AndroidDevice) (deviceOptions []AndroidDeviceOption) {
if dev.SerialNumber != "" {
deviceOptions = append(deviceOptions, WithSerialNumber(dev.SerialNumber))
@@ -75,6 +81,9 @@ func GetAndroidDeviceOptions(dev *AndroidDevice) (deviceOptions []AndroidDeviceO
if dev.LogOn {
deviceOptions = append(deviceOptions, WithAdbLogOn(true))
}
if dev.ClosePopup {
deviceOptions = append(deviceOptions, WithAdbClosePopup(true))
}
return
}
@@ -145,6 +154,7 @@ type AndroidDevice struct {
UIA2IP string `json:"uia2_ip,omitempty" yaml:"uia2_ip,omitempty"` // uiautomator2 server ip
UIA2Port int `json:"uia2_port,omitempty" yaml:"uia2_port,omitempty"` // uiautomator2 server port
LogOn bool `json:"log_on,omitempty" yaml:"log_on,omitempty"`
ClosePopup bool `json:"close_popup,omitempty" yaml:"close_popup,omitempty"`
}
func (dev *AndroidDevice) UUID() string {
@@ -183,6 +193,8 @@ func (dev *AndroidDevice) NewDriver(capabilities Capabilities) (driverExt *Drive
}
}
driverExt.ClosePopup = dev.ClosePopup
return driverExt, nil
}

367
hrp/pkg/uixt/cv_vedem.go Normal file
View File

@@ -0,0 +1,367 @@
package uixt
import (
"bytes"
"fmt"
"image"
"io/ioutil"
"mime/multipart"
"net/http"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/httprunner/httprunner/v4/hrp/internal/code"
"github.com/httprunner/httprunner/v4/hrp/internal/env"
"github.com/httprunner/httprunner/v4/hrp/internal/json"
)
type IMResult struct {
Image string `json:"image"`
Points []PointF `json:"points"`
}
type IMResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Result []IMResult `json:"result"`
}
type veDEMIMService struct{}
func newVEDEMIMService() (*veDEMIMService, error) {
if err := checkIMEnv(); err != nil {
return nil, err
}
return &veDEMIMService{}, nil
}
func checkIMEnv() error {
if env.VEDEM_IM_URL == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_URL missed")
}
if env.VEDEM_IM_AK == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_AK missed")
}
if env.VEDEM_IM_SK == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_SK missed")
}
return nil
}
func (s *veDEMIMService) getIMResult(searchImage []byte, sourceImage []byte) ([]IMResult, error) {
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
bodyWriter.WriteField("withDet", "true")
// bodyWriter.WriteField("timestampOnly", "true")
formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png")
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
}
size, err := formWriter.Write(searchImage)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
}
formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png")
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
}
_, err = formWriter.Write(sourceImage)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
}
err = bodyWriter.Close()
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("close body writer error: %v", err))
}
req, err := http.NewRequest("POST", env.VEDEM_IM_URL, bodyBuf)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("construct request error: %v", err))
}
token := builtin.Sign("auth-v2", env.VEDEM_IM_AK, env.VEDEM_IM_SK, bodyBuf.Bytes())
req.Header.Add("Agw-Auth", token)
req.Header.Add("Content-Type", bodyWriter.FormDataContentType())
var resp *http.Response
// retry 3 times
for i := 1; i <= 3; i++ {
resp, err = client.Do(req)
if err == nil {
break
}
var logID string
if resp != nil {
logID = getLogID(resp.Header)
}
log.Error().Err(err).
Str("logID", logID).
Int("imageBufSize", size).
Msgf("request CV service failed, retry %d", i)
time.Sleep(1 * time.Second)
}
if resp == nil {
return nil, code.CVServiceConnectionError
}
defer resp.Body.Close()
results, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("read response body error: %v", err))
}
if resp.StatusCode != http.StatusOK {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("unexpected response status code: %d, results: %v",
resp.StatusCode, string(results)))
}
var cvResult IMResponse
err = json.Unmarshal(results, &cvResult)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("json unmarshal response body error: %v", err))
}
return cvResult.Result, nil
}
func (s *veDEMIMService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
data := NewData(map[string]interface{}{}, options...)
cvResults, err := s.getIMResult(byteSearch, byteSource)
if err != nil {
log.Error().Err(err).Msg("getIMResult failed")
return
}
var rects []image.Rectangle
var cvImages []string
for _, cvResult := range cvResults {
rect = image.Rectangle{
// cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(cvResult.Points[0].X),
Y: int(cvResult.Points[0].Y),
},
Max: image.Point{
X: int(cvResult.Points[2].X),
Y: int(cvResult.Points[2].Y),
},
}
if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] {
cvImages = append(cvImages, cvResult.Image)
rects = append(rects, rect)
// match exactly, and not specify index, return the first one
if data.Index == 0 {
return rect, nil
}
}
}
if len(rects) == 0 {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("image not found"))
}
// get index
idx := data.Index
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
} else if idx < 0 {
idx = len(rects) + idx
}
// index out of range
if idx >= len(rects) {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("image found, index %d out of range", idx))
}
return rects[idx], nil
}
type CPResult struct {
Point PointF `json:"point"`
Width float64 `json:"width"`
Height float64 `json:"height"`
}
type CPResponse struct {
Code int `json:"code"`
Message string `json:"message"`
Result []CPResult `json:"result"`
}
type veDEMCPService struct{}
func newVEDEMCPService() (*veDEMCPService, error) {
if err := checkCPEnv(); err != nil {
return nil, err
}
return &veDEMCPService{}, nil
}
func checkCPEnv() error {
if env.VEDEM_CP_URL == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_URL missed")
}
if env.VEDEM_CP_AK == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_AK missed")
}
if env.VEDEM_CP_SK == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_SK missed")
}
return nil
}
func (s *veDEMCPService) getCPResult(sourceImage []byte) ([]CPResult, error) {
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
bodyWriter.WriteField("withDet", "true")
// bodyWriter.WriteField("timestampOnly", "true")
formWriter, err := bodyWriter.CreateFormFile("image", "image.png")
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
}
_, err = formWriter.Write(sourceImage)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
}
err = bodyWriter.Close()
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("close body writer error: %v", err))
}
req, err := http.NewRequest("POST", env.VEDEM_CP_URL, bodyBuf)
if err != nil {
return nil, errors.Wrap(code.CVRequestError,
fmt.Sprintf("construct request error: %v", err))
}
token := builtin.Sign("auth-v2", env.VEDEM_CP_AK, env.VEDEM_CP_SK, bodyBuf.Bytes())
req.Header.Add("Agw-Auth", token)
req.Header.Add("Content-Type", bodyWriter.FormDataContentType())
var resp *http.Response
// retry 3 times
for i := 1; i <= 3; i++ {
resp, err = client.Do(req)
if err == nil {
break
}
var logID string
if resp != nil {
logID = getLogID(resp.Header)
}
log.Error().Err(err).
Str("logID", logID).
Msgf("request CV service failed, retry %d", i)
time.Sleep(1 * time.Second)
}
if resp == nil {
return nil, code.CVServiceConnectionError
}
defer resp.Body.Close()
results, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("read response body error: %v", err))
}
if resp.StatusCode != http.StatusOK {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("unexpected response status code: %d, results: %v",
resp.StatusCode, string(results)))
}
var cpResult CPResponse
err = json.Unmarshal(results, &cpResult)
if err != nil {
return nil, errors.Wrap(code.CVResponseError,
fmt.Sprintf("json unmarshal response body error: %v", err))
}
return cpResult.Result, nil
}
func (s *veDEMCPService) FindPopupCloseButton(byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) {
data := NewData(map[string]interface{}{}, options...)
cpResults, err := s.getCPResult(byteSource)
if err != nil {
log.Error().Err(err).Msg("getCPResult failed")
return
}
var rects []image.Rectangle
for _, cpResult := range cpResults {
rect = image.Rectangle{
// cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(cpResult.Point.X),
Y: int(cpResult.Point.Y),
},
Max: image.Point{
X: int(cpResult.Point.X + cpResult.Width),
Y: int(cpResult.Point.Y + cpResult.Height),
},
}
if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] {
rects = append(rects, rect)
// match exactly, and not specify index, return the first one
if data.Index == 0 {
return rect, nil
}
}
}
if len(rects) == 0 {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("popup close button not found"))
}
// get index
idx := data.Index
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
} else if idx < 0 {
idx = len(rects) + idx
}
// index out of range
if idx >= len(rects) {
return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError,
fmt.Sprintf("popup close button found, index %d out of range", idx))
}
return rects[idx], nil
}

27
hrp/pkg/uixt/ext.go
View File

@@ -187,17 +187,26 @@ func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
}
}
// TemplateMatchMode is the type of the template matching operation.
type TemplateMatchMode int
type MatchMethod int
// MatchMode is the type of the matching operation.
type MatchMode int
type CVArgs struct {
matchMode TemplateMatchMode
threshold float64
matchMethod MatchMethod
matchMode MatchMode
threshold float64
}
type CVOption func(*CVArgs)
func WithTemplateMatchMode(mode TemplateMatchMode) CVOption {
func WithCVMatchMethod(method MatchMethod) CVOption {
return func(args *CVArgs) {
args.matchMethod = method
}
}
func WithCVMatchMode(mode MatchMode) CVOption {
return func(args *CVArgs) {
args.matchMode = mode
}
@@ -216,8 +225,12 @@ type DriverExt struct {
frame *bytes.Buffer
doneMjpegStream chan bool
scale float64
ocrService OCRService // used to get text from image
screenShots []string // cache screenshot paths
ocrService OCRService // used to get text from image
screenShots []string // cache screenshot paths
StartTime time.Time // used to associate screenshots name
perfStop chan struct{} // stop performance monitor
perfData []string // save perf data
ClosePopup bool
CVArgs
}

18
hrp/pkg/uixt/ios_device.go
View File

@@ -3,6 +3,8 @@ package uixt
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"io"
builtinLog "log"
"net"
@@ -13,9 +15,6 @@ import (
"strconv"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/code"
"github.com/httprunner/httprunner/v4/hrp/internal/env"
"github.com/httprunner/httprunner/v4/hrp/pkg/gidevice"
@@ -122,7 +121,11 @@ func WithXCTest(bundleID string) IOSDeviceOption {
device.XCTestBundleID = bundleID
}
}
func WithClosePopup(isTrue bool) IOSDeviceOption {
return func(device *IOSDevice) {
device.ClosePopup = isTrue
}
}
func WithIOSPerfOptions(options ...gidevice.PerfOption) IOSDeviceOption {
return func(device *IOSDevice) {
device.PerfOptions = &gidevice.PerfOptions{}
@@ -214,6 +217,9 @@ func GetIOSDeviceOptions(dev *IOSDevice) (deviceOptions []IOSDeviceOption) {
if dev.DismissAlertButtonSelector != "" {
deviceOptions = append(deviceOptions, WithDismissAlertButtonSelector(dev.DismissAlertButtonSelector))
}
if dev.ClosePopup {
deviceOptions = append(deviceOptions, WithClosePopup(true))
}
return
}
@@ -280,6 +286,8 @@ type IOSDevice struct {
// pcap monitor
pcapStop chan struct{} // stop pcap monitor
pcapFile string // saved pcap file path
ClosePopup bool `json:"close_popup,omitempty" yaml:"close_popup,omitempty"`
}
func (dev *IOSDevice) UUID() string {
@@ -353,6 +361,8 @@ func (dev *IOSDevice) NewDriver(capabilities Capabilities) (driverExt *DriverExt
}
}
driverExt.ClosePopup = dev.ClosePopup
return driverExt, nil
}

61
hrp/pkg/uixt/ocr_vedem.go
View File

@@ -296,64 +296,3 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf *bytes.Buffer, opti
return rects, nil
}
type OCRService interface {
GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error)
FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) (rect image.Rectangle, err error)
FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) (rects []image.Rectangle, err error)
}
func (dExt *DriverExt) GetTextsByOCR(options ...DataOption) (texts OCRTexts, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil {
return
}
ocrTexts, err := dExt.ocrService.GetTexts(bufSource, options...)
if err != nil {
log.Error().Err(err).Msg("GetTexts failed")
return
}
return ocrTexts, nil
}
func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil {
return
}
rect, err := dExt.ocrService.FindText(ocrText, bufSource, options...)
if err != nil {
log.Warn().Msgf("FindText failed: %s", err.Error())
return
}
log.Info().Str("ocrText", ocrText).
Interface("rect", rect).Msgf("FindTextByOCR success")
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil {
return
}
rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource, options...)
if err != nil {
log.Warn().Msgf("FindTexts failed: %s", err.Error())
return
}
log.Info().Interface("ocrTexts", ocrTexts).
Interface("rects", rects).Msgf("FindTextsByOCR success")
for _, rect := range rects {
x, y, width, height := dExt.MappingToRectInUIKit(rect)
points = append(points, []float64{x, y, width, height})
}
return
}

23
hrp/pkg/uixt/opencv_off.go
View File

@@ -1,23 +0,0 @@
//go:build !opencv
package uixt
import (
"image"
"github.com/rs/zerolog/log"
)
func (dExt *DriverExt) extendCV(options ...CVOption) (err error) {
return nil
}
func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
log.Fatal().Msg("opencv is not supported")
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
log.Fatal().Msg("opencv is not supported")
return
}

59
hrp/pkg/uixt/tap.go
View File

@@ -5,6 +5,10 @@ import (
)
func (dExt *DriverExt) TapAbsXY(x, y float64, options ...DataOption) error {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
// tap on absolute coordinate [x, y]
return dExt.Driver.TapFloat(x, y, options...)
}
@@ -22,6 +26,10 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error {
}
func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...)
if err != nil {
return PointF{}, err
@@ -35,6 +43,10 @@ func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point P
}
func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
ps, err := dExt.FindTextsByOCR(ocrText, options...)
if err != nil {
return nil, err
@@ -52,6 +64,10 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin
}
func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...)
if err != nil {
return PointF{}, err
@@ -116,6 +132,11 @@ func (dExt *DriverExt) DoubleTapXY(x, y float64) error {
return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
}
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
x = x * float64(dExt.windowSize.Width)
y = y * float64(dExt.windowSize.Height)
return dExt.Driver.DoubleTapFloat(x, y)
@@ -126,6 +147,18 @@ func (dExt *DriverExt) DoubleTap(param string) (err error) {
}
func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
// click on element, find by name attribute
ele, err := dExt.FindUIElement(param)
if err == nil {
return ele.DoubleTap()
}
var x, y, width, height float64
if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
return err
@@ -133,3 +166,29 @@ func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (
return dExt.Driver.DoubleTapFloat(x+width*xOffset, y+height*yOffset)
}
// TapWithNumber sends one or more taps
func (dExt *DriverExt) TapWithNumber(param string, numberOfTaps int) (err error) {
return dExt.TapWithNumberOffset(param, numberOfTaps, 0.5, 0.5)
}
func (dExt *DriverExt) TapWithNumberOffset(param string, numberOfTaps int, xOffset, yOffset float64) (err error) {
// close popup if necessary
if dExt.ClosePopup {
dExt.ClosePopupHandler()
}
if numberOfTaps <= 0 {
numberOfTaps = 1
}
var x, y, width, height float64
if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
return err
}
x = x + width*xOffset
y = y + height*yOffset
touchActions := NewTouchActions().Tap(NewTouchActionTap().WithXYFloat(x, y).WithCount(numberOfTaps))
return dExt.PerformTouchActions(touchActions)
}

143
hrp/pkg/uixt/opencv.go → hrp/pkg/uixt/template_matching.go
View File

@@ -7,20 +7,16 @@ import (
"fmt"
"image"
"image/color"
"io/ioutil"
"log"
"math"
"os"
"github.com/pkg/errors"
"gocv.io/x/gocv"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
)
const (
// TmCcoeffNormed maps to TM_CCOEFF_NORMED
TmCcoeffNormed TemplateMatchMode = iota
TmCcoeffNormed MatchMode = iota
// TmSqdiff maps to TM_SQDIFF
TmSqdiff
// TmSqdiffNormed maps to TM_SQDIFF_NORMED
@@ -44,109 +40,6 @@ const (
DmNotMatch
)
// extendCV 获得扩展后的 Driver，
// 并指定匹配阀值，
// 获取当前设备的 Scale，
// 默认匹配模式为 TmCcoeffNormed，
// 默认关闭 OpenCV 匹配值计算后的输出
func (dExt *DriverExt) extendCV(options ...CVOption) (err error) {
for _, option := range options {
option(&dExt.CVArgs)
}
if dExt.threshold == 0 {
dExt.threshold = 0.95 // default threshold
}
if dExt.matchMode == 0 {
dExt.matchMode = TmCcoeffNormed // default match mode
}
Debug(DebugMode(DmOff))
return
}
func (dExt *DriverExt) Debug(dm DebugMode) {
Debug(DebugMode(dm))
}
func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = dExt.matchMode
newExt.threshold = threshold
return
}
func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = matchMode
newExt.threshold = dExt.threshold
return
}
// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
// pathSource := filepath.Join(sExt.pathname, GenFilename())
// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
// return image.Rectangle{}, err
// }
//
// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), TemplateMatchMode(sExt.MatchMode)); err != nil {
// return image.Rectangle{}, err
// }
// return
// }
func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(search); err != nil {
return nil, err
}
if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil {
return nil, err
}
if rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil {
return nil, err
}
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err
}
if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil {
return 0, 0, 0, 0, err
}
var rect image.Rectangle
if rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil {
return 0, 0, 0, 0, err
}
// if rect, err = dExt.findImgRect(search); err != nil {
// return 0, 0, 0, 0, err
// }
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func getBufFromDisk(name string) (*bytes.Buffer, error) {
var f *os.File
var err error
if f, err = os.Open(name); err != nil {
return nil, err
}
var all []byte
if all, err = ioutil.ReadAll(f); err != nil {
return nil, err
}
return bytes.NewBuffer(all), nil
}
var debug = DmOff
const dmOutputMsg = `[DEBUG] The current value is '%.4f', the expected value is '%.4f'`
@@ -159,9 +52,13 @@ const DefaultMatchMode = TmCcoeffNormed
var fillColor = color.RGBA{R: 255, G: 255, B: 255, A: 0}
func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) {
func init() {
Debug(DebugMode(DmOff))
}
func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (loc image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
matchMode = []MatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
@@ -174,9 +71,9 @@ func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, m
return getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
}
func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) {
func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (loc image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
matchMode = []MatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
@@ -190,9 +87,9 @@ func FindImageLocationFromDisk(source, search string, threshold float32, matchMo
return getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
}
func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) {
func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (locs []image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
matchMode = []MatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
@@ -224,9 +121,9 @@ func FindAllImageLocationsFromDisk(source, search string, threshold float32, mat
return locs, nil
}
func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) {
func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (locs []image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
matchMode = []MatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
@@ -270,7 +167,7 @@ func getPts(loc image.Point, width, height int) [][]image.Point {
}
}
func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) {
func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return image.Rectangle{}, err
@@ -287,7 +184,7 @@ func FindImageRectFromDisk(source, search string, threshold float32, matchMode .
return
}
func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) {
func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
@@ -309,7 +206,7 @@ func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMo
return
}
func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) {
func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return image.Rectangle{}, err
@@ -326,7 +223,7 @@ func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, match
return
}
func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) {
func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
@@ -401,7 +298,7 @@ func getMatsFromRaw(bufImage, bufTpl *bytes.Buffer, flags gocv.IMReadFlag) (matI
}
// getMatchingLocation 获取匹配的图片位置
func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode TemplateMatchMode) (loc image.Point, err error) {
func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode MatchMode) (loc image.Point, err error) {
if threshold > 1 {
threshold = 1.0
}
@@ -431,7 +328,7 @@ func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32,
}
// getMatchingResult 匹配图片并返回匹配值和位置
func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) {
func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode MatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) {
matResult, tmpMask := gocv.NewMat(), gocv.NewMat()
defer func() {
_ = matResult.Close()
@@ -443,7 +340,7 @@ func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMat
}
// getValLoc 根据不同的匹配模式返回匹配值和位置
func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode TemplateMatchMode) (val float32, loc image.Point) {
func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode MatchMode) (val float32, loc image.Point) {
val, loc = maxVal, maxLoc
switch matchMode {

20
hrp/pkg/uixt/template_matching_off.go Normal file
View File

@@ -0,0 +1,20 @@
//go:build !opencv
package uixt
import (
"bytes"
"image"
"github.com/rs/zerolog/log"
)
func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) {
log.Fatal().Msg("opencv is not supported")
return
}
func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) {
log.Fatal().Msg("opencv is not supported")
return
}

0
hrp/pkg/uixt/opencv_test.go → hrp/pkg/uixt/template_matching_test.go
View File

23
hrp/step_mobile_ui.go
View File

@@ -11,6 +11,29 @@ import (
"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
)
// ios setting options
var (
WithUDID = uixt.WithUDID
WithWDAPort = uixt.WithWDAPort
WithWDAMjpegPort = uixt.WithWDAMjpegPort
WithLogOn = uixt.WithLogOn
WithResetHomeOnStartup = uixt.WithResetHomeOnStartup
WithSnapshotMaxDepth = uixt.WithSnapshotMaxDepth
WithAcceptAlertButtonSelector = uixt.WithAcceptAlertButtonSelector
WithDismissAlertButtonSelector = uixt.WithDismissAlertButtonSelector
WithClosePopup = uixt.WithClosePopup
WithPerfOptions = uixt.WithPerfOptions
)
// android setting options
var (
WithSerialNumber = uixt.WithSerialNumber
WithAdbIP = uixt.WithAdbIP
WithAdbPort = uixt.WithAdbPort
WithAdbLogOn = uixt.WithAdbLogOn
WithAdbClosePopup = uixt.WithAdbClosePopup
)
type MobileStep struct {
Serial string `json:"serial,omitempty" yaml:"serial,omitempty"`
uixt.MobileAction `yaml:",inline"`