From f6ed75474dd8e6027df9eb7ebac62b12b0a23d23 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Wed, 23 Nov 2022 22:51:21 +0800 Subject: [PATCH] feat: support closing app popup window by cv service --- hrp/internal/code/code.go | 7 + hrp/internal/env/env.go | 6 + hrp/pkg/uixt/algorithm.go | 327 ++++++++++++++++ hrp/pkg/uixt/android_device.go | 12 + hrp/pkg/uixt/cv_vedem.go | 367 ++++++++++++++++++ hrp/pkg/uixt/ext.go | 27 +- hrp/pkg/uixt/ios_device.go | 18 +- hrp/pkg/uixt/ocr_vedem.go | 61 --- hrp/pkg/uixt/opencv_off.go | 23 -- hrp/pkg/uixt/tap.go | 59 +++ .../uixt/{opencv.go => template_matching.go} | 143 +------ hrp/pkg/uixt/template_matching_off.go | 20 + ...encv_test.go => template_matching_test.go} | 0 hrp/step_mobile_ui.go | 23 ++ 14 files changed, 875 insertions(+), 218 deletions(-) create mode 100644 hrp/pkg/uixt/algorithm.go create mode 100644 hrp/pkg/uixt/cv_vedem.go delete mode 100644 hrp/pkg/uixt/opencv_off.go rename hrp/pkg/uixt/{opencv.go => template_matching.go} (71%) create mode 100644 hrp/pkg/uixt/template_matching_off.go rename hrp/pkg/uixt/{opencv_test.go => template_matching_test.go} (100%) diff --git a/hrp/internal/code/code.go b/hrp/internal/code/code.go index 3479c8b6..ed4f2e30 100644 --- a/hrp/internal/code/code.go +++ b/hrp/internal/code/code.go @@ -82,6 +82,13 @@ var ( ) // CV related: [90, 100) +var ( + CVEnvMissedError = errors.New("CV env missed error") // 90 + CVRequestError = errors.New("CV prepare request error") // 91 + CVServiceConnectionError = errors.New("CV service connect error") // 92 + CVResponseError = errors.New("CV parse response error") // 93 + CVImageNotFoundError = errors.New("CV image not found") // 94 +) var errorsMap = map[error]int{ // environment diff --git a/hrp/internal/env/env.go b/hrp/internal/env/env.go index 03b0c3c9..dda2d1e9 100644 --- a/hrp/internal/env/env.go +++ b/hrp/internal/env/env.go @@ -13,6 +13,12 @@ var ( VEDEM_OCR_URL = os.Getenv("VEDEM_OCR_URL") VEDEM_OCR_AK = os.Getenv("VEDEM_OCR_AK") VEDEM_OCR_SK = os.Getenv("VEDEM_OCR_SK") + VEDEM_IM_URL = os.Getenv("VEDEM_IM_URL") + VEDEM_IM_AK = os.Getenv("VEDEM_IM_AK") + VEDEM_IM_SK = os.Getenv("VEDEM_IM_SK") + VEDEM_CP_URL = os.Getenv("VEDEM_CP_URL") + VEDEM_CP_AK = os.Getenv("VEDEM_CP_AK") + VEDEM_CP_SK = os.Getenv("VEDEM_CP_SK") DISABLE_GA = os.Getenv("DISABLE_GA") DISABLE_SENTRY = os.Getenv("DISABLE_SENTRY") PYPI_INDEX_URL = os.Getenv("PYPI_INDEX_URL") diff --git a/hrp/pkg/uixt/algorithm.go b/hrp/pkg/uixt/algorithm.go new file mode 100644 index 00000000..6671777c --- /dev/null +++ b/hrp/pkg/uixt/algorithm.go @@ -0,0 +1,327 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +const ( + IconMatching MatchMethod = iota + TemplateMatching + MultiScaleTemplateMatchingPre + MultiScaleTemplateMatching + KAZEMatching + BRISKMatching + AKAZEMatching + ORBMatching + SIFTMatching + SURFMatching + BRIEFMatching +) + +func getBufFromDisk(name string) (*bytes.Buffer, error) { + var f *os.File + var err error + if f, err = os.Open(name); err != nil { + return nil, err + } + var all []byte + if all, err = ioutil.ReadAll(f); err != nil { + return nil, err + } + return bytes.NewBuffer(all), nil +} + +func getBufFromNetwork(name string) (*bytes.Buffer, error) { + res, err := http.Get(name) + if err != nil { + return nil, err + } + defer func() { + _ = res.Body.Close() + }() + + imageBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return bytes.NewBuffer(imageBytes), nil +} + +type ImageTemplate struct { + FilePath string `json:"filepath"` + rawBuffer *bytes.Buffer +} + +func NewImageTemplate(filepath string) ImageTemplate { + return ImageTemplate{FilePath: filepath} +} + +func (i *ImageTemplate) read() (imageBuffer *bytes.Buffer, err error) { + if i.rawBuffer.Len() != 0 { + return i.rawBuffer, nil + } + if strings.HasPrefix(i.FilePath, "http://") || strings.HasPrefix(i.FilePath, "https://") { + imageBuffer, err = getBufFromNetwork(i.FilePath) + if err != nil { + return nil, errors.Wrap(err, "failed to request image") + } + } else if builtin.IsFilePathExists(i.FilePath) { + imageBuffer, err = getBufFromDisk(i.FilePath) + if err != nil { + return nil, errors.Wrap(err, "failed to read image") + } + } else { + return nil, errors.New(fmt.Sprintf("not found image: %s", i.FilePath)) + } + i.rawBuffer = imageBuffer + + return imageBuffer, nil +} + +// extendCV 获得扩展后的 Driver， +// 并指定匹配阀值， +// 获取当前设备的 Scale， +func extendCV(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) { + for _, option := range options { + option(&dExt.CVArgs) + } + + if dExt.threshold == 0 { + dExt.threshold = 0.95 // default threshold + } + + return +} + +func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.Driver = dExt.Driver + newExt.scale = dExt.scale + newExt.matchMode = dExt.matchMode + newExt.threshold = threshold + return +} + +func (dExt *DriverExt) OnlyOnceMatchMode(matchMode MatchMode) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.Driver = dExt.Driver + newExt.scale = dExt.scale + newExt.matchMode = matchMode + newExt.threshold = dExt.threshold + return +} + +// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { +// pathSource := filepath.Join(sExt.pathname, GenFilename()) +// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { +// return image.Rectangle{}, err +// } +// +// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), MatchMode(sExt.MatchMode)); err != nil { +// return image.Rectangle{}, err +// } +// return +// } + +func (dExt *DriverExt) FindAllImageRect(search string, options ...DataOption) (rects []image.Rectangle, err error) { + var bufSource, bufSearch *bytes.Buffer + im := NewImageTemplate(search) + if bufSearch, err = im.read(); err != nil { + return nil, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return nil, err + } + + switch dExt.matchMethod { + case IconMatching: + fallthrough + case TemplateMatching: + rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), MatchMode(dExt.matchMode)) + } + return rects, err +} + +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { + var bufSource, bufSearch *bytes.Buffer + im := NewImageTemplate(imagePath) + if bufSearch, err = im.read(); err != nil { + return 0, 0, 0, 0, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return 0, 0, 0, 0, err + } + + var rect image.Rectangle + switch dExt.matchMethod { + case IconMatching: + service, err := newVEDEMIMService() + if err != nil { + return 0, 0, 0, 0, err + } + rect, err = service.FindImage(bufSearch.Bytes(), bufSource.Bytes(), options...) + case TemplateMatching: + rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), MatchMode(dExt.matchMode)) + default: + return 0, 0, 0, 0, errors.New("method not supported") + } + + if err != nil { + return 0, 0, 0, 0, err + } + + // if rect, err = dExt.findImgRect(search); err != nil { + // return 0, 0, 0, 0, err + // } + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + +type CVService interface { + FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) +} + +func (dExt *DriverExt) FindImageByCV(cvImage string, options ...DataOption) (rect image.Rectangle, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + service, err := newVEDEMIMService() + if err != nil { + return + } + im := NewImageTemplate(cvImage) + bufSearch, err := im.read() + if err != nil { + log.Error().Err(err).Msg("failed to get image") + return + } + rect, err = service.FindImage(bufSearch.Bytes(), bufSource.Bytes(), options...) + if err != nil { + log.Warn().Msgf("FindImage failed: %s", err.Error()) + return + } + + log.Info().Str("cvImage", cvImage). + Interface("rect", rect).Msgf("FindImageByCV success") + return +} + +func (dExt *DriverExt) ClosePopupHandler() { + retryCount := 3 // 重试次数 + for retryCount > 0 { + rect, err := dExt.FindPopupCloseButton() + if err != nil { + break + } + + x, y, width, height := dExt.MappingToRectInUIKit(rect) + pointX := x + width*0.5 + pointY := y + height*0.5 + err = dExt.TapAbsXY(pointX, pointY) + if err != nil { + break + } + + retryCount-- + } +} + +func (dExt *DriverExt) FindPopupCloseButton(options ...DataOption) (rect image.Rectangle, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + service, err := newVEDEMCPService() + if err != nil { + return + } + + rect, err = service.FindPopupCloseButton(bufSource.Bytes(), options...) + if err != nil { + log.Warn().Msgf("FindPopupCloseButton failed: %s", err.Error()) + return + } + + log.Info().Interface("rect", rect).Msgf("FindPopupCloseButton success") + return +} + +type OCRService interface { + GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error) + FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) (rect image.Rectangle, err error) + FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) (rects []image.Rectangle, err error) +} + +func (dExt *DriverExt) GetTextsByOCR(options ...DataOption) (texts OCRTexts, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + ocrTexts, err := dExt.ocrService.GetTexts(bufSource, options...) + if err != nil { + log.Error().Err(err).Msg("GetTexts failed") + return + } + + return ocrTexts, nil +} + +func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + rect, err := dExt.ocrService.FindText(ocrText, bufSource, options...) + if err != nil { + log.Warn().Msgf("FindText failed: %s", err.Error()) + return + } + + log.Info().Str("ocrText", ocrText). + Interface("rect", rect).Msgf("FindTextByOCR success") + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource, options...) + if err != nil { + log.Warn().Msgf("FindTexts failed: %s", err.Error()) + return + } + + log.Info().Interface("ocrTexts", ocrTexts). + Interface("rects", rects).Msgf("FindTextsByOCR success") + for _, rect := range rects { + x, y, width, height := dExt.MappingToRectInUIKit(rect) + points = append(points, []float64{x, y, width, height}) + } + + return +} diff --git a/hrp/pkg/uixt/android_device.go b/hrp/pkg/uixt/android_device.go index b511659d..d17aeb6e 100644 --- a/hrp/pkg/uixt/android_device.go +++ b/hrp/pkg/uixt/android_device.go @@ -59,6 +59,12 @@ func WithAdbLogOn(logOn bool) AndroidDeviceOption { } } +func WithAdbClosePopup(isTrue bool) AndroidDeviceOption { + return func(device *AndroidDevice) { + device.ClosePopup = isTrue + } +} + func GetAndroidDeviceOptions(dev *AndroidDevice) (deviceOptions []AndroidDeviceOption) { if dev.SerialNumber != "" { deviceOptions = append(deviceOptions, WithSerialNumber(dev.SerialNumber)) @@ -75,6 +81,9 @@ func GetAndroidDeviceOptions(dev *AndroidDevice) (deviceOptions []AndroidDeviceO if dev.LogOn { deviceOptions = append(deviceOptions, WithAdbLogOn(true)) } + if dev.ClosePopup { + deviceOptions = append(deviceOptions, WithAdbClosePopup(true)) + } return } @@ -145,6 +154,7 @@ type AndroidDevice struct { UIA2IP string `json:"uia2_ip,omitempty" yaml:"uia2_ip,omitempty"` // uiautomator2 server ip UIA2Port int `json:"uia2_port,omitempty" yaml:"uia2_port,omitempty"` // uiautomator2 server port LogOn bool `json:"log_on,omitempty" yaml:"log_on,omitempty"` + ClosePopup bool `json:"close_popup,omitempty" yaml:"close_popup,omitempty"` } func (dev *AndroidDevice) UUID() string { @@ -183,6 +193,8 @@ func (dev *AndroidDevice) NewDriver(capabilities Capabilities) (driverExt *Drive } } + driverExt.ClosePopup = dev.ClosePopup + return driverExt, nil } diff --git a/hrp/pkg/uixt/cv_vedem.go b/hrp/pkg/uixt/cv_vedem.go new file mode 100644 index 00000000..7d0c70ca --- /dev/null +++ b/hrp/pkg/uixt/cv_vedem.go @@ -0,0 +1,367 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "io/ioutil" + "mime/multipart" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" + "github.com/httprunner/httprunner/v4/hrp/internal/env" + "github.com/httprunner/httprunner/v4/hrp/internal/json" +) + +type IMResult struct { + Image string `json:"image"` + Points []PointF `json:"points"` +} + +type IMResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result []IMResult `json:"result"` +} + +type veDEMIMService struct{} + +func newVEDEMIMService() (*veDEMIMService, error) { + if err := checkIMEnv(); err != nil { + return nil, err + } + return &veDEMIMService{}, nil +} + +func checkIMEnv() error { + if env.VEDEM_IM_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_URL missed") + } + if env.VEDEM_IM_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_AK missed") + } + if env.VEDEM_IM_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_IM_SK missed") + } + return nil +} + +func (s *veDEMIMService) getIMResult(searchImage []byte, sourceImage []byte) ([]IMResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("withDet", "true") + // bodyWriter.WriteField("timestampOnly", "true") + + formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + size, err := formWriter.Write(searchImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + _, err = formWriter.Write(sourceImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + err = bodyWriter.Close() + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + } + + req, err := http.NewRequest("POST", env.VEDEM_IM_URL, bodyBuf) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + } + + token := builtin.Sign("auth-v2", env.VEDEM_IM_AK, env.VEDEM_IM_SK, bodyBuf.Bytes()) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + resp, err = client.Do(req) + if err == nil { + break + } + + var logID string + if resp != nil { + logID = getLogID(resp.Header) + } + log.Error().Err(err). + Str("logID", logID). + Int("imageBufSize", size). + Msgf("request CV service failed, retry %d", i) + time.Sleep(1 * time.Second) + } + if resp == nil { + return nil, code.CVServiceConnectionError + } + + defer resp.Body.Close() + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + } + + var cvResult IMResponse + err = json.Unmarshal(results, &cvResult) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("json unmarshal response body error: %v", err)) + } + + return cvResult.Result, nil +} + +func (s *veDEMIMService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := NewData(map[string]interface{}{}, options...) + + cvResults, err := s.getIMResult(byteSearch, byteSource) + if err != nil { + log.Error().Err(err).Msg("getIMResult failed") + return + } + + var rects []image.Rectangle + var cvImages []string + for _, cvResult := range cvResults { + rect = image.Rectangle{ + // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(cvResult.Points[0].X), + Y: int(cvResult.Points[0].Y), + }, + Max: image.Point{ + X: int(cvResult.Points[2].X), + Y: int(cvResult.Points[2].Y), + }, + } + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { + cvImages = append(cvImages, cvResult.Image) + + rects = append(rects, rect) + + // match exactly, and not specify index, return the first one + if data.Index == 0 { + return rect, nil + } + } + } + + if len(rects) == 0 { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image not found")) + } + + // get index + idx := data.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image found, index %d out of range", idx)) + } + + return rects[idx], nil +} + +type CPResult struct { + Point PointF `json:"point"` + Width float64 `json:"width"` + Height float64 `json:"height"` +} + +type CPResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result []CPResult `json:"result"` +} + +type veDEMCPService struct{} + +func newVEDEMCPService() (*veDEMCPService, error) { + if err := checkCPEnv(); err != nil { + return nil, err + } + return &veDEMCPService{}, nil +} + +func checkCPEnv() error { + if env.VEDEM_CP_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_URL missed") + } + if env.VEDEM_CP_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_AK missed") + } + if env.VEDEM_CP_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_SK missed") + } + return nil +} + +func (s *veDEMCPService) getCPResult(sourceImage []byte) ([]CPResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("withDet", "true") + // bodyWriter.WriteField("timestampOnly", "true") + + formWriter, err := bodyWriter.CreateFormFile("image", "image.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + _, err = formWriter.Write(sourceImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + err = bodyWriter.Close() + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + } + + req, err := http.NewRequest("POST", env.VEDEM_CP_URL, bodyBuf) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + } + + token := builtin.Sign("auth-v2", env.VEDEM_CP_AK, env.VEDEM_CP_SK, bodyBuf.Bytes()) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + resp, err = client.Do(req) + if err == nil { + break + } + + var logID string + if resp != nil { + logID = getLogID(resp.Header) + } + log.Error().Err(err). + Str("logID", logID). + Msgf("request CV service failed, retry %d", i) + time.Sleep(1 * time.Second) + } + if resp == nil { + return nil, code.CVServiceConnectionError + } + + defer resp.Body.Close() + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + } + + var cpResult CPResponse + err = json.Unmarshal(results, &cpResult) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("json unmarshal response body error: %v", err)) + } + + return cpResult.Result, nil +} + +func (s *veDEMCPService) FindPopupCloseButton(byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := NewData(map[string]interface{}{}, options...) + + cpResults, err := s.getCPResult(byteSource) + if err != nil { + log.Error().Err(err).Msg("getCPResult failed") + return + } + + var rects []image.Rectangle + for _, cpResult := range cpResults { + rect = image.Rectangle{ + // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(cpResult.Point.X), + Y: int(cpResult.Point.Y), + }, + Max: image.Point{ + X: int(cpResult.Point.X + cpResult.Width), + Y: int(cpResult.Point.Y + cpResult.Height), + }, + } + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { + rects = append(rects, rect) + + // match exactly, and not specify index, return the first one + if data.Index == 0 { + return rect, nil + } + } + } + + if len(rects) == 0 { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("popup close button not found")) + } + + // get index + idx := data.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("popup close button found, index %d out of range", idx)) + } + + return rects[idx], nil +} diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index ca09aaed..e49e13b2 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -187,17 +187,26 @@ func WithIgnoreNotFoundError(ignoreError bool) ActionOption { } } -// TemplateMatchMode is the type of the template matching operation. -type TemplateMatchMode int +type MatchMethod int + +// MatchMode is the type of the matching operation. +type MatchMode int type CVArgs struct { - matchMode TemplateMatchMode - threshold float64 + matchMethod MatchMethod + matchMode MatchMode + threshold float64 } type CVOption func(*CVArgs) -func WithTemplateMatchMode(mode TemplateMatchMode) CVOption { +func WithCVMatchMethod(method MatchMethod) CVOption { + return func(args *CVArgs) { + args.matchMethod = method + } +} + +func WithCVMatchMode(mode MatchMode) CVOption { return func(args *CVArgs) { args.matchMode = mode } @@ -216,8 +225,12 @@ type DriverExt struct { frame *bytes.Buffer doneMjpegStream chan bool scale float64 - ocrService OCRService // used to get text from image - screenShots []string // cache screenshot paths + ocrService OCRService // used to get text from image + screenShots []string // cache screenshot paths + StartTime time.Time // used to associate screenshots name + perfStop chan struct{} // stop performance monitor + perfData []string // save perf data + ClosePopup bool CVArgs } diff --git a/hrp/pkg/uixt/ios_device.go b/hrp/pkg/uixt/ios_device.go index 93828a8d..6d30bbad 100644 --- a/hrp/pkg/uixt/ios_device.go +++ b/hrp/pkg/uixt/ios_device.go @@ -3,6 +3,8 @@ package uixt import ( "context" "fmt" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" "io" builtinLog "log" "net" @@ -13,9 +15,6 @@ import ( "strconv" "time" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/httprunner/httprunner/v4/hrp/internal/code" "github.com/httprunner/httprunner/v4/hrp/internal/env" "github.com/httprunner/httprunner/v4/hrp/pkg/gidevice" @@ -122,7 +121,11 @@ func WithXCTest(bundleID string) IOSDeviceOption { device.XCTestBundleID = bundleID } } - +func WithClosePopup(isTrue bool) IOSDeviceOption { + return func(device *IOSDevice) { + device.ClosePopup = isTrue + } +} func WithIOSPerfOptions(options ...gidevice.PerfOption) IOSDeviceOption { return func(device *IOSDevice) { device.PerfOptions = &gidevice.PerfOptions{} @@ -214,6 +217,9 @@ func GetIOSDeviceOptions(dev *IOSDevice) (deviceOptions []IOSDeviceOption) { if dev.DismissAlertButtonSelector != "" { deviceOptions = append(deviceOptions, WithDismissAlertButtonSelector(dev.DismissAlertButtonSelector)) } + if dev.ClosePopup { + deviceOptions = append(deviceOptions, WithClosePopup(true)) + } return } @@ -280,6 +286,8 @@ type IOSDevice struct { // pcap monitor pcapStop chan struct{} // stop pcap monitor pcapFile string // saved pcap file path + + ClosePopup bool `json:"close_popup,omitempty" yaml:"close_popup,omitempty"` } func (dev *IOSDevice) UUID() string { @@ -353,6 +361,8 @@ func (dev *IOSDevice) NewDriver(capabilities Capabilities) (driverExt *DriverExt } } + driverExt.ClosePopup = dev.ClosePopup + return driverExt, nil } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 15db742d..ee392b17 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -296,64 +296,3 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf *bytes.Buffer, opti return rects, nil } - -type OCRService interface { - GetTexts(imageBuf *bytes.Buffer, options ...DataOption) (ocrTexts OCRTexts, err error) - FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) (rect image.Rectangle, err error) - FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) (rects []image.Rectangle, err error) -} - -func (dExt *DriverExt) GetTextsByOCR(options ...DataOption) (texts OCRTexts, err error) { - var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { - return - } - - ocrTexts, err := dExt.ocrService.GetTexts(bufSource, options...) - if err != nil { - log.Error().Err(err).Msg("GetTexts failed") - return - } - - return ocrTexts, nil -} - -func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { - var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { - return - } - - rect, err := dExt.ocrService.FindText(ocrText, bufSource, options...) - if err != nil { - log.Warn().Msgf("FindText failed: %s", err.Error()) - return - } - - log.Info().Str("ocrText", ocrText). - Interface("rect", rect).Msgf("FindTextByOCR success") - x, y, width, height = dExt.MappingToRectInUIKit(rect) - return -} - -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) { - var bufSource *bytes.Buffer - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_ocr")); err != nil { - return - } - - rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource, options...) - if err != nil { - log.Warn().Msgf("FindTexts failed: %s", err.Error()) - return - } - - log.Info().Interface("ocrTexts", ocrTexts). - Interface("rects", rects).Msgf("FindTextsByOCR success") - for _, rect := range rects { - x, y, width, height := dExt.MappingToRectInUIKit(rect) - points = append(points, []float64{x, y, width, height}) - } - - return -} diff --git a/hrp/pkg/uixt/opencv_off.go b/hrp/pkg/uixt/opencv_off.go deleted file mode 100644 index 445c4350..00000000 --- a/hrp/pkg/uixt/opencv_off.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build !opencv - -package uixt - -import ( - "image" - - "github.com/rs/zerolog/log" -) - -func (dExt *DriverExt) extendCV(options ...CVOption) (err error) { - return nil -} - -func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { - log.Fatal().Msg("opencv is not supported") - return -} - -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { - log.Fatal().Msg("opencv is not supported") - return -} diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 5e5cdb70..61b336d5 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -5,6 +5,10 @@ import ( ) func (dExt *DriverExt) TapAbsXY(x, y float64, options ...DataOption) error { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } // tap on absolute coordinate [x, y] return dExt.Driver.TapFloat(x, y, options...) } @@ -22,6 +26,10 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error { } func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...) if err != nil { return PointF{}, err @@ -35,6 +43,10 @@ func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point P } func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } ps, err := dExt.FindTextsByOCR(ocrText, options...) if err != nil { return nil, err @@ -52,6 +64,10 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin } func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...) if err != nil { return PointF{}, err @@ -116,6 +132,11 @@ func (dExt *DriverExt) DoubleTapXY(x, y float64) error { return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) } + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } + x = x * float64(dExt.windowSize.Width) y = y * float64(dExt.windowSize.Height) return dExt.Driver.DoubleTapFloat(x, y) @@ -126,6 +147,18 @@ func (dExt *DriverExt) DoubleTap(param string) (err error) { } func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) { + + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } + + // click on element, find by name attribute + ele, err := dExt.FindUIElement(param) + if err == nil { + return ele.DoubleTap() + } + var x, y, width, height float64 if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { return err @@ -133,3 +166,29 @@ func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) ( return dExt.Driver.DoubleTapFloat(x+width*xOffset, y+height*yOffset) } + +// TapWithNumber sends one or more taps +func (dExt *DriverExt) TapWithNumber(param string, numberOfTaps int) (err error) { + return dExt.TapWithNumberOffset(param, numberOfTaps, 0.5, 0.5) +} + +func (dExt *DriverExt) TapWithNumberOffset(param string, numberOfTaps int, xOffset, yOffset float64) (err error) { + // close popup if necessary + if dExt.ClosePopup { + dExt.ClosePopupHandler() + } + + if numberOfTaps <= 0 { + numberOfTaps = 1 + } + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { + return err + } + + x = x + width*xOffset + y = y + height*yOffset + + touchActions := NewTouchActions().Tap(NewTouchActionTap().WithXYFloat(x, y).WithCount(numberOfTaps)) + return dExt.PerformTouchActions(touchActions) +} diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/template_matching.go similarity index 71% rename from hrp/pkg/uixt/opencv.go rename to hrp/pkg/uixt/template_matching.go index 98f7286b..e7c5a417 100644 --- a/hrp/pkg/uixt/opencv.go +++ b/hrp/pkg/uixt/template_matching.go @@ -7,20 +7,16 @@ import ( "fmt" "image" "image/color" - "io/ioutil" "log" "math" - "os" "github.com/pkg/errors" "gocv.io/x/gocv" - - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" ) const ( // TmCcoeffNormed maps to TM_CCOEFF_NORMED - TmCcoeffNormed TemplateMatchMode = iota + TmCcoeffNormed MatchMode = iota // TmSqdiff maps to TM_SQDIFF TmSqdiff // TmSqdiffNormed maps to TM_SQDIFF_NORMED @@ -44,109 +40,6 @@ const ( DmNotMatch ) -// extendCV 获得扩展后的 Driver， -// 并指定匹配阀值， -// 获取当前设备的 Scale， -// 默认匹配模式为 TmCcoeffNormed， -// 默认关闭 OpenCV 匹配值计算后的输出 -func (dExt *DriverExt) extendCV(options ...CVOption) (err error) { - for _, option := range options { - option(&dExt.CVArgs) - } - - if dExt.threshold == 0 { - dExt.threshold = 0.95 // default threshold - } - if dExt.matchMode == 0 { - dExt.matchMode = TmCcoeffNormed // default match mode - } - Debug(DebugMode(DmOff)) - return -} - -func (dExt *DriverExt) Debug(dm DebugMode) { - Debug(DebugMode(dm)) -} - -func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.Driver = dExt.Driver - newExt.scale = dExt.scale - newExt.matchMode = dExt.matchMode - newExt.threshold = threshold - return -} - -func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.Driver = dExt.Driver - newExt.scale = dExt.scale - newExt.matchMode = matchMode - newExt.threshold = dExt.threshold - return -} - -// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { -// pathSource := filepath.Join(sExt.pathname, GenFilename()) -// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { -// return image.Rectangle{}, err -// } -// -// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), TemplateMatchMode(sExt.MatchMode)); err != nil { -// return image.Rectangle{}, err -// } -// return -// } - -func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(search); err != nil { - return nil, err - } - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil { - return nil, err - } - - if rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil { - return nil, err - } - return -} - -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(imagePath); err != nil { - return 0, 0, 0, 0, err - } - if bufSource, err = dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_cv")); err != nil { - return 0, 0, 0, 0, err - } - - var rect image.Rectangle - if rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil { - return 0, 0, 0, 0, err - } - - // if rect, err = dExt.findImgRect(search); err != nil { - // return 0, 0, 0, 0, err - // } - x, y, width, height = dExt.MappingToRectInUIKit(rect) - return -} - -func getBufFromDisk(name string) (*bytes.Buffer, error) { - var f *os.File - var err error - if f, err = os.Open(name); err != nil { - return nil, err - } - var all []byte - if all, err = ioutil.ReadAll(f); err != nil { - return nil, err - } - return bytes.NewBuffer(all), nil -} - var debug = DmOff const dmOutputMsg = `[DEBUG] The current value is '%.4f', the expected value is '%.4f'` @@ -159,9 +52,13 @@ const DefaultMatchMode = TmCcoeffNormed var fillColor = color.RGBA{R: 255, G: 255, B: 255, A: 0} -func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) { +func init() { + Debug(DebugMode(DmOff)) +} + +func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (loc image.Point, err error) { if len(matchMode) == 0 { - matchMode = []TemplateMatchMode{DefaultMatchMode} + matchMode = []MatchMode{DefaultMatchMode} } var matImage, matTpl gocv.Mat if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { @@ -174,9 +71,9 @@ func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, m return getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) } -func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) { +func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (loc image.Point, err error) { if len(matchMode) == 0 { - matchMode = []TemplateMatchMode{DefaultMatchMode} + matchMode = []MatchMode{DefaultMatchMode} } var matImage, matTpl gocv.Mat if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { @@ -190,9 +87,9 @@ func FindImageLocationFromDisk(source, search string, threshold float32, matchMo return getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) } -func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) { +func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (locs []image.Point, err error) { if len(matchMode) == 0 { - matchMode = []TemplateMatchMode{DefaultMatchMode} + matchMode = []MatchMode{DefaultMatchMode} } var matImage, matTpl gocv.Mat if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { @@ -224,9 +121,9 @@ func FindAllImageLocationsFromDisk(source, search string, threshold float32, mat return locs, nil } -func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) { +func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (locs []image.Point, err error) { if len(matchMode) == 0 { - matchMode = []TemplateMatchMode{DefaultMatchMode} + matchMode = []MatchMode{DefaultMatchMode} } var matImage, matTpl gocv.Mat if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { @@ -270,7 +167,7 @@ func getPts(loc image.Point, width, height int) [][]image.Point { } } -func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) { +func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) { var matTpl gocv.Mat if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { return image.Rectangle{}, err @@ -287,7 +184,7 @@ func FindImageRectFromDisk(source, search string, threshold float32, matchMode . return } -func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) { +func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) { var matTpl gocv.Mat if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { return nil, err @@ -309,7 +206,7 @@ func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMo return } -func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) { +func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) { var matTpl gocv.Mat if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { return image.Rectangle{}, err @@ -326,7 +223,7 @@ func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, match return } -func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) { +func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) { var matTpl gocv.Mat if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { return nil, err @@ -401,7 +298,7 @@ func getMatsFromRaw(bufImage, bufTpl *bytes.Buffer, flags gocv.IMReadFlag) (matI } // getMatchingLocation 获取匹配的图片位置 -func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode TemplateMatchMode) (loc image.Point, err error) { +func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode MatchMode) (loc image.Point, err error) { if threshold > 1 { threshold = 1.0 } @@ -431,7 +328,7 @@ func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, } // getMatchingResult 匹配图片并返回匹配值和位置 -func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) { +func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode MatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) { matResult, tmpMask := gocv.NewMat(), gocv.NewMat() defer func() { _ = matResult.Close() @@ -443,7 +340,7 @@ func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMat } // getValLoc 根据不同的匹配模式返回匹配值和位置 -func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode TemplateMatchMode) (val float32, loc image.Point) { +func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode MatchMode) (val float32, loc image.Point) { val, loc = maxVal, maxLoc switch matchMode { diff --git a/hrp/pkg/uixt/template_matching_off.go b/hrp/pkg/uixt/template_matching_off.go new file mode 100644 index 00000000..2d12f035 --- /dev/null +++ b/hrp/pkg/uixt/template_matching_off.go @@ -0,0 +1,20 @@ +//go:build !opencv + +package uixt + +import ( + "bytes" + "image" + + "github.com/rs/zerolog/log" +) + +func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rects []image.Rectangle, err error) { + log.Fatal().Msg("opencv is not supported") + return +} + +func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...MatchMode) (rect image.Rectangle, err error) { + log.Fatal().Msg("opencv is not supported") + return +} diff --git a/hrp/pkg/uixt/opencv_test.go b/hrp/pkg/uixt/template_matching_test.go similarity index 100% rename from hrp/pkg/uixt/opencv_test.go rename to hrp/pkg/uixt/template_matching_test.go diff --git a/hrp/step_mobile_ui.go b/hrp/step_mobile_ui.go index ff8b3a2c..7fa7595b 100644 --- a/hrp/step_mobile_ui.go +++ b/hrp/step_mobile_ui.go @@ -11,6 +11,29 @@ import ( "github.com/httprunner/httprunner/v4/hrp/pkg/uixt" ) +// ios setting options +var ( + WithUDID = uixt.WithUDID + WithWDAPort = uixt.WithWDAPort + WithWDAMjpegPort = uixt.WithWDAMjpegPort + WithLogOn = uixt.WithLogOn + WithResetHomeOnStartup = uixt.WithResetHomeOnStartup + WithSnapshotMaxDepth = uixt.WithSnapshotMaxDepth + WithAcceptAlertButtonSelector = uixt.WithAcceptAlertButtonSelector + WithDismissAlertButtonSelector = uixt.WithDismissAlertButtonSelector + WithClosePopup = uixt.WithClosePopup + WithPerfOptions = uixt.WithPerfOptions +) + +// android setting options +var ( + WithSerialNumber = uixt.WithSerialNumber + WithAdbIP = uixt.WithAdbIP + WithAdbPort = uixt.WithAdbPort + WithAdbLogOn = uixt.WithAdbLogOn + WithAdbClosePopup = uixt.WithAdbClosePopup +) + type MobileStep struct { Serial string `json:"serial,omitempty" yaml:"serial,omitempty"` uixt.MobileAction `yaml:",inline"`