From 598ae509deca1ca8dac06d211783fb7415e1e5ef Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 28 Aug 2022 09:45:49 +0800 Subject: [PATCH] feat: move uixt to hrp internal --- go.mod | 3 +- go.sum | 6 +- hrp/internal/uixt/README.md | 41 ++++ hrp/internal/uixt/drag.go | 29 +++ hrp/internal/uixt/drag_test.go | 23 ++ hrp/internal/uixt/ext.go | 335 ++++++++++++++++++++++++++++++ hrp/internal/uixt/ext_ocr.go | 137 ++++++++++++ hrp/internal/uixt/ext_ocr_test.go | 21 ++ hrp/internal/uixt/gesture.go | 44 ++++ hrp/internal/uixt/gesture_test.go | 28 +++ hrp/internal/uixt/swipe.go | 135 ++++++++++++ hrp/internal/uixt/swipe_test.go | 33 +++ hrp/internal/uixt/tap.go | 88 ++++++++ hrp/internal/uixt/tap_test.go | 48 +++++ hrp/internal/uixt/touch.go | 33 +++ hrp/internal/uixt/touch_test.go | 55 +++++ hrp/step_ios_ui.go | 20 +- 17 files changed, 1057 insertions(+), 22 deletions(-) create mode 100644 hrp/internal/uixt/README.md create mode 100644 hrp/internal/uixt/drag.go create mode 100644 hrp/internal/uixt/drag_test.go create mode 100644 hrp/internal/uixt/ext.go create mode 100644 hrp/internal/uixt/ext_ocr.go create mode 100644 hrp/internal/uixt/ext_ocr_test.go create mode 100644 hrp/internal/uixt/gesture.go create mode 100644 hrp/internal/uixt/gesture_test.go create mode 100644 hrp/internal/uixt/swipe.go create mode 100644 hrp/internal/uixt/swipe_test.go create mode 100644 hrp/internal/uixt/tap.go create mode 100644 hrp/internal/uixt/tap_test.go create mode 100644 hrp/internal/uixt/touch.go create mode 100644 hrp/internal/uixt/touch_test.go diff --git a/go.mod b/go.mod index 46cbfb1a..592963da 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.16 require ( github.com/andybalholm/brotli v1.0.4 - github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009 github.com/denisbrodbeck/machineid v1.0.1 github.com/electricbubble/gwda v0.4.0 + github.com/electricbubble/opencv-helper v0.0.3 github.com/fatih/color v1.13.0 github.com/getsentry/sentry-go v0.13.0 github.com/go-errors/errors v1.0.1 @@ -31,6 +31,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + gocv.io/x/gocv v0.31.0 // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 google.golang.org/grpc v1.45.0 diff --git a/go.sum b/go.sum index 31c6c4f1..0d59b3cb 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/debugtalk/gwda v0.0.0-20220824022606-02ad6ca51de7 h1:DjPOXlkeCsxtFzieys2RjYEn6OCoAPQNiLmG2eeSVgw= github.com/debugtalk/gwda v0.0.0-20220824022606-02ad6ca51de7/go.mod h1:kyzKpP1/iKJ2i4AxmT8sEmSvB8Pz5NcDVwc/m/Jsg6k= -github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009 h1:JYD/5UFNWfaDOY4GIGboszuW7yKHXgiepo/balYm684= -github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009/go.mod h1:R10UCNr8u2xpS377k0YeutGShr0Nq5S6eEALQ6WGyu8= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -272,6 +270,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/httprunner/funplugin v0.5.0 h1:Laoe8URu71qeyST9wvRtGSkDWc8Y3T1IrnvFSTHmO84= github.com/httprunner/funplugin v0.5.0/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= +github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= @@ -519,8 +518,9 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -gocv.io/x/gocv v0.27.0 h1:3X8I74ULsWHd4m7DQRv2Nqx5VkKscfUFnKgLNodiboI= gocv.io/x/gocv v0.27.0/go.mod h1:n4LnYjykU6y9gn48yZf4eLCdtuSb77XxSkW6g0wGf/A= +gocv.io/x/gocv v0.31.0 h1:BHDtK8v+YPvoSPQTTiZB2fM/7BLg6511JqkruY2z6LQ= +gocv.io/x/gocv v0.31.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/hrp/internal/uixt/README.md b/hrp/internal/uixt/README.md new file mode 100644 index 00000000..e9191b29 --- /dev/null +++ b/hrp/internal/uixt/README.md @@ -0,0 +1,41 @@ +# uixt + +From v4.3.0，HttpRunner will support mobile UI automation testing: + +- iOS: based on [appium/WebDriverAgent], with client library [electricbubble/gwda] in golang +- Android: based on UiAutomation + +Some UI recognition algorithms are also introduced for both iOS and Android: + +- OpenCV: based on [OpenCV 4], with golang bindings [hybridgroup/gocv] and helper utils [electricbubble/gwda-ext-opencv] +- OCR: based on OCR API service from [volcengine], other API service may be extended + +## Dependencies + +### OpenCV + +[OpenCV 4] should be pre-installed. + +You can install OpenCV 4.6.0 using Homebrew on macOS. + +```bash +$ brew install opencv +``` + +You can get more installation introduction on [hybridgroup/gocv]. + +### OCR + +OCR API is a paid service, you need to pre-purchase and configure the account key. + +## Thanks + +This uixt module is initially forked from [electricbubble/gwda-ext-opencv] and made a lot of changes. + + +[electricbubble/gwda-ext-opencv]: https://github.com/electricbubble/gwda-ext-opencv +[appium/WebDriverAgent]: https://github.com/appium/WebDriverAgent +[electricbubble/gwda]: https://github.com/electricbubble/gwda +[OpenCV 4]: https://opencv.org/ +[hybridgroup/gocv]: https://github.com/hybridgroup/gocv +[volcengine]: https://www.volcengine.com/product/text-recognition diff --git a/hrp/internal/uixt/drag.go b/hrp/internal/uixt/drag.go new file mode 100644 index 00000000..b4c5ea56 --- /dev/null +++ b/hrp/internal/uixt/drag.go @@ -0,0 +1,29 @@ +package uixt + +func (dExt *DriverExt) Drag(pathname string, toX, toY int, pressForDuration ...float64) (err error) { + return dExt.DragFloat(pathname, float64(toX), float64(toY), pressForDuration...) +} + +func (dExt *DriverExt) DragFloat(pathname string, toX, toY float64, pressForDuration ...float64) (err error) { + return dExt.DragOffsetFloat(pathname, toX, toY, 0.5, 0.5, pressForDuration...) +} + +func (dExt *DriverExt) DragOffset(pathname string, toX, toY int, xOffset, yOffset float64, pressForDuration ...float64) (err error) { + return dExt.DragOffsetFloat(pathname, float64(toX), float64(toY), xOffset, yOffset, pressForDuration...) +} + +func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffset float64, pressForDuration ...float64) (err error) { + if len(pressForDuration) == 0 { + pressForDuration = []float64{1.0} + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := y + height*yOffset + + return dExt.WebDriver.DragFloat(fromX, fromY, toX, toY, pressForDuration[0]) +} diff --git a/hrp/internal/uixt/drag_test.go b/hrp/internal/uixt/drag_test.go new file mode 100644 index 00000000..018d4fd7 --- /dev/null +++ b/hrp/internal/uixt/drag_test.go @@ -0,0 +1,23 @@ +package uixt + +import ( + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExt_Drag(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_map.png" + + // err = driverExt.Drag(pathSearch, 300, 500, 2) + // checkErr(t, err) + + err = driverExt.DragOffset(pathSearch, 300, 500, 2.1, 0.5, 2) + checkErr(t, err) +} diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go new file mode 100644 index 00000000..4b24ccb8 --- /dev/null +++ b/hrp/internal/uixt/ext.go @@ -0,0 +1,335 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "image/jpeg" + "image/png" + "io/ioutil" + "mime" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/electricbubble/gwda" + cvHelper "github.com/electricbubble/opencv-helper" + "github.com/pkg/errors" +) + +// TemplateMatchMode is the type of the template matching operation. +type TemplateMatchMode int + +const ( + // TmSqdiff maps to TM_SQDIFF + TmSqdiff TemplateMatchMode = iota + // TmSqdiffNormed maps to TM_SQDIFF_NORMED + TmSqdiffNormed + // TmCcorr maps to TM_CCORR + TmCcorr + // TmCcorrNormed maps to TM_CCORR_NORMED + TmCcorrNormed + // TmCcoeff maps to TM_CCOEFF + TmCcoeff + // TmCcoeffNormed maps to TM_CCOEFF_NORMED + TmCcoeffNormed +) + +type DebugMode int + +const ( + // DmOff no output + DmOff DebugMode = iota + // DmEachMatch output matched and mismatched values + DmEachMatch + // DmNotMatch output only values that do not match + DmNotMatch +) + +type DriverExt struct { + gwda.WebDriver + windowSize gwda.Size + scale float64 + MatchMode TemplateMatchMode + Threshold float64 + frame *bytes.Buffer + doneMjpegStream chan bool +} + +// Extend 获得扩展后的 Driver， +// 并指定匹配阀值， +// 获取当前设备的 Scale， +// 默认匹配模式为 TmCcoeffNormed， +// 默认关闭 OpenCV 匹配值计算后的输出 +func Extend(driver gwda.WebDriver, threshold float64, matchMode ...TemplateMatchMode) (dExt *DriverExt, err error) { + dExt = &DriverExt{WebDriver: driver} + dExt.doneMjpegStream = make(chan bool, 1) + + if dExt.scale, err = dExt.Scale(); err != nil { + return &DriverExt{}, err + } + + // get device window size + dExt.windowSize, err = dExt.WebDriver.WindowSize() + if err != nil { + return nil, errors.Wrap(err, "failed to get windows size") + } + + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{TmCcoeffNormed} + } + dExt.MatchMode = matchMode[0] + cvHelper.Debug(cvHelper.DebugMode(DmOff)) + dExt.Threshold = threshold + return dExt, nil +} + +func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.WebDriver = dExt.WebDriver + newExt.scale = dExt.scale + newExt.MatchMode = dExt.MatchMode + newExt.Threshold = threshold + return +} + +func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.WebDriver = dExt.WebDriver + newExt.scale = dExt.scale + newExt.MatchMode = matchMode + newExt.Threshold = dExt.Threshold + return +} + +func (dExt *DriverExt) Debug(dm DebugMode) { + cvHelper.Debug(cvHelper.DebugMode(dm)) +} + +func (dExt *DriverExt) ConnectMjpegStream(httpClient *http.Client) (err error) { + if httpClient == nil { + return errors.New(`'httpClient' can't be nil`) + } + + var req *http.Request + if req, err = http.NewRequest(http.MethodGet, "http://*", nil); err != nil { + return err + } + + var resp *http.Response + if resp, err = httpClient.Do(req); err != nil { + return err + } + // defer func() { _ = resp.Body.Close() }() + + var boundary string + if _, param, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err != nil { + return err + } else { + boundary = strings.Trim(param["boundary"], "-") + } + + mjpegReader := multipart.NewReader(resp.Body, boundary) + + go func() { + for { + select { + case <-dExt.doneMjpegStream: + _ = resp.Body.Close() + return + default: + var part *multipart.Part + if part, err = mjpegReader.NextPart(); err != nil { + dExt.frame = nil + continue + } + + raw := new(bytes.Buffer) + if _, err = raw.ReadFrom(part); err != nil { + dExt.frame = nil + continue + } + dExt.frame = raw + } + } + }() + + return +} + +func (dExt *DriverExt) CloseMjpegStream() { + dExt.doneMjpegStream <- true +} + +func (dExt *DriverExt) takeScreenShot() (raw *bytes.Buffer, err error) { + // 优先使用 MJPEG 流进行截图，性能最优 + // 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口 + if dExt.frame != nil { + return dExt.frame, nil + } + if raw, err = dExt.WebDriver.Screenshot(); err != nil { + return nil, err + } + return +} + +// saveScreenShot saves image file to $CWD/screenshots/ folder +func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) { + img, format, err := image.Decode(raw) + if err != nil { + return "", errors.Wrap(err, "decode screenshot image failed") + } + + dir, _ := os.Getwd() + screenshotsDir := filepath.Join(dir, "screenshots") + if err = os.MkdirAll(screenshotsDir, os.ModePerm); err != nil { + return "", errors.Wrap(err, "create screenshots directory failed") + } + screenshotPath := filepath.Join(screenshotsDir, + fmt.Sprintf("%s.%s", fileName, format)) + + file, err := os.Create(screenshotPath) + if err != nil { + return "", errors.Wrap(err, "create screenshot image file failed") + } + defer func() { + _ = file.Close() + }() + + switch format { + case "png": + err = png.Encode(file, img) + case "jpeg": + err = jpeg.Encode(file, img, nil) + default: + return "", fmt.Errorf("unsupported image format: %s", format) + } + if err != nil { + return "", errors.Wrap(err, "encode screenshot image failed") + } + + return screenshotPath, nil +} + +// ScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder +func (dExt *DriverExt) ScreenShot(fileName string) (string, error) { + raw, err := dExt.takeScreenShot() + if err != nil { + return "", errors.Wrap(err, "screenshot by WDA failed") + } + + return dExt.saveScreenShot(raw, fileName) +} + +// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { +// pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename()) +// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { +// return image.Rectangle{}, err +// } +// +// if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil { +// return image.Rectangle{}, err +// } +// return +// } + +func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(search); err != nil { + return nil, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return nil, err + } + + if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil { + return nil, err + } + return +} + +func getBufFromDisk(name string) (*bytes.Buffer, error) { + var f *os.File + var err error + if f, err = os.Open(name); err != nil { + return nil, err + } + var all []byte + if all, err = ioutil.ReadAll(f); err != nil { + return nil, err + } + return bytes.NewBuffer(all), nil +} + +// isPathExists returns true if path exists, whether path is file or dir +func isPathExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} + +func (dExt *DriverExt) FindUIElement(param string) (ele gwda.WebElement, err error) { + var selector gwda.BySelector + if strings.HasPrefix(param, "/") { + // xpath + selector = gwda.BySelector{ + XPath: param, + } + } else { + // name + selector = gwda.BySelector{ + LinkText: gwda.NewElementAttribute().WithName(param), + } + } + + return dExt.WebDriver.FindElement(selector) +} + +func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) { + // click on text, using OCR + if !isPathExists(search) { + return dExt.FindTextByOCR(search) + } + // click on image, using opencv + return dExt.FindImageRectInUIKit(search) +} + +func (dExt *DriverExt) FindImageRectInUIKit(search string) (x, y, width, height float64, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(search); err != nil { + return 0, 0, 0, 0, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return 0, 0, 0, 0, err + } + + var rect image.Rectangle + if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil { + return 0, 0, 0, 0, err + } + + // if rect, err = dExt.findImgRect(search); err != nil { + // return 0, 0, 0, 0, err + // } + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + +func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { + x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale + width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale + return +} + +func (dExt *DriverExt) PerformTouchActions(touchActions *gwda.TouchActions) error { + return dExt.PerformAppiumTouchActions(touchActions) +} + +func (dExt *DriverExt) PerformActions(actions *gwda.W3CActions) error { + return dExt.PerformW3CActions(actions) +} + +// IsExist diff --git a/hrp/internal/uixt/ext_ocr.go b/hrp/internal/uixt/ext_ocr.go new file mode 100644 index 00000000..78581b3e --- /dev/null +++ b/hrp/internal/uixt/ext_ocr.go @@ -0,0 +1,137 @@ +package uixt + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "image" + "io/ioutil" + "mime/multipart" + "net/http" + "time" +) + +var client = &http.Client{ + Timeout: time.Second * 10, +} + +type Point struct { + X float32 `json:"x"` + Y float32 `json:"y"` +} + +type OCRResult struct { + Text string `json:"text"` + Points []Point `json:"points"` +} + +type ResponseOCR struct { + Code int `json:"code"` + Message string `json:"message"` + OCRResult []OCRResult `json:"ocrResult"` +} + +type veDEMOCRService struct{} + +func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("withDet", "true") + // bodyWriter.WriteField("timestampOnly", "true") + + formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") + if err != nil { + return nil, fmt.Errorf("create form file error: %v", err) + } + _, err = formWriter.Write(imageBuf) + if err != nil { + return nil, fmt.Errorf("write form error: %v", err) + } + + err = bodyWriter.Close() + if err != nil { + return nil, fmt.Errorf("close body writer error: %v", err) + } + + url, _ := base64.StdEncoding.DecodeString("aHR0cHM6Ly9odWJibGUuYnl0ZWRhbmNlLm5ldC92aWRlby9hcGkvdjEvYWxnb3JpdGhtL29jcg==") + req, err := http.NewRequest("POST", string(url), bodyBuf) + if err != nil { + return nil, fmt.Errorf("construct request error: %v", err) + } + + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("http reqeust OCR server error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected response status code: %d", resp.StatusCode) + } + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body error: %v", err) + } + + var ocrResult ResponseOCR + err = json.Unmarshal(results, &ocrResult) + if err != nil { + return nil, fmt.Errorf("json unmarshal response body error: %v", err) + } + + return ocrResult.OCRResult, nil +} + +func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) { + ocrResults, err := s.getOCRResult(imageBuf) + if err != nil { + return + } + + for _, ocrResult := range ocrResults { + if ocrResult.Text != text { + continue + } + + // only find the first matched one + rect = image.Rectangle{ + // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(ocrResult.Points[0].X), + Y: int(ocrResult.Points[0].Y), + }, + Max: image.Point{ + X: int(ocrResult.Points[2].X), + Y: int(ocrResult.Points[2].Y), + }, + } + return + } + + return image.Rectangle{}, fmt.Errorf("text %s not found", text) +} + +type OCRService interface { + FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) +} + +func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("screenshot error: %v", err) + return + } + + service := &veDEMOCRService{} + rect, err := service.FindText(search, bufSource.Bytes()) + if err != nil { + err = fmt.Errorf("find text failed: %v", err) + return + } + + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} diff --git a/hrp/internal/uixt/ext_ocr_test.go b/hrp/internal/uixt/ext_ocr_test.go new file mode 100644 index 00000000..53e1dee3 --- /dev/null +++ b/hrp/internal/uixt/ext_ocr_test.go @@ -0,0 +1,21 @@ +package uixt + +import ( + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExtOCR(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + x, y, width, height, err := driverExt.FindTextByOCR("抖音") + checkErr(t, err) + + t.Logf("x: %v, y: %v, width: %v, height: %v", x, y, width, height) + driver.TapFloat(x, y-20) +} diff --git a/hrp/internal/uixt/gesture.go b/hrp/internal/uixt/gesture.go new file mode 100644 index 00000000..75462167 --- /dev/null +++ b/hrp/internal/uixt/gesture.go @@ -0,0 +1,44 @@ +package uixt + +import ( + "image" + "sort" + + "github.com/electricbubble/gwda" +) + +func (dExt *DriverExt) GesturePassword(pathname string, password ...int) (err error) { + var rects []image.Rectangle + if rects, err = dExt.FindAllImageRect(pathname); err != nil { + return err + } + + sort.Slice(rects, func(i, j int) bool { + if rects[i].Min.Y < rects[j].Min.Y { + return true + } else if rects[i].Min.Y == rects[j].Min.Y { + if rects[i].Min.X < rects[j].Min.X { + return true + } + } + return false + }) + + touchActions := gwda.NewTouchActions(len(password)*2 + 1) + for i := range password { + x, y, width, height := dExt.MappingToRectInUIKit(rects[password[i]]) + x = x + width*0.5 + y = y + height*0.5 + + if i == 0 { + touchActions.Press(gwda.NewTouchActionPress().WithXYFloat(x, y)). + Wait(0.2) + } else { + touchActions.MoveTo(gwda.NewTouchActionMoveTo().WithXYFloat(x, y)). + Wait(0.2) + } + } + touchActions.Release() + + return dExt.PerformTouchActions(touchActions) +} diff --git a/hrp/internal/uixt/gesture_test.go b/hrp/internal/uixt/gesture_test.go new file mode 100644 index 00000000..8308d132 --- /dev/null +++ b/hrp/internal/uixt/gesture_test.go @@ -0,0 +1,28 @@ +package uixt + +import ( + "strconv" + "strings" + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExt_GesturePassword(t *testing.T) { + split := strings.Split("6304258", "") + password := make([]int, len(split)) + for i := range split { + password[i], _ = strconv.Atoi(split[i]) + } + + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + err = driverExt.GesturePassword(pathSearch, password...) + checkErr(t, err) +} diff --git a/hrp/internal/uixt/swipe.go b/hrp/internal/uixt/swipe.go new file mode 100644 index 00000000..44b9d36c --- /dev/null +++ b/hrp/internal/uixt/swipe.go @@ -0,0 +1,135 @@ +package uixt + +func (dExt *DriverExt) SwipeTo(direction string) (err error) { + width := dExt.windowSize.Width + height := dExt.windowSize.Height + + var fromX, fromY, toX, toY int + switch direction { + case "up": + fromX, fromY, toX, toY = width/2, height*3/4, width/2, height*1/4 + case "down": + fromX, fromY, toX, toY = width/2, height*1/4, width/2, height*3/4 + case "left": + fromX, fromY, toX, toY = width*3/4, height/2, width*1/4, height/2 + case "right": + fromX, fromY, toX, toY = width*1/4, height/2, width*3/4, height/2 + } + return dExt.WebDriver.Swipe(fromX, fromY, toX, toY) +} + +func (dExt *DriverExt) Swipe(pathname string, toX, toY int) (err error) { + return dExt.SwipeFloat(pathname, float64(toX), float64(toY)) +} + +func (dExt *DriverExt) SwipeFloat(pathname string, toX, toY float64) (err error) { + return dExt.SwipeOffsetFloat(pathname, toX, toY, 0.5, 0.5) +} + +func (dExt *DriverExt) SwipeOffset(pathname string, toX, toY int, xOffset, yOffset float64) (err error) { + return dExt.SwipeOffsetFloat(pathname, float64(toX), float64(toY), xOffset, yOffset) +} + +func (dExt *DriverExt) SwipeOffsetFloat(pathname string, toX, toY, xOffset, yOffset float64) (err error) { + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := y + height*yOffset + + return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY) +} + +func (dExt *DriverExt) SwipeUp(pathname string, distance ...float64) (err error) { + return dExt.SwipeUpOffset(pathname, 0.5, 0.9, distance...) +} + +func (dExt *DriverExt) SwipeUpOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) { + if len(distance) == 0 { + distance = []float64{1.0} + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := (y + height) - height*(1.0-yOffset) + + toX := fromX + toY := fromY - height*distance[0] + + return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY) +} + +func (dExt *DriverExt) SwipeDown(pathname string, distance ...float64) (err error) { + return dExt.SwipeDownOffset(pathname, 0.5, 0.1, distance...) +} + +func (dExt *DriverExt) SwipeDownOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) { + if len(distance) == 0 { + distance = []float64{1.0} + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := y + height*yOffset + + toX := fromX + toY := fromY + height*distance[0] + + return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY) +} + +func (dExt *DriverExt) SwipeLeft(pathname string, distance ...float64) (err error) { + return dExt.SwipeLeftOffset(pathname, 0.9, 0.5, distance...) +} + +func (dExt *DriverExt) SwipeLeftOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) { + if len(distance) == 0 { + distance = []float64{1.0} + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := y + height*yOffset + + toX := fromX - width*distance[0] + toY := fromY + + return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY) +} + +func (dExt *DriverExt) SwipeRight(pathname string, distance ...float64) (err error) { + return dExt.SwipeRightOffset(pathname, 0.1, 0.5, distance...) +} + +func (dExt *DriverExt) SwipeRightOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) { + if len(distance) == 0 { + distance = []float64{1.0} + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + fromX := x + width*xOffset + fromY := y + height*yOffset + + toX := fromX + width*distance[0] + toY := fromY + + return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY) +} diff --git a/hrp/internal/uixt/swipe_test.go b/hrp/internal/uixt/swipe_test.go new file mode 100644 index 00000000..f8e0650c --- /dev/null +++ b/hrp/internal/uixt/swipe_test.go @@ -0,0 +1,33 @@ +package uixt + +import ( + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExt_Swipe(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/flag7.png" + + // gwda.SetDebug(true) + + err = driverExt.Swipe(pathSearch, 300, 500) + checkErr(t, err) + + err = driverExt.SwipeFloat(pathSearch, 300.9, 500) + checkErr(t, err) + + err = driverExt.SwipeOffset(pathSearch, 300, 500, 0.2, 0.5) + checkErr(t, err) + + driverExt.Debug(DmNotMatch) + + err = driverExt.OnlyOnceThreshold(0.92).SwipeOffsetFloat(pathSearch, 300.9, 499.1, 0.2, 0.5) + checkErr(t, err) +} diff --git a/hrp/internal/uixt/tap.go b/hrp/internal/uixt/tap.go new file mode 100644 index 00000000..d2c11290 --- /dev/null +++ b/hrp/internal/uixt/tap.go @@ -0,0 +1,88 @@ +package uixt + +import ( + "fmt" + + "github.com/electricbubble/gwda" +) + +func (dExt *DriverExt) TapXY(x, y float64) error { + // tap on coordinate: [x, y] should be relative + if x > 1 || y > 1 { + return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) + } + + x = x * float64(dExt.windowSize.Width) + y = y * float64(dExt.windowSize.Height) + return dExt.WebDriver.TapFloat(x, y) +} + +func (dExt *DriverExt) Tap(param string) error { + return dExt.TapOffset(param, 0.5, 0.5) +} + +func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64) (err error) { + // click on element, find by name attribute + ele, err := dExt.FindUIElement(param) + if err == nil { + return ele.Click() + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { + return err + } + + return dExt.WebDriver.TapFloat(x+width*xOffset, y+height*yOffset) +} + +func (dExt *DriverExt) DoubleTapXY(x, y float64) error { + // double tap on coordinate: [x, y] should be relative + if x > 1 || y > 1 { + return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) + } + + x = x * float64(dExt.windowSize.Width) + y = y * float64(dExt.windowSize.Height) + return dExt.WebDriver.DoubleTapFloat(x, y) +} + +func (dExt *DriverExt) DoubleTap(param string) (err error) { + return dExt.DoubleTapOffset(param, 0.5, 0.5) +} + +func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) { + // click on element, find by name attribute + ele, err := dExt.FindUIElement(param) + if err == nil { + return ele.DoubleTap() + } + + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { + return err + } + + return dExt.WebDriver.DoubleTapFloat(x+width*xOffset, y+height*yOffset) +} + +// TapWithNumber sends one or more taps +func (dExt *DriverExt) TapWithNumber(param string, numberOfTaps int) (err error) { + return dExt.TapWithNumberOffset(param, numberOfTaps, 0.5, 0.5) +} + +func (dExt *DriverExt) TapWithNumberOffset(param string, numberOfTaps int, xOffset, yOffset float64) (err error) { + if numberOfTaps <= 0 { + numberOfTaps = 1 + } + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil { + return err + } + + x = x + width*xOffset + y = y + height*yOffset + + touchActions := gwda.NewTouchActions().Tap(gwda.NewTouchActionTap().WithXYFloat(x, y).WithCount(numberOfTaps)) + return dExt.PerformTouchActions(touchActions) +} diff --git a/hrp/internal/uixt/tap_test.go b/hrp/internal/uixt/tap_test.go new file mode 100644 index 00000000..f519af3b --- /dev/null +++ b/hrp/internal/uixt/tap_test.go @@ -0,0 +1,48 @@ +package uixt + +import ( + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExt_TapWithNumber(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/flag7.png" + + // gwda.SetDebug(true) + + err = driverExt.TapWithNumber(pathSearch, 3) + checkErr(t, err) + + err = driverExt.TapWithNumberOffset(pathSearch, 3, 0.5, 0.75) + checkErr(t, err) +} + +func TestDriverExt_TapXY(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + err = driverExt.TapXY(0.4, 0.5) + checkErr(t, err) +} + +func TestDriverExt_TapWithOCR(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + // 需要点击文字上方的图标 + err = driverExt.TapOffset("抖音", 0.5, -1) + checkErr(t, err) +} diff --git a/hrp/internal/uixt/touch.go b/hrp/internal/uixt/touch.go new file mode 100644 index 00000000..6c06ae81 --- /dev/null +++ b/hrp/internal/uixt/touch.go @@ -0,0 +1,33 @@ +package uixt + +func (dExt *DriverExt) ForceTouch(pathname string, pressure float64, duration ...float64) (err error) { + return dExt.ForceTouchOffset(pathname, pressure, 0.5, 0.5, duration...) +} + +func (dExt *DriverExt) ForceTouchOffset(pathname string, pressure, xOffset, yOffset float64, duration ...float64) (err error) { + if len(duration) == 0 { + duration = []float64{1.0} + } + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + return dExt.ForceTouchFloat(x+width*xOffset, y+height*yOffset, pressure, duration[0]) +} + +func (dExt *DriverExt) TouchAndHold(pathname string, duration ...float64) (err error) { + return dExt.TouchAndHoldOffset(pathname, 0.5, 0.5, duration...) +} + +func (dExt *DriverExt) TouchAndHoldOffset(pathname string, xOffset, yOffset float64, duration ...float64) (err error) { + if len(duration) == 0 { + duration = []float64{1.0} + } + var x, y, width, height float64 + if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil { + return err + } + + return dExt.TouchAndHoldFloat(x+width*xOffset, y+height*yOffset, duration[0]) +} diff --git a/hrp/internal/uixt/touch_test.go b/hrp/internal/uixt/touch_test.go new file mode 100644 index 00000000..514e2fae --- /dev/null +++ b/hrp/internal/uixt/touch_test.go @@ -0,0 +1,55 @@ +package uixt + +import ( + "testing" + + "github.com/electricbubble/gwda" +) + +func TestDriverExt_ForceTouch(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png" + + err = driverExt.ForceTouch(pathSearch, 0.5, 3) + checkErr(t, err) + + // err = driverExt.ForceTouchOffset(pathSearch, 0.5, 0.1, 0.9) + // checkErr(t, err) + + // err = driverExt.ForceTouchOffset(pathSearch, 0.2, 1.1, -1) + // checkErr(t, err) +} + +func TestDriverExt_TouchAndHold(t *testing.T) { + driver, err := gwda.NewUSBDriver(nil) + checkErr(t, err) + + driverExt, err := Extend(driver, 0.95) + checkErr(t, err) + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png" + + // err = driverExt.TouchAndHold(pathSearch) + // checkErr(t, err) + + // err = driverExt.TouchAndHold(pathSearch, 3) + // checkErr(t, err) + + err = driverExt.TouchAndHoldOffset(pathSearch, 0.8, 0.1) + checkErr(t, err) +} + +func checkErr(t *testing.T, err error, msg ...string) { + if err != nil { + if len(msg) == 0 { + t.Fatal(err) + } else { + t.Fatal(msg, err) + } + } +} diff --git a/hrp/step_ios_ui.go b/hrp/step_ios_ui.go index 6b9358b8..bf02c121 100644 --- a/hrp/step_ios_ui.go +++ b/hrp/step_ios_ui.go @@ -6,9 +6,10 @@ import ( "time" "github.com/electricbubble/gwda" - "github.com/httprunner/uixt" "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/uixt" ) const ( @@ -651,23 +652,6 @@ func (w *wdaClient) doValidation(iValidators []interface{}) (validateResults []* return } -func (w *wdaClient) findElement(param string) (ele gwda.WebElement, err error) { - var selector gwda.BySelector - if strings.HasPrefix(param, "/") { - // xpath - selector = gwda.BySelector{ - XPath: param, - } - } else { - // name - selector = gwda.BySelector{ - LinkText: gwda.NewElementAttribute().WithName(param), - } - } - - return w.DriverExt.FindElement(selector) -} - func (w *wdaClient) assertName(name string, exists bool) bool { selector := gwda.BySelector{ LinkText: gwda.NewElementAttribute().WithName(name),