feat: move uixt to hrp internal

2026-05-13 17:29:56 +08:00 · 2022-08-28 09:45:49 +08:00
parent d3c1c08030
commit 598ae509de
17 changed files with 1057 additions and 22 deletions
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,9 @@ go 1.16

 require (
 	github.com/andybalholm/brotli v1.0.4
-	github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/electricbubble/gwda v0.4.0
+	github.com/electricbubble/opencv-helper v0.0.3
 	github.com/fatih/color v1.13.0
 	github.com/getsentry/sentry-go v0.13.0
 	github.com/go-errors/errors v1.0.1
@@ -31,6 +31,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	gocv.io/x/gocv v0.31.0 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
 	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
 	google.golang.org/grpc v1.45.0
--- a/go.sum
+++ b/go.sum
@@ -96,8 +96,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/debugtalk/gwda v0.0.0-20220824022606-02ad6ca51de7 h1:DjPOXlkeCsxtFzieys2RjYEn6OCoAPQNiLmG2eeSVgw=
 github.com/debugtalk/gwda v0.0.0-20220824022606-02ad6ca51de7/go.mod h1:kyzKpP1/iKJ2i4AxmT8sEmSvB8Pz5NcDVwc/m/Jsg6k=
-github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009 h1:JYD/5UFNWfaDOY4GIGboszuW7yKHXgiepo/balYm684=
-github.com/debugtalk/gwda-ext v0.0.0-20220826161333-0588d8320009/go.mod h1:R10UCNr8u2xpS377k0YeutGShr0Nq5S6eEALQ6WGyu8=
 github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
@@ -272,6 +270,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/httprunner/funplugin v0.5.0 h1:Laoe8URu71qeyST9wvRtGSkDWc8Y3T1IrnvFSTHmO84=
 github.com/httprunner/funplugin v0.5.0/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc=
+github.com/hybridgroup/mjpeg v0.0.0-20140228234708-4680f319790e/go.mod h1:eagM805MRKrioHYuU7iKLUyFPVKqVV6um5DAvCkUtXs=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
@@ -519,8 +518,9 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-gocv.io/x/gocv v0.27.0 h1:3X8I74ULsWHd4m7DQRv2Nqx5VkKscfUFnKgLNodiboI=
 gocv.io/x/gocv v0.27.0/go.mod h1:n4LnYjykU6y9gn48yZf4eLCdtuSb77XxSkW6g0wGf/A=
+gocv.io/x/gocv v0.31.0 h1:BHDtK8v+YPvoSPQTTiZB2fM/7BLg6511JqkruY2z6LQ=
+gocv.io/x/gocv v0.31.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
--- a/hrp/internal/uixt/README.md
+++ b/hrp/internal/uixt/README.md
@@ -0,0 +1,41 @@
+# uixt
+
+From v4.3.0，HttpRunner will support mobile UI automation testing:
+
+- iOS: based on [appium/WebDriverAgent], with client library [electricbubble/gwda] in golang
+- Android: based on UiAutomation
+
+Some UI recognition algorithms are also introduced for both iOS and Android:
+
+- OpenCV: based on [OpenCV 4], with golang bindings [hybridgroup/gocv] and helper utils [electricbubble/gwda-ext-opencv]
+- OCR: based on OCR API service from [volcengine], other API service may be extended
+
+## Dependencies
+
+### OpenCV
+
+[OpenCV 4] should be pre-installed.
+
+You can install OpenCV 4.6.0 using Homebrew on macOS.
+
+```bash
+$ brew install opencv
+```
+
+You can get more installation introduction on [hybridgroup/gocv].
+
+### OCR
+
+OCR API is a paid service, you need to pre-purchase and configure the account key.
+
+## Thanks
+
+This uixt module is initially forked from [electricbubble/gwda-ext-opencv] and made a lot of changes.
+
+
+[electricbubble/gwda-ext-opencv]: https://github.com/electricbubble/gwda-ext-opencv
+[appium/WebDriverAgent]: https://github.com/appium/WebDriverAgent
+[electricbubble/gwda]: https://github.com/electricbubble/gwda
+[OpenCV 4]: https://opencv.org/
+[hybridgroup/gocv]: https://github.com/hybridgroup/gocv
+[volcengine]: https://www.volcengine.com/product/text-recognition
--- a/hrp/internal/uixt/drag.go
+++ b/hrp/internal/uixt/drag.go
@@ -0,0 +1,29 @@
+package uixt
+
+func (dExt *DriverExt) Drag(pathname string, toX, toY int, pressForDuration ...float64) (err error) {
+	return dExt.DragFloat(pathname, float64(toX), float64(toY), pressForDuration...)
+}
+
+func (dExt *DriverExt) DragFloat(pathname string, toX, toY float64, pressForDuration ...float64) (err error) {
+	return dExt.DragOffsetFloat(pathname, toX, toY, 0.5, 0.5, pressForDuration...)
+}
+
+func (dExt *DriverExt) DragOffset(pathname string, toX, toY int, xOffset, yOffset float64, pressForDuration ...float64) (err error) {
+	return dExt.DragOffsetFloat(pathname, float64(toX), float64(toY), xOffset, yOffset, pressForDuration...)
+}
+
+func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffset float64, pressForDuration ...float64) (err error) {
+	if len(pressForDuration) == 0 {
+		pressForDuration = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	return dExt.WebDriver.DragFloat(fromX, fromY, toX, toY, pressForDuration[0])
+}
--- a/hrp/internal/uixt/drag_test.go
+++ b/hrp/internal/uixt/drag_test.go
@@ -0,0 +1,23 @@
+package uixt
+
+import (
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExt_Drag(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_map.png"
+
+	// err = driverExt.Drag(pathSearch, 300, 500, 2)
+	// checkErr(t, err)
+
+	err = driverExt.DragOffset(pathSearch, 300, 500, 2.1, 0.5, 2)
+	checkErr(t, err)
+}
--- a/hrp/internal/uixt/ext.go
+++ b/hrp/internal/uixt/ext.go
@@ -0,0 +1,335 @@
+package uixt
+
+import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"image/png"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/electricbubble/gwda"
+	cvHelper "github.com/electricbubble/opencv-helper"
+	"github.com/pkg/errors"
+)
+
+// TemplateMatchMode is the type of the template matching operation.
+type TemplateMatchMode int
+
+const (
+	// TmSqdiff maps to TM_SQDIFF
+	TmSqdiff TemplateMatchMode = iota
+	// TmSqdiffNormed maps to TM_SQDIFF_NORMED
+	TmSqdiffNormed
+	// TmCcorr maps to TM_CCORR
+	TmCcorr
+	// TmCcorrNormed maps to TM_CCORR_NORMED
+	TmCcorrNormed
+	// TmCcoeff maps to TM_CCOEFF
+	TmCcoeff
+	// TmCcoeffNormed maps to TM_CCOEFF_NORMED
+	TmCcoeffNormed
+)
+
+type DebugMode int
+
+const (
+	// DmOff no output
+	DmOff DebugMode = iota
+	// DmEachMatch output matched and mismatched values
+	DmEachMatch
+	// DmNotMatch output only values that do not match
+	DmNotMatch
+)
+
+type DriverExt struct {
+	gwda.WebDriver
+	windowSize      gwda.Size
+	scale           float64
+	MatchMode       TemplateMatchMode
+	Threshold       float64
+	frame           *bytes.Buffer
+	doneMjpegStream chan bool
+}
+
+// Extend 获得扩展后的 Driver，
+// 并指定匹配阀值，
+// 获取当前设备的 Scale，
+// 默认匹配模式为 TmCcoeffNormed，
+// 默认关闭 OpenCV 匹配值计算后的输出
+func Extend(driver gwda.WebDriver, threshold float64, matchMode ...TemplateMatchMode) (dExt *DriverExt, err error) {
+	dExt = &DriverExt{WebDriver: driver}
+	dExt.doneMjpegStream = make(chan bool, 1)
+
+	if dExt.scale, err = dExt.Scale(); err != nil {
+		return &DriverExt{}, err
+	}
+
+	// get device window size
+	dExt.windowSize, err = dExt.WebDriver.WindowSize()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get windows size")
+	}
+
+	if len(matchMode) == 0 {
+		matchMode = []TemplateMatchMode{TmCcoeffNormed}
+	}
+	dExt.MatchMode = matchMode[0]
+	cvHelper.Debug(cvHelper.DebugMode(DmOff))
+	dExt.Threshold = threshold
+	return dExt, nil
+}
+
+func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
+	newExt = new(DriverExt)
+	newExt.WebDriver = dExt.WebDriver
+	newExt.scale = dExt.scale
+	newExt.MatchMode = dExt.MatchMode
+	newExt.Threshold = threshold
+	return
+}
+
+func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
+	newExt = new(DriverExt)
+	newExt.WebDriver = dExt.WebDriver
+	newExt.scale = dExt.scale
+	newExt.MatchMode = matchMode
+	newExt.Threshold = dExt.Threshold
+	return
+}
+
+func (dExt *DriverExt) Debug(dm DebugMode) {
+	cvHelper.Debug(cvHelper.DebugMode(dm))
+}
+
+func (dExt *DriverExt) ConnectMjpegStream(httpClient *http.Client) (err error) {
+	if httpClient == nil {
+		return errors.New(`'httpClient' can't be nil`)
+	}
+
+	var req *http.Request
+	if req, err = http.NewRequest(http.MethodGet, "http://*", nil); err != nil {
+		return err
+	}
+
+	var resp *http.Response
+	if resp, err = httpClient.Do(req); err != nil {
+		return err
+	}
+	// defer func() { _ = resp.Body.Close() }()
+
+	var boundary string
+	if _, param, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err != nil {
+		return err
+	} else {
+		boundary = strings.Trim(param["boundary"], "-")
+	}
+
+	mjpegReader := multipart.NewReader(resp.Body, boundary)
+
+	go func() {
+		for {
+			select {
+			case <-dExt.doneMjpegStream:
+				_ = resp.Body.Close()
+				return
+			default:
+				var part *multipart.Part
+				if part, err = mjpegReader.NextPart(); err != nil {
+					dExt.frame = nil
+					continue
+				}
+
+				raw := new(bytes.Buffer)
+				if _, err = raw.ReadFrom(part); err != nil {
+					dExt.frame = nil
+					continue
+				}
+				dExt.frame = raw
+			}
+		}
+	}()
+
+	return
+}
+
+func (dExt *DriverExt) CloseMjpegStream() {
+	dExt.doneMjpegStream <- true
+}
+
+func (dExt *DriverExt) takeScreenShot() (raw *bytes.Buffer, err error) {
+	// 优先使用 MJPEG 流进行截图，性能最优
+	// 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口
+	if dExt.frame != nil {
+		return dExt.frame, nil
+	}
+	if raw, err = dExt.WebDriver.Screenshot(); err != nil {
+		return nil, err
+	}
+	return
+}
+
+// saveScreenShot saves image file to $CWD/screenshots/ folder
+func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) {
+	img, format, err := image.Decode(raw)
+	if err != nil {
+		return "", errors.Wrap(err, "decode screenshot image failed")
+	}
+
+	dir, _ := os.Getwd()
+	screenshotsDir := filepath.Join(dir, "screenshots")
+	if err = os.MkdirAll(screenshotsDir, os.ModePerm); err != nil {
+		return "", errors.Wrap(err, "create screenshots directory failed")
+	}
+	screenshotPath := filepath.Join(screenshotsDir,
+		fmt.Sprintf("%s.%s", fileName, format))
+
+	file, err := os.Create(screenshotPath)
+	if err != nil {
+		return "", errors.Wrap(err, "create screenshot image file failed")
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+
+	switch format {
+	case "png":
+		err = png.Encode(file, img)
+	case "jpeg":
+		err = jpeg.Encode(file, img, nil)
+	default:
+		return "", fmt.Errorf("unsupported image format: %s", format)
+	}
+	if err != nil {
+		return "", errors.Wrap(err, "encode screenshot image failed")
+	}
+
+	return screenshotPath, nil
+}
+
+// ScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
+func (dExt *DriverExt) ScreenShot(fileName string) (string, error) {
+	raw, err := dExt.takeScreenShot()
+	if err != nil {
+		return "", errors.Wrap(err, "screenshot by WDA failed")
+	}
+
+	return dExt.saveScreenShot(raw, fileName)
+}
+
+// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
+// 	pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename())
+// 	if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
+// 		return image.Rectangle{}, err
+// 	}
+//
+// 	if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil {
+// 		return image.Rectangle{}, err
+// 	}
+// 	return
+// }
+
+func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
+	var bufSource, bufSearch *bytes.Buffer
+	if bufSearch, err = getBufFromDisk(search); err != nil {
+		return nil, err
+	}
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		return nil, err
+	}
+
+	if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil {
+		return nil, err
+	}
+	return
+}
+
+func getBufFromDisk(name string) (*bytes.Buffer, error) {
+	var f *os.File
+	var err error
+	if f, err = os.Open(name); err != nil {
+		return nil, err
+	}
+	var all []byte
+	if all, err = ioutil.ReadAll(f); err != nil {
+		return nil, err
+	}
+	return bytes.NewBuffer(all), nil
+}
+
+// isPathExists returns true if path exists, whether path is file or dir
+func isPathExists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (dExt *DriverExt) FindUIElement(param string) (ele gwda.WebElement, err error) {
+	var selector gwda.BySelector
+	if strings.HasPrefix(param, "/") {
+		// xpath
+		selector = gwda.BySelector{
+			XPath: param,
+		}
+	} else {
+		// name
+		selector = gwda.BySelector{
+			LinkText: gwda.NewElementAttribute().WithName(param),
+		}
+	}
+
+	return dExt.WebDriver.FindElement(selector)
+}
+
+func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) {
+	// click on text, using OCR
+	if !isPathExists(search) {
+		return dExt.FindTextByOCR(search)
+	}
+	// click on image, using opencv
+	return dExt.FindImageRectInUIKit(search)
+}
+
+func (dExt *DriverExt) FindImageRectInUIKit(search string) (x, y, width, height float64, err error) {
+	var bufSource, bufSearch *bytes.Buffer
+	if bufSearch, err = getBufFromDisk(search); err != nil {
+		return 0, 0, 0, 0, err
+	}
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	var rect image.Rectangle
+	if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	// if rect, err = dExt.findImgRect(search); err != nil {
+	// 	return 0, 0, 0, 0, err
+	// }
+	x, y, width, height = dExt.MappingToRectInUIKit(rect)
+	return
+}
+
+func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
+	x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale
+	width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale
+	return
+}
+
+func (dExt *DriverExt) PerformTouchActions(touchActions *gwda.TouchActions) error {
+	return dExt.PerformAppiumTouchActions(touchActions)
+}
+
+func (dExt *DriverExt) PerformActions(actions *gwda.W3CActions) error {
+	return dExt.PerformW3CActions(actions)
+}
+
+// IsExist
--- a/hrp/internal/uixt/ext_ocr.go
+++ b/hrp/internal/uixt/ext_ocr.go
@@ -0,0 +1,137 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"image"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"time"
+)
+
+var client = &http.Client{
+	Timeout: time.Second * 10,
+}
+
+type Point struct {
+	X float32 `json:"x"`
+	Y float32 `json:"y"`
+}
+
+type OCRResult struct {
+	Text   string  `json:"text"`
+	Points []Point `json:"points"`
+}
+
+type ResponseOCR struct {
+	Code      int         `json:"code"`
+	Message   string      `json:"message"`
+	OCRResult []OCRResult `json:"ocrResult"`
+}
+
+type veDEMOCRService struct{}
+
+func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) {
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	bodyWriter.WriteField("withDet", "true")
+	// bodyWriter.WriteField("timestampOnly", "true")
+
+	formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png")
+	if err != nil {
+		return nil, fmt.Errorf("create form file error: %v", err)
+	}
+	_, err = formWriter.Write(imageBuf)
+	if err != nil {
+		return nil, fmt.Errorf("write form error: %v", err)
+	}
+
+	err = bodyWriter.Close()
+	if err != nil {
+		return nil, fmt.Errorf("close body writer error: %v", err)
+	}
+
+	url, _ := base64.StdEncoding.DecodeString("aHR0cHM6Ly9odWJibGUuYnl0ZWRhbmNlLm5ldC92aWRlby9hcGkvdjEvYWxnb3JpdGhtL29jcg==")
+	req, err := http.NewRequest("POST", string(url), bodyBuf)
+	if err != nil {
+		return nil, fmt.Errorf("construct request error: %v", err)
+	}
+
+	req.Header.Add("Content-Type", bodyWriter.FormDataContentType())
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http reqeust OCR server error: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response status code: %d", resp.StatusCode)
+	}
+
+	results, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response body error: %v", err)
+	}
+
+	var ocrResult ResponseOCR
+	err = json.Unmarshal(results, &ocrResult)
+	if err != nil {
+		return nil, fmt.Errorf("json unmarshal response body error: %v", err)
+	}
+
+	return ocrResult.OCRResult, nil
+}
+
+func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) {
+	ocrResults, err := s.getOCRResult(imageBuf)
+	if err != nil {
+		return
+	}
+
+	for _, ocrResult := range ocrResults {
+		if ocrResult.Text != text {
+			continue
+		}
+
+		// only find the first matched one
+		rect = image.Rectangle{
+			// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
+			Min: image.Point{
+				X: int(ocrResult.Points[0].X),
+				Y: int(ocrResult.Points[0].Y),
+			},
+			Max: image.Point{
+				X: int(ocrResult.Points[2].X),
+				Y: int(ocrResult.Points[2].Y),
+			},
+		}
+		return
+	}
+
+	return image.Rectangle{}, fmt.Errorf("text %s not found", text)
+}
+
+type OCRService interface {
+	FindText(text string, imageBuf []byte) (rect image.Rectangle, err error)
+}
+
+func (dExt *DriverExt) FindTextByOCR(search string) (x, y, width, height float64, err error) {
+	var bufSource *bytes.Buffer
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		err = fmt.Errorf("screenshot error: %v", err)
+		return
+	}
+
+	service := &veDEMOCRService{}
+	rect, err := service.FindText(search, bufSource.Bytes())
+	if err != nil {
+		err = fmt.Errorf("find text failed: %v", err)
+		return
+	}
+
+	x, y, width, height = dExt.MappingToRectInUIKit(rect)
+	return
+}
--- a/hrp/internal/uixt/ext_ocr_test.go
+++ b/hrp/internal/uixt/ext_ocr_test.go
@@ -0,0 +1,21 @@
+package uixt
+
+import (
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExtOCR(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	x, y, width, height, err := driverExt.FindTextByOCR("抖音")
+	checkErr(t, err)
+
+	t.Logf("x: %v, y: %v, width: %v, height: %v", x, y, width, height)
+	driver.TapFloat(x, y-20)
+}
--- a/hrp/internal/uixt/gesture.go
+++ b/hrp/internal/uixt/gesture.go
@@ -0,0 +1,44 @@
+package uixt
+
+import (
+	"image"
+	"sort"
+
+	"github.com/electricbubble/gwda"
+)
+
+func (dExt *DriverExt) GesturePassword(pathname string, password ...int) (err error) {
+	var rects []image.Rectangle
+	if rects, err = dExt.FindAllImageRect(pathname); err != nil {
+		return err
+	}
+
+	sort.Slice(rects, func(i, j int) bool {
+		if rects[i].Min.Y < rects[j].Min.Y {
+			return true
+		} else if rects[i].Min.Y == rects[j].Min.Y {
+			if rects[i].Min.X < rects[j].Min.X {
+				return true
+			}
+		}
+		return false
+	})
+
+	touchActions := gwda.NewTouchActions(len(password)*2 + 1)
+	for i := range password {
+		x, y, width, height := dExt.MappingToRectInUIKit(rects[password[i]])
+		x = x + width*0.5
+		y = y + height*0.5
+
+		if i == 0 {
+			touchActions.Press(gwda.NewTouchActionPress().WithXYFloat(x, y)).
+				Wait(0.2)
+		} else {
+			touchActions.MoveTo(gwda.NewTouchActionMoveTo().WithXYFloat(x, y)).
+				Wait(0.2)
+		}
+	}
+	touchActions.Release()
+
+	return dExt.PerformTouchActions(touchActions)
+}
--- a/hrp/internal/uixt/gesture_test.go
+++ b/hrp/internal/uixt/gesture_test.go
@@ -0,0 +1,28 @@
+package uixt
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExt_GesturePassword(t *testing.T) {
+	split := strings.Split("6304258", "")
+	password := make([]int, len(split))
+	for i := range split {
+		password[i], _ = strconv.Atoi(split[i])
+	}
+
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
+
+	err = driverExt.GesturePassword(pathSearch, password...)
+	checkErr(t, err)
+}
--- a/hrp/internal/uixt/swipe.go
+++ b/hrp/internal/uixt/swipe.go
@@ -0,0 +1,135 @@
+package uixt
+
+func (dExt *DriverExt) SwipeTo(direction string) (err error) {
+	width := dExt.windowSize.Width
+	height := dExt.windowSize.Height
+
+	var fromX, fromY, toX, toY int
+	switch direction {
+	case "up":
+		fromX, fromY, toX, toY = width/2, height*3/4, width/2, height*1/4
+	case "down":
+		fromX, fromY, toX, toY = width/2, height*1/4, width/2, height*3/4
+	case "left":
+		fromX, fromY, toX, toY = width*3/4, height/2, width*1/4, height/2
+	case "right":
+		fromX, fromY, toX, toY = width*1/4, height/2, width*3/4, height/2
+	}
+	return dExt.WebDriver.Swipe(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) Swipe(pathname string, toX, toY int) (err error) {
+	return dExt.SwipeFloat(pathname, float64(toX), float64(toY))
+}
+
+func (dExt *DriverExt) SwipeFloat(pathname string, toX, toY float64) (err error) {
+	return dExt.SwipeOffsetFloat(pathname, toX, toY, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) SwipeOffset(pathname string, toX, toY int, xOffset, yOffset float64) (err error) {
+	return dExt.SwipeOffsetFloat(pathname, float64(toX), float64(toY), xOffset, yOffset)
+}
+
+func (dExt *DriverExt) SwipeOffsetFloat(pathname string, toX, toY, xOffset, yOffset float64) (err error) {
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) SwipeUp(pathname string, distance ...float64) (err error) {
+	return dExt.SwipeUpOffset(pathname, 0.5, 0.9, distance...)
+}
+
+func (dExt *DriverExt) SwipeUpOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) {
+	if len(distance) == 0 {
+		distance = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := (y + height) - height*(1.0-yOffset)
+
+	toX := fromX
+	toY := fromY - height*distance[0]
+
+	return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) SwipeDown(pathname string, distance ...float64) (err error) {
+	return dExt.SwipeDownOffset(pathname, 0.5, 0.1, distance...)
+}
+
+func (dExt *DriverExt) SwipeDownOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) {
+	if len(distance) == 0 {
+		distance = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	toX := fromX
+	toY := fromY + height*distance[0]
+
+	return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) SwipeLeft(pathname string, distance ...float64) (err error) {
+	return dExt.SwipeLeftOffset(pathname, 0.9, 0.5, distance...)
+}
+
+func (dExt *DriverExt) SwipeLeftOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) {
+	if len(distance) == 0 {
+		distance = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	toX := fromX - width*distance[0]
+	toY := fromY
+
+	return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) SwipeRight(pathname string, distance ...float64) (err error) {
+	return dExt.SwipeRightOffset(pathname, 0.1, 0.5, distance...)
+}
+
+func (dExt *DriverExt) SwipeRightOffset(pathname string, xOffset, yOffset float64, distance ...float64) (err error) {
+	if len(distance) == 0 {
+		distance = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	toX := fromX + width*distance[0]
+	toY := fromY
+
+	return dExt.WebDriver.SwipeFloat(fromX, fromY, toX, toY)
+}
--- a/hrp/internal/uixt/swipe_test.go
+++ b/hrp/internal/uixt/swipe_test.go
@@ -0,0 +1,33 @@
+package uixt
+
+import (
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExt_Swipe(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/flag7.png"
+
+	// gwda.SetDebug(true)
+
+	err = driverExt.Swipe(pathSearch, 300, 500)
+	checkErr(t, err)
+
+	err = driverExt.SwipeFloat(pathSearch, 300.9, 500)
+	checkErr(t, err)
+
+	err = driverExt.SwipeOffset(pathSearch, 300, 500, 0.2, 0.5)
+	checkErr(t, err)
+
+	driverExt.Debug(DmNotMatch)
+
+	err = driverExt.OnlyOnceThreshold(0.92).SwipeOffsetFloat(pathSearch, 300.9, 499.1, 0.2, 0.5)
+	checkErr(t, err)
+}
--- a/hrp/internal/uixt/tap.go
+++ b/hrp/internal/uixt/tap.go
@@ -0,0 +1,88 @@
+package uixt
+
+import (
+	"fmt"
+
+	"github.com/electricbubble/gwda"
+)
+
+func (dExt *DriverExt) TapXY(x, y float64) error {
+	// tap on coordinate: [x, y] should be relative
+	if x > 1 || y > 1 {
+		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
+	}
+
+	x = x * float64(dExt.windowSize.Width)
+	y = y * float64(dExt.windowSize.Height)
+	return dExt.WebDriver.TapFloat(x, y)
+}
+
+func (dExt *DriverExt) Tap(param string) error {
+	return dExt.TapOffset(param, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64) (err error) {
+	// click on element, find by name attribute
+	ele, err := dExt.FindUIElement(param)
+	if err == nil {
+		return ele.Click()
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
+		return err
+	}
+
+	return dExt.WebDriver.TapFloat(x+width*xOffset, y+height*yOffset)
+}
+
+func (dExt *DriverExt) DoubleTapXY(x, y float64) error {
+	// double tap on coordinate: [x, y] should be relative
+	if x > 1 || y > 1 {
+		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
+	}
+
+	x = x * float64(dExt.windowSize.Width)
+	y = y * float64(dExt.windowSize.Height)
+	return dExt.WebDriver.DoubleTapFloat(x, y)
+}
+
+func (dExt *DriverExt) DoubleTap(param string) (err error) {
+	return dExt.DoubleTapOffset(param, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) {
+	// click on element, find by name attribute
+	ele, err := dExt.FindUIElement(param)
+	if err == nil {
+		return ele.DoubleTap()
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
+		return err
+	}
+
+	return dExt.WebDriver.DoubleTapFloat(x+width*xOffset, y+height*yOffset)
+}
+
+// TapWithNumber sends one or more taps
+func (dExt *DriverExt) TapWithNumber(param string, numberOfTaps int) (err error) {
+	return dExt.TapWithNumberOffset(param, numberOfTaps, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) TapWithNumberOffset(param string, numberOfTaps int, xOffset, yOffset float64) (err error) {
+	if numberOfTaps <= 0 {
+		numberOfTaps = 1
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
+		return err
+	}
+
+	x = x + width*xOffset
+	y = y + height*yOffset
+
+	touchActions := gwda.NewTouchActions().Tap(gwda.NewTouchActionTap().WithXYFloat(x, y).WithCount(numberOfTaps))
+	return dExt.PerformTouchActions(touchActions)
+}
--- a/hrp/internal/uixt/tap_test.go
+++ b/hrp/internal/uixt/tap_test.go
@@ -0,0 +1,48 @@
+package uixt
+
+import (
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExt_TapWithNumber(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/flag7.png"
+
+	// gwda.SetDebug(true)
+
+	err = driverExt.TapWithNumber(pathSearch, 3)
+	checkErr(t, err)
+
+	err = driverExt.TapWithNumberOffset(pathSearch, 3, 0.5, 0.75)
+	checkErr(t, err)
+}
+
+func TestDriverExt_TapXY(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	err = driverExt.TapXY(0.4, 0.5)
+	checkErr(t, err)
+}
+
+func TestDriverExt_TapWithOCR(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	// 需要点击文字上方的图标
+	err = driverExt.TapOffset("抖音", 0.5, -1)
+	checkErr(t, err)
+}
--- a/hrp/internal/uixt/touch.go
+++ b/hrp/internal/uixt/touch.go
@@ -0,0 +1,33 @@
+package uixt
+
+func (dExt *DriverExt) ForceTouch(pathname string, pressure float64, duration ...float64) (err error) {
+	return dExt.ForceTouchOffset(pathname, pressure, 0.5, 0.5, duration...)
+}
+
+func (dExt *DriverExt) ForceTouchOffset(pathname string, pressure, xOffset, yOffset float64, duration ...float64) (err error) {
+	if len(duration) == 0 {
+		duration = []float64{1.0}
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	return dExt.ForceTouchFloat(x+width*xOffset, y+height*yOffset, pressure, duration[0])
+}
+
+func (dExt *DriverExt) TouchAndHold(pathname string, duration ...float64) (err error) {
+	return dExt.TouchAndHoldOffset(pathname, 0.5, 0.5, duration...)
+}
+
+func (dExt *DriverExt) TouchAndHoldOffset(pathname string, xOffset, yOffset float64, duration ...float64) (err error) {
+	if len(duration) == 0 {
+		duration = []float64{1.0}
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	return dExt.TouchAndHoldFloat(x+width*xOffset, y+height*yOffset, duration[0])
+}
--- a/hrp/internal/uixt/touch_test.go
+++ b/hrp/internal/uixt/touch_test.go
@@ -0,0 +1,55 @@
+package uixt
+
+import (
+	"testing"
+
+	"github.com/electricbubble/gwda"
+)
+
+func TestDriverExt_ForceTouch(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png"
+
+	err = driverExt.ForceTouch(pathSearch, 0.5, 3)
+	checkErr(t, err)
+
+	// err = driverExt.ForceTouchOffset(pathSearch, 0.5, 0.1, 0.9)
+	// checkErr(t, err)
+
+	// err = driverExt.ForceTouchOffset(pathSearch, 0.2, 1.1, -1)
+	// checkErr(t, err)
+}
+
+func TestDriverExt_TouchAndHold(t *testing.T) {
+	driver, err := gwda.NewUSBDriver(nil)
+	checkErr(t, err)
+
+	driverExt, err := Extend(driver, 0.95)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png"
+
+	// err = driverExt.TouchAndHold(pathSearch)
+	// checkErr(t, err)
+
+	// err = driverExt.TouchAndHold(pathSearch, 3)
+	// checkErr(t, err)
+
+	err = driverExt.TouchAndHoldOffset(pathSearch, 0.8, 0.1)
+	checkErr(t, err)
+}
+
+func checkErr(t *testing.T, err error, msg ...string) {
+	if err != nil {
+		if len(msg) == 0 {
+			t.Fatal(err)
+		} else {
+			t.Fatal(msg, err)
+		}
+	}
+}
--- a/hrp/step_ios_ui.go
+++ b/hrp/step_ios_ui.go
@@ -6,9 +6,10 @@ import (
 	"time"

 	"github.com/electricbubble/gwda"
-	"github.com/httprunner/uixt"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/uixt"
 )

 const (
@@ -651,23 +652,6 @@ func (w *wdaClient) doValidation(iValidators []interface{}) (validateResults []*
 	return
 }

-func (w *wdaClient) findElement(param string) (ele gwda.WebElement, err error) {
-	var selector gwda.BySelector
-	if strings.HasPrefix(param, "/") {
-		// xpath
-		selector = gwda.BySelector{
-			XPath: param,
-		}
-	} else {
-		// name
-		selector = gwda.BySelector{
-			LinkText: gwda.NewElementAttribute().WithName(param),
-		}
-	}
-
-	return w.DriverExt.FindElement(selector)
-}
-
 func (w *wdaClient) assertName(name string, exists bool) bool {
 	selector := gwda.BySelector{
 		LinkText: gwda.NewElementAttribute().WithName(name),