refactor: move uixt from hrp internal to pkg

hrp/pkg/uixt/README.md

@@ -0,0 +1,51 @@
+# uixt
+
+From v4.3.0，HttpRunner will support mobile UI automation testing:
+
+- iOS: based on [appium/WebDriverAgent], with forked client library [electricbubble/gwda] in golang
+- Android: based on [appium-uiautomator2-server], with forked client library [electricbubble/guia2] in golang
+
+Some UI recognition algorithms are also introduced for both iOS and Android:
+
+- OpenCV: based on [OpenCV 4], with golang bindings [hybridgroup/gocv] and helper utils [electricbubble/gwda-ext-opencv]
+- OCR: based on OCR API service from [volcengine], other API service may be extended
+
+## Dependencies
+
+### OpenCV
+
+[OpenCV 4] should be pre-installed.
+
+You can install OpenCV 4.6.0 using Homebrew on macOS.
+
+```bash
+$ brew install opencv
+```
+
+You can get more installation introduction on [hybridgroup/gocv].
+
+### OCR
+
+OCR API is a paid service, you need to pre-purchase and configure the account key.
+
+```bash
+$ make build tags=ocr
+```
+
+## Thanks
+
+This uixt module is initially forked from the following repos and made a lot of changes.
+
+- [electricbubble/gwda-ext-opencv]
+- [electricbubble/gwda]
+- [electricbubble/guia2]
+
+
+[electricbubble/gwda-ext-opencv]: https://github.com/electricbubble/gwda-ext-opencv
+[appium/WebDriverAgent]: https://github.com/appium/WebDriverAgent
+[electricbubble/gwda]: https://github.com/electricbubble/gwda
+[electricbubble/guia2]: https://github.com/electricbubble/guia2
+[OpenCV 4]: https://opencv.org/
+[hybridgroup/gocv]: https://github.com/hybridgroup/gocv
+[volcengine]: https://www.volcengine.com/product/text-recognition
+[appium-uiautomator2-server]: https://github.com/appium/appium-uiautomator2-server

hrp/pkg/uixt/android_action.go

@@ -0,0 +1,158 @@
+package uixt
+
+import "strings"
+
+type touchGesture struct {
+	Touch PointF  `json:"touch"`
+	Time  float64 `json:"time"`
+}
+
+type TouchAction []touchGesture
+
+func NewTouchAction(cap ...int) *TouchAction {
+	if len(cap) == 0 || cap[0] <= 0 {
+		cap = []int{8}
+	}
+	tmp := make(TouchAction, 0, cap[0])
+	return &tmp
+}
+
+func (ta *TouchAction) Add(x, y int, startTime ...float64) *TouchAction {
+	return ta.AddFloat(float64(x), float64(y), startTime...)
+}
+
+func (ta *TouchAction) AddFloat(x, y float64, startTime ...float64) *TouchAction {
+	if len(startTime) == 0 {
+		var tmp float64 = 0
+		if len(*ta) != 0 {
+			g := (*ta)[len(*ta)-1]
+			tmp = g.Time + 0.05
+		}
+		startTime = []float64{tmp}
+	}
+	*ta = append(*ta, touchGesture{Touch: PointF{x, y}, Time: startTime[0]})
+	return ta
+}
+
+func (ta *TouchAction) AddPoint(point Point, startTime ...float64) *TouchAction {
+	return ta.AddFloat(float64(point.X), float64(point.Y), startTime...)
+}
+
+func (ta *TouchAction) AddPointF(point PointF, startTime ...float64) *TouchAction {
+	return ta.AddFloat(point.X, point.Y, startTime...)
+}
+
+func (ud *uiaDriver) MultiPointerGesture(gesture1 *TouchAction, gesture2 *TouchAction, tas ...*TouchAction) (err error) {
+	// Must provide coordinates for at least 2 pointers
+	actions := make([]*TouchAction, 0)
+	actions = append(actions, gesture1, gesture2)
+	if len(tas) != 0 {
+		actions = append(actions, tas...)
+	}
+	data := map[string]interface{}{
+		"actions": actions,
+	}
+	// register(postHandler, new MultiPointerGesture("/wd/hub/session/:sessionId/touch/multi/perform"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "/touch/multi/perform")
+	return
+}
+
+type w3cGesture map[string]interface{}
+
+func _newW3CGesture() w3cGesture {
+	return make(w3cGesture)
+}
+
+func (g w3cGesture) _set(key string, value interface{}) w3cGesture {
+	g[key] = value
+	return g
+}
+
+func (g w3cGesture) pause(duration float64) w3cGesture {
+	return g._set("type", "pause").
+		_set("duration", duration)
+}
+
+func (g w3cGesture) keyDown(value string) w3cGesture {
+	return g._set("type", "keyDown").
+		_set("value", value)
+}
+
+func (g w3cGesture) keyUp(value string) w3cGesture {
+	return g._set("type", "keyUp").
+		_set("value", value)
+}
+
+func (g w3cGesture) pointerDown(button int) w3cGesture {
+	return g._set("type", "pointerDown")._set("button", button)
+}
+
+func (g w3cGesture) pointerUp(button int) w3cGesture {
+	return g._set("type", "pointerUp")._set("button", button)
+}
+
+func (g w3cGesture) pointerMove(x, y float64, origin string, duration float64, pressureAndSize ...float64) w3cGesture {
+	switch len(pressureAndSize) {
+	case 1:
+		g._set("pressure", pressureAndSize[0])
+	case 2:
+		g._set("pressure", pressureAndSize[0])
+		g._set("size", pressureAndSize[1])
+	}
+	return g._set("type", "pointerMove").
+		_set("duration", duration).
+		_set("origin", origin).
+		_set("x", x).
+		_set("y", y)
+}
+
+func (g w3cGesture) size(size ...float64) w3cGesture {
+	if len(size) == 0 {
+		size = []float64{1.0}
+	}
+	return g._set("size", size[0])
+}
+
+func (g w3cGesture) pressure(pressure ...float64) w3cGesture {
+	if len(pressure) == 0 {
+		pressure = []float64{1.0}
+	}
+	return g._set("pressure", pressure[0])
+}
+
+type W3CGestures []w3cGesture
+
+func NewW3CGestures(cap ...int) *W3CGestures {
+	if len(cap) == 0 || cap[0] <= 0 {
+		cap = []int{8}
+	}
+	tmp := make(W3CGestures, 0, cap[0])
+	return &tmp
+}
+
+func (g *W3CGestures) Pause(duration ...float64) *W3CGestures {
+	if len(duration) == 0 || duration[0] < 0 {
+		duration = []float64{0.5}
+	}
+	*g = append(*g, _newW3CGesture().pause(duration[0]*1000))
+	return g
+}
+
+func (g *W3CGestures) KeyDown(value string) *W3CGestures {
+	*g = append(*g, _newW3CGesture().keyDown(value))
+	return g
+}
+
+func (g *W3CGestures) KeyUp(value string) *W3CGestures {
+	*g = append(*g, _newW3CGesture().keyUp(value))
+	return g
+}
+
+func (g *W3CGestures) SendKeys(text string) *W3CGestures {
+	ss := strings.Split(text, "")
+	for i := range ss {
+		g.KeyDown(ss[i])
+		g.KeyUp(ss[i])
+	}
+	return g
+}

hrp/pkg/uixt/android_device.go

@@ -0,0 +1,686 @@
+package uixt
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os/exec"
+	"reflect"
+	"strings"
+	"syscall"
+
+	"github.com/electricbubble/gadb"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+var (
+	AdbServerHost  = "localhost"
+	AdbServerPort  = gadb.AdbServerPort // 5037
+	UIA2ServerPort = 6790
+	DeviceTempPath = "/data/local/tmp"
+)
+
+const forwardToPrefix = "forward-to-"
+
+func InitUIAClient(device *AndroidDevice) (*DriverExt, error) {
+	var deviceOptions []AndroidDeviceOption
+	if device.SerialNumber != "" {
+		deviceOptions = append(deviceOptions, WithSerialNumber(device.SerialNumber))
+	}
+	if device.IP != "" {
+		deviceOptions = append(deviceOptions, WithAdbIP(device.IP))
+	}
+	if device.Port != 0 {
+		deviceOptions = append(deviceOptions, WithAdbPort(device.Port))
+	}
+
+	// init uia device
+	androidDevice, err := NewAndroidDevice(deviceOptions...)
+	if err != nil {
+		return nil, err
+	}
+
+	driver, err := androidDevice.NewUSBDriver(nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to init UIA driver")
+	}
+	fmt.Println(driver)
+
+	var driverExt *DriverExt
+
+	driverExt, err = Extend(driver)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to extend UIA Driver")
+	}
+
+	if device.LogOn {
+		err = driverExt.Driver.StartCaptureLog("hrp_adb_log")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	driverExt.UUID = androidDevice.UUID()
+	return driverExt, err
+}
+
+type AndroidDeviceOption func(*AndroidDevice)
+
+func WithSerialNumber(serial string) AndroidDeviceOption {
+	return func(device *AndroidDevice) {
+		device.SerialNumber = serial
+	}
+}
+
+func WithAdbIP(ip string) AndroidDeviceOption {
+	return func(device *AndroidDevice) {
+		device.IP = ip
+	}
+}
+
+func WithAdbPort(port int) AndroidDeviceOption {
+	return func(device *AndroidDevice) {
+		device.Port = port
+	}
+}
+
+func WithAdbLogOn(logOn bool) AndroidDeviceOption {
+	return func(device *AndroidDevice) {
+		device.LogOn = logOn
+	}
+}
+
+func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, err error) {
+	deviceList, err := DeviceList()
+	if err != nil {
+		return nil, fmt.Errorf("get attached devices failed: %v", err)
+	}
+
+	device = &AndroidDevice{
+		Port: UIA2ServerPort,
+		IP:   AdbServerHost,
+	}
+	for _, option := range options {
+		option(device)
+	}
+
+	serialNumber := device.SerialNumber
+	for _, dev := range deviceList {
+		// find device by serial number if specified
+		if serialNumber != "" && dev.Serial() != serialNumber {
+			continue
+		}
+
+		device.SerialNumber = dev.Serial()
+		device.d = dev
+		device.logcat = NewAdbLogcat(serialNumber)
+		return device, nil
+	}
+
+	return nil, fmt.Errorf("device %s not found", device.SerialNumber)
+}
+
+type AndroidDevice struct {
+	d            gadb.Device
+	logcat       *DeviceLogcat
+	SerialNumber string `json:"serial,omitempty" yaml:"serial,omitempty"`
+	IP           string `json:"ip,omitempty" yaml:"ip,omitempty"`
+	Port         int    `json:"port,omitempty" yaml:"port,omitempty"`
+	MjpegPort    int    `json:"mjpeg_port,omitempty" yaml:"mjpeg_port,omitempty"`
+	LogOn        bool   `json:"log_on,omitempty" yaml:"log_on,omitempty"`
+}
+
+func (o AndroidDevice) UUID() string {
+	return o.SerialNumber
+}
+
+func DeviceList() (devices []gadb.Device, err error) {
+	var adbClient gadb.Client
+	if adbClient, err = gadb.NewClientWith(AdbServerHost, AdbServerPort); err != nil {
+		return nil, err
+	}
+
+	return adbClient.DeviceList()
+}
+
+// NewUSBDriver creates new client via USB connected device, this will also start a new session.
+// TODO: replace uiaDriver with WebDriver
+func (dev *AndroidDevice) NewUSBDriver(capabilities Capabilities) (driver *uiaDriver, err error) {
+	var localPort int
+	if localPort, err = getFreePort(); err != nil {
+		return nil, err
+	}
+	if err = dev.d.Forward(localPort, UIA2ServerPort); err != nil {
+		return nil, err
+	}
+
+	rawURL := fmt.Sprintf("http://%s%d:6790/wd/hub", forwardToPrefix, localPort)
+	driver, err = NewUIADriver(capabilities, rawURL)
+	if err != nil {
+		_ = dev.d.ForwardKill(localPort)
+		return nil, err
+	}
+	driver.adbDevice = dev.d
+	driver.logcat = dev.logcat
+	driver.localPort = localPort
+
+	return driver, nil
+}
+
+// NewHTTPDriver creates new remote HTTP client, this will also start a new session.
+// TODO: replace uiaDriver with WebDriver
+func (dev *AndroidDevice) NewHTTPDriver(capabilities Capabilities) (driver *uiaDriver, err error) {
+	rawURL := fmt.Sprintf("http://%s:%d/wd/hub", dev.IP, dev.Port)
+	if driver, err = NewUIADriver(capabilities, rawURL); err != nil {
+		return nil, err
+	}
+	driver.adbDevice = dev.d
+	return driver, nil
+}
+
+func getFreePort() (int, error) {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		return 0, errors.Wrap(err, "resolve tcp addr failed")
+	}
+
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return 0, errors.Wrap(err, "listen tcp addr failed")
+	}
+	defer func() { _ = l.Close() }()
+	return l.Addr().(*net.TCPAddr).Port, nil
+}
+
+type DeviceLogcat struct {
+	serial    string
+	logBuffer *bytes.Buffer
+	errs      []error
+	stopping  chan struct{}
+	done      chan struct{}
+	cmd       *exec.Cmd
+}
+
+func NewAdbLogcat(serial string) *DeviceLogcat {
+	return &DeviceLogcat{
+		serial:    serial,
+		logBuffer: new(bytes.Buffer),
+		stopping:  make(chan struct{}),
+		done:      make(chan struct{}),
+	}
+}
+
+// CatchLogcatContext starts logcat with timeout context
+func (l *DeviceLogcat) CatchLogcatContext(timeoutCtx context.Context) (err error) {
+	if err = l.CatchLogcat(); err != nil {
+		return
+	}
+	go func() {
+		select {
+		case <-timeoutCtx.Done():
+			_ = l.Stop()
+		case <-l.stopping:
+		}
+	}()
+	return
+}
+
+func (l *DeviceLogcat) Stop() error {
+	select {
+	case <-l.stopping:
+	default:
+		close(l.stopping)
+		<-l.done
+		close(l.done)
+	}
+	return l.Errors()
+}
+
+func (l *DeviceLogcat) Errors() (err error) {
+	for _, e := range l.errs {
+		if err != nil {
+			err = fmt.Errorf("%v |[DeviceLogcatErr] %v", err, e)
+		} else {
+			err = fmt.Errorf("[DeviceLogcatErr] %v", e)
+		}
+	}
+	return
+}
+
+func (l *DeviceLogcat) CatchLogcat() (err error) {
+	if l.cmd != nil {
+		err = fmt.Errorf("logcat already start")
+	}
+	cmdLine := fmt.Sprintf("adb -s %s logcat -c && adb -s %s logcat -v time -s iesqaMonitor:V", l.serial, l.serial)
+	l.cmd = builtin.Command(cmdLine)
+	l.cmd.Stderr = l.logBuffer
+	l.cmd.Stdout = l.logBuffer
+	l.cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	if err = l.cmd.Start(); err != nil {
+		return
+	}
+	go func() {
+		<-l.stopping
+		if e := syscall.Kill(-l.cmd.Process.Pid, syscall.SIGKILL); e != nil {
+			l.errs = append(l.errs, fmt.Errorf("kill logcat process err:%v", e))
+		}
+		l.done <- struct{}{}
+	}()
+	return
+}
+
+func (l *DeviceLogcat) BufferedLogcat() (err error) {
+	// -d: dump the current buffered logcat result and exits
+	cmdLine := fmt.Sprintf("adb -s %s logcat -d", l.serial)
+	cmd := builtin.Command(cmdLine)
+	cmd.Stdout = l.logBuffer
+	cmd.Stderr = l.logBuffer
+	if err = cmd.Run(); err != nil {
+		return
+	}
+	return
+}
+
+type ExportPoint struct {
+	Start     int         `json:"start" yaml:"start"`
+	End       int         `json:"end" yaml:"end"`
+	From      interface{} `json:"from" yaml:"from"`
+	To        interface{} `json:"to" yaml:"to"`
+	Operation string      `json:"operation" yaml:"operation"`
+	Ext       string      `json:"ext" yaml:"ext"`
+	RunTime   int         `json:"run_time,omitempty" yaml:"run_time,omitempty"`
+}
+
+func ConvertPoints(data string) (eps []ExportPoint) {
+	lines := strings.Split(data, "\n")
+	for _, line := range lines {
+		if strings.Contains(line, "ext") {
+			idx := strings.Index(line, "{")
+			line = line[idx:]
+			p := ExportPoint{}
+			err := json.Unmarshal([]byte(line), &p)
+			if err != nil {
+				log.Error().Msg("failed to parse point data")
+				continue
+			}
+			eps = append(eps, p)
+		}
+	}
+	return
+}
+
+type UiSelectorHelper struct {
+	value *bytes.Buffer
+}
+
+func NewUiSelectorHelper() UiSelectorHelper {
+	return UiSelectorHelper{value: bytes.NewBufferString("new UiSelector()")}
+}
+
+func (s UiSelectorHelper) String() string {
+	return s.value.String() + ";"
+}
+
+// Text Set the search criteria to match the visible text displayed
+// in a widget (for example, the text label to launch an app).
+//
+// The text for the element must match exactly with the string in your input
+// argument. Matching is case-sensitive.
+func (s UiSelectorHelper) Text(text string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.text("%s")`, text))
+	return s
+}
+
+// TextMatches Set the search criteria to match the visible text displayed in a layout
+// element, using a regular expression.
+//
+// The text in the widget must match exactly with the string in your
+// input argument.
+func (s UiSelectorHelper) TextMatches(regex string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.textMatches("%s")`, regex))
+	return s
+}
+
+// TextStartsWith Set the search criteria to match visible text in a widget that is
+// prefixed by the text parameter.
+//
+// The matching is case-insensitive.
+func (s UiSelectorHelper) TextStartsWith(text string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.textStartsWith("%s")`, text))
+	return s
+}
+
+// TextContains Set the search criteria to match the visible text in a widget
+// where the visible text must contain the string in your input argument.
+//
+// The matching is case-sensitive.
+func (s UiSelectorHelper) TextContains(text string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.textContains("%s")`, text))
+	return s
+}
+
+// ClassName Set the search criteria to match the class property
+// for a widget (for example, "android.widget.Button").
+func (s UiSelectorHelper) ClassName(className string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.className("%s")`, className))
+	return s
+}
+
+// ClassNameMatches Set the search criteria to match the class property
+// for a widget, using a regular expression.
+func (s UiSelectorHelper) ClassNameMatches(regex string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.classNameMatches("%s")`, regex))
+	return s
+}
+
+// Description Set the search criteria to match the content-description
+// property for a widget.
+//
+// The content-description is typically used
+// by the Android Accessibility framework to
+// provide an audio prompt for the widget when
+// the widget is selected. The content-description
+// for the widget must match exactly
+// with the string in your input argument.
+//
+// Matching is case-sensitive.
+func (s UiSelectorHelper) Description(desc string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.description("%s")`, desc))
+	return s
+}
+
+// DescriptionMatches Set the search criteria to match the content-description
+// property for a widget.
+//
+// The content-description is typically used
+// by the Android Accessibility framework to
+// provide an audio prompt for the widget when
+// the widget is selected. The content-description
+// for the widget must match exactly
+// with the string in your input argument.
+func (s UiSelectorHelper) DescriptionMatches(regex string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.descriptionMatches("%s")`, regex))
+	return s
+}
+
+// DescriptionStartsWith Set the search criteria to match the content-description
+// property for a widget.
+//
+// The content-description is typically used
+// by the Android Accessibility framework to
+// provide an audio prompt for the widget when
+// the widget is selected. The content-description
+// for the widget must start
+// with the string in your input argument.
+//
+// Matching is case-insensitive.
+func (s UiSelectorHelper) DescriptionStartsWith(desc string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.descriptionStartsWith("%s")`, desc))
+	return s
+}
+
+// DescriptionContains Set the search criteria to match the content-description
+// property for a widget.
+//
+// The content-description is typically used
+// by the Android Accessibility framework to
+// provide an audio prompt for the widget when
+// the widget is selected. The content-description
+// for the widget must contain
+// the string in your input argument.
+//
+// Matching is case-insensitive.
+func (s UiSelectorHelper) DescriptionContains(desc string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.descriptionContains("%s")`, desc))
+	return s
+}
+
+// ResourceId Set the search criteria to match the given resource ID.
+func (s UiSelectorHelper) ResourceId(id string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.resourceId("%s")`, id))
+	return s
+}
+
+// ResourceIdMatches Set the search criteria to match the resource ID
+// of the widget, using a regular expression.
+func (s UiSelectorHelper) ResourceIdMatches(regex string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.resourceIdMatches("%s")`, regex))
+	return s
+}
+
+// Index Set the search criteria to match the widget by its node
+// index in the layout hierarchy.
+//
+// The index value must be 0 or greater.
+//
+// Using the index can be unreliable and should only
+// be used as a last resort for matching. Instead,
+// consider using the `Instance(int)` method.
+func (s UiSelectorHelper) Index(index int) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.index(%d)`, index))
+	return s
+}
+
+// Instance Set the search criteria to match the
+// widget by its instance number.
+//
+// The instance value must be 0 or greater, where
+// the first instance is 0.
+//
+// For example, to simulate a user click on
+// the third image that is enabled in a UI screen, you
+// could specify a a search criteria where the instance is
+// 2, the `className(String)` matches the image
+// widget class, and `enabled(boolean)` is true.
+// The code would look like this:
+//  `new UiSelector().className("android.widget.ImageView")
+//    .enabled(true).instance(2);`
+func (s UiSelectorHelper) Instance(instance int) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.instance(%d)`, instance))
+	return s
+}
+
+// Enabled Set the search criteria to match widgets that are enabled.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Enabled(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.enabled(%t)`, b))
+	return s
+}
+
+// Focused Set the search criteria to match widgets that have focus.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Focused(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.focused(%t)`, b))
+	return s
+}
+
+// Focusable Set the search criteria to match widgets that are focusable.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Focusable(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.focusable(%t)`, b))
+	return s
+}
+
+// Scrollable Set the search criteria to match widgets that are scrollable.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Scrollable(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.scrollable(%t)`, b))
+	return s
+}
+
+// Selected Set the search criteria to match widgets that
+// are currently selected.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Selected(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.selected(%t)`, b))
+	return s
+}
+
+// Checked Set the search criteria to match widgets that
+// are currently checked (usually for checkboxes).
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Checked(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.checked(%t)`, b))
+	return s
+}
+
+// Checkable Set the search criteria to match widgets that are checkable.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Checkable(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.checkable(%t)`, b))
+	return s
+}
+
+// Clickable Set the search criteria to match widgets that are clickable.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) Clickable(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.clickable(%t)`, b))
+	return s
+}
+
+// LongClickable Set the search criteria to match widgets that are long-clickable.
+//
+// Typically, using this search criteria alone is not useful.
+// You should also include additional criteria, such as text,
+// content-description, or the class name for a widget.
+//
+// If no other search criteria is specified, and there is more
+// than one matching widget, the first widget in the tree
+// is selected.
+func (s UiSelectorHelper) LongClickable(b bool) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.longClickable(%t)`, b))
+	return s
+}
+
+// packageName Set the search criteria to match the package name
+// of the application that contains the widget.
+func (s UiSelectorHelper) packageName(name string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.packageName(%s)`, name))
+	return s
+}
+
+// PackageNameMatches Set the search criteria to match the package name
+// of the application that contains the widget.
+func (s UiSelectorHelper) PackageNameMatches(regex string) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.packageNameMatches(%s)`, regex))
+	return s
+}
+
+// ChildSelector Adds a child UiSelector criteria to this selector.
+//
+// Use this selector to narrow the search scope to
+// child widgets under a specific parent widget.
+func (s UiSelectorHelper) ChildSelector(selector UiSelectorHelper) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.childSelector(%s)`, selector.value.String()))
+	return s
+}
+
+func (s UiSelectorHelper) PatternSelector(selector UiSelectorHelper) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.patternSelector(%s)`, selector.value.String()))
+	return s
+}
+
+func (s UiSelectorHelper) ContainerSelector(selector UiSelectorHelper) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.containerSelector(%s)`, selector.value.String()))
+	return s
+}
+
+// FromParent Adds a child UiSelector criteria to this selector which is used to
+// start search from the parent widget.
+//
+// Use this selector to narrow the search scope to
+// sibling widgets as well all child widgets under a parent.
+func (s UiSelectorHelper) FromParent(selector UiSelectorHelper) UiSelectorHelper {
+	s.value.WriteString(fmt.Sprintf(`.fromParent(%s)`, selector.value.String()))
+	return s
+}
+
+type AndroidBySelector struct {
+	// Set the search criteria to match the given resource ResourceIdID.
+	ResourceIdID string `json:"id"`
+	// Set the search criteria to match the content-description property for a widget.
+	ContentDescription string `json:"accessibility id"`
+	XPath              string `json:"xpath"`
+	// Set the search criteria to match the class property for a widget (for example, "android.widget.Button").
+	ClassName   string `json:"class name"`
+	UiAutomator string `json:"-android uiautomator"`
+}
+
+func (by AndroidBySelector) getMethodAndSelector() (method, selector string) {
+	vBy := reflect.ValueOf(by)
+	tBy := reflect.TypeOf(by)
+	for i := 0; i < vBy.NumField(); i++ {
+		vi := vBy.Field(i).Interface()
+		// switch vi := vi.(type) {
+		// case string:
+		// 	selector = vi
+		// }
+		selector = vi.(string)
+		if selector != "" && selector != "UNKNOWN" {
+			method = tBy.Field(i).Tag.Get("json")
+			return
+		}
+	}
+	return
+}

hrp/pkg/uixt/android_device_test.go

@@ -0,0 +1,18 @@
+package uixt
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+func TestConvertPoints(t *testing.T) {
+	data := "10-09 20:16:48.216 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317808206,\"ext\":\"输入\",\"from\":{\"x\":0.0,\"y\":0.0},\"operation\":\"Gtf-SendKeys\",\"run_time\":627,\"start\":1665317807579,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":0.0,\"y\":0.0}}\n10-09 20:18:22.899 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317902898,\"ext\":\"进入直播间\",\"from\":{\"x\":717.0,\"y\":2117.5},\"operation\":\"Gtf-Tap\",\"run_time\":121,\"start\":1665317902777,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":717.0,\"y\":2117.5}}\n10-09 20:18:32.063 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317912062,\"ext\":\"第一次上划\",\"from\":{\"x\":1437.0,\"y\":2409.9},\"operation\":\"Gtf-Swipe\",\"run_time\":32,\"start\":1665317912030,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":1437.0,\"y\":2409.9}}"
+	eps := ConvertPoints(data)
+	if len(eps) != 3 {
+		t.Fatal()
+	}
+	jsons, _ := json.Marshal(eps)
+	println(fmt.Sprintf("%v", string(jsons)))
+}

hrp/pkg/uixt/android_driver.go

@@ -0,0 +1,996 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/electricbubble/gadb"
+	"github.com/rs/zerolog/log"
+)
+
+var errDriverNotImplemented = errors.New("driver method not implemented")
+
+type uiaDriver struct {
+	Driver
+
+	adbDevice gadb.Device
+	logcat    *DeviceLogcat
+	localPort int
+}
+
+func NewUIADriver(capabilities Capabilities, urlPrefix string) (driver *uiaDriver, err error) {
+	if capabilities == nil {
+		capabilities = NewCapabilities()
+	}
+	driver = new(uiaDriver)
+	if driver.urlPrefix, err = url.Parse(urlPrefix); err != nil {
+		return nil, err
+	}
+	var localPort int
+	{
+		tmpURL, _ := url.Parse(driver.urlPrefix.String())
+		hostname := tmpURL.Hostname()
+		if strings.HasPrefix(hostname, forwardToPrefix) {
+			localPort, _ = strconv.Atoi(strings.TrimPrefix(hostname, forwardToPrefix))
+		}
+	}
+	conn, err := net.Dial("tcp", fmt.Sprintf(":%d", localPort))
+	if err != nil {
+		return nil, fmt.Errorf("adb forward: %w", err)
+	}
+	driver.client = convertToHTTPClient(conn)
+	if session, err := driver.NewSession(capabilities); err != nil {
+		return nil, err
+	} else {
+		driver.sessionId = session.SessionId
+	}
+	return
+}
+
+type BatteryStatus int
+
+const (
+	_                                  = iota
+	BatteryStatusUnknown BatteryStatus = iota
+	BatteryStatusCharging
+	BatteryStatusDischarging
+	BatteryStatusNotCharging
+	BatteryStatusFull
+)
+
+func (bs BatteryStatus) String() string {
+	switch bs {
+	case BatteryStatusUnknown:
+		return "unknown"
+	case BatteryStatusCharging:
+		return "charging"
+	case BatteryStatusDischarging:
+		return "discharging"
+	case BatteryStatusNotCharging:
+		return "not charging"
+	case BatteryStatusFull:
+		return "full"
+	default:
+		return fmt.Sprintf("unknown status code (%d)", bs)
+	}
+}
+
+func (ud *uiaDriver) Close() (err error) {
+	if ud.sessionId == "" {
+		return nil
+	}
+	if _, err = ud.httpDELETE("/session", ud.sessionId); err == nil {
+		ud.sessionId = ""
+	}
+
+	return err
+}
+
+func (ud *uiaDriver) NewSession(capabilities Capabilities) (sessionInfo SessionInfo, err error) {
+	// register(postHandler, new NewSession("/wd/hub/session"))
+	var rawResp rawResponse
+	data := map[string]interface{}{"capabilities": capabilities}
+	if rawResp, err = ud.httpPOST(data, "/session"); err != nil {
+		return SessionInfo{SessionId: ""}, err
+	}
+	reply := new(struct{ Value struct{ SessionId string } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return SessionInfo{SessionId: ""}, err
+	}
+	sessionID := reply.Value.SessionId
+	// d.sessionIdCache[sessionID] = true
+	return SessionInfo{SessionId: sessionID}, nil
+}
+
+func (ud *uiaDriver) ActiveSession() (sessionInfo SessionInfo, err error) {
+	// [[FBRoute GET:@""] respondWithTarget:self action:@selector(handleGetActiveSession:)]
+	return SessionInfo{SessionId: ud.sessionId}, nil
+}
+
+func (ud *uiaDriver) SessionIDs() (sessionIDs []string, err error) {
+	// register(getHandler, new GetSessions("/wd/hub/sessions"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/sessions"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value []struct{ SessionId string } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+
+	sessionIDs = make([]string, len(reply.Value))
+	for i := range reply.Value {
+		sessionIDs[i] = reply.Value[i].SessionId
+	}
+	return
+}
+
+func (ud *uiaDriver) SessionDetails() (scrollData map[string]interface{}, err error) {
+	// register(getHandler, new GetSessionDetails("/wd/hub/session/:sessionId"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value map[string]interface{} })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+
+	scrollData = reply.Value
+	return
+}
+
+func (ud *uiaDriver) DeleteSession() (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Status() (deviceStatus DeviceStatus, err error) {
+	// register(getHandler, new Status("/wd/hub/status"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/status"); err != nil {
+		return DeviceStatus{Ready: false}, err
+	}
+	reply := new(struct {
+		Value struct {
+			// Message string
+			Ready bool
+		}
+	})
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return DeviceStatus{Ready: false}, err
+	}
+	return DeviceStatus{Ready: true}, nil
+}
+
+func (ud *uiaDriver) DeviceInfo() (deviceInfo DeviceInfo, err error) {
+	// register(getHandler, new GetDeviceInfo("/wd/hub/session/:sessionId/appium/device/info"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "appium/device/info"); err != nil {
+		return DeviceInfo{}, err
+	}
+	reply := new(struct{ Value struct{ DeviceInfo } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return DeviceInfo{}, err
+	}
+	deviceInfo = reply.Value.DeviceInfo
+	return
+}
+
+func (ud *uiaDriver) Location() (location Location, err error) {
+	// TODO
+	return location, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) BatteryInfo() (batteryInfo BatteryInfo, err error) {
+	// register(getHandler, new GetBatteryInfo("/wd/hub/session/:sessionId/appium/device/battery_info"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "appium/device/battery_info"); err != nil {
+		return BatteryInfo{}, err
+	}
+	reply := new(struct{ Value struct{ BatteryInfo } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return BatteryInfo{}, err
+	}
+	if reply.Value.Level == -1 || reply.Value.Status == -1 {
+		return reply.Value.BatteryInfo, errors.New("cannot be retrieved from the system")
+	}
+	batteryInfo = reply.Value.BatteryInfo
+	return
+}
+
+func (ud *uiaDriver) WindowSize() (size Size, err error) {
+	// register(getHandler, new GetDeviceSize("/wd/hub/session/:sessionId/window/:windowHandle/size"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "window/:windowHandle/size"); err != nil {
+		return Size{}, err
+	}
+	reply := new(struct{ Value struct{ Size } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Size{}, err
+	}
+	size = reply.Value.Size
+	return
+}
+
+func (ud *uiaDriver) Screen() (screen Screen, err error) {
+	// TODO
+	return screen, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Scale() (scale float64, err error) {
+	return 1, nil
+}
+
+// PressBack simulates a short press on the BACK button.
+func (ud *uiaDriver) PressBack() (err error) {
+	// register(postHandler, new PressBack("/wd/hub/session/:sessionId/back"))
+	_, err = ud.httpPOST(nil, "/session", ud.sessionId, "back")
+	return
+}
+
+func (ud *uiaDriver) StartCamera() (err error) {
+	if _, err = ud.adbDevice.RunShellCommand("rm", "-r", "/sdcard/DCIM/Camera"); err != nil {
+		return err
+	}
+	time.Sleep(5 * time.Second)
+	var version string
+	if version, err = ud.adbDevice.RunShellCommand("getprop", "ro.build.version.release"); err != nil {
+		return err
+	}
+	if version == "11" || version == "12" {
+		if _, err = ud.adbDevice.RunShellCommand("am", "start", "-a", "android.media.action.STILL_IMAGE_CAMERA"); err != nil {
+			return err
+		}
+		time.Sleep(5 * time.Second)
+		if _, err = ud.adbDevice.RunShellCommand("input", "swipe", "750", "1000", "250", "1000"); err != nil {
+			return err
+		}
+		time.Sleep(5 * time.Second)
+		if _, err = ud.adbDevice.RunShellCommand("input", "keyevent", "27"); err != nil {
+			return err
+		}
+		return
+	} else {
+		if _, err = ud.adbDevice.RunShellCommand("am", "start", "-a", "android.media.action.VIDEO_CAPTURE"); err != nil {
+			return err
+		}
+		time.Sleep(5 * time.Second)
+		if _, err = ud.adbDevice.RunShellCommand("input", "keyevent", "27"); err != nil {
+			return err
+		}
+		return
+	}
+}
+
+func (ud *uiaDriver) StopCamera() (err error) {
+	err = ud.PressBack()
+	if err != nil {
+		return err
+	}
+	err = ud.Homescreen()
+	if err != nil {
+		return err
+	}
+
+	// kill samsung shell command
+	if _, err = ud.adbDevice.RunShellCommand("am", "force-stop", "com.sec.android.app.camera"); err != nil {
+		return err
+	}
+
+	// kill other camera (huawei mi)
+	if _, err = ud.adbDevice.RunShellCommand("am", "force-stop", "com.android.camera2"); err != nil {
+		return err
+	}
+	return
+}
+
+func (ud *uiaDriver) ActiveAppInfo() (info AppInfo, err error) {
+	// TODO
+	return info, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) ActiveAppsList() (appsList []AppBaseInfo, err error) {
+	// TODO
+	return appsList, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) AppState(bundleId string) (runState AppState, err error) {
+	// TODO
+	return runState, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) IsLocked() (locked bool, err error) {
+	// TODO
+	return locked, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Unlock() (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Lock() (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Homescreen() (err error) {
+	return ud.PressKeyCode(KCHome, KMEmpty)
+}
+
+func (ud *uiaDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta, flags ...KeyFlag) (err error) {
+	if len(flags) == 0 {
+		flags = []KeyFlag{KFFromSystem}
+	}
+	return ud._pressKeyCode(keyCode, metaState, KFFromSystem)
+}
+
+func (ud *uiaDriver) _pressKeyCode(keyCode KeyCode, metaState KeyMeta, flags ...KeyFlag) (err error) {
+	// register(postHandler, new PressKeyCodeAsync("/wd/hub/session/:sessionId/appium/device/press_keycode"))
+	data := map[string]interface{}{
+		"keycode": keyCode,
+	}
+	if metaState != KMEmpty {
+		data["metastate"] = metaState
+	}
+	if len(flags) != 0 {
+		data["flags"] = flags[0]
+	}
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/device/press_keycode")
+	return
+}
+
+func (ud *uiaDriver) AlertText() (text string, err error) {
+	// register(getHandler, new GetAlertText("/wd/hub/session/:sessionId/alert/text"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "alert/text"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+
+	text = reply.Value
+	return
+}
+
+func (ud *uiaDriver) AlertButtons() (btnLabels []string, err error) {
+	// TODO
+	return btnLabels, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) AlertAccept(label ...string) (err error) {
+	data := map[string]interface{}{
+		"buttonLabel": nil,
+	}
+	if len(label) != 0 {
+		data["buttonLabel"] = label[0]
+	}
+	// register(postHandler, new AcceptAlert("/wd/hub/session/:sessionId/alert/accept"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "alert/accept")
+	return
+}
+
+func (ud *uiaDriver) AlertDismiss(label ...string) (err error) {
+	data := map[string]interface{}{
+		"buttonLabel": nil,
+	}
+	if len(label) != 0 {
+		data["buttonLabel"] = label[0]
+	}
+	// register(postHandler, new DismissAlert("/wd/hub/session/:sessionId/alert/dismiss"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "alert/dismiss")
+	return
+}
+
+func (ud *uiaDriver) AlertSendKeys(text string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) check() error {
+	if ud.adbDevice.Serial() == "" {
+		return errors.New("adb daemon: the device is not ready")
+	}
+	return nil
+}
+
+func (ud *uiaDriver) AppLaunch(bundleId string, launchOpt ...AppLaunchOption) (err error) {
+	if err = ud.check(); err != nil {
+		return err
+	}
+
+	var sOutput string
+	if sOutput, err = ud.adbDevice.RunShellCommand("monkey -p", bundleId, "-c android.intent.category.LAUNCHER 1"); err != nil {
+		return err
+	}
+	if strings.Contains(sOutput, "monkey aborted") {
+		return fmt.Errorf("app launch: %s", strings.TrimSpace(sOutput))
+	}
+
+	if len(launchOpt) != 0 {
+		var ce error
+		exists := func(ud WebDriver) (bool, error) {
+			for _, opt := range launchOpt {
+				if waitForComplete, ok := opt["androidBySelector"]; ok {
+					for _, e := range waitForComplete.([]BySelector) {
+						_, ce = ud.FindElement(e)
+						if ce == nil {
+							return true, nil
+						}
+					}
+				}
+			}
+			return false, nil
+		}
+		if err = ud.WaitWithTimeoutAndInterval(exists, 45, 1); err != nil {
+			return fmt.Errorf("app launch (waitForComplete): %s: %w", err.Error(), ce)
+		}
+	}
+	return
+}
+
+func (ud *uiaDriver) AppLaunchUnattached(bundleId string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+// Dispose corresponds to the command:
+//  adb -s $serial forward --remove $localPort
+func (ud *uiaDriver) Dispose() (err error) {
+	if err = ud.check(); err != nil {
+		return err
+	}
+	if ud.localPort == 0 {
+		return nil
+	}
+	return ud.adbDevice.ForwardKill(ud.localPort)
+}
+
+func (ud *uiaDriver) AppTerminate(bundleId string) (successful bool, err error) {
+	if err = ud.check(); err != nil {
+		return false, err
+	}
+
+	_, err = ud.adbDevice.RunShellCommand("am force-stop", bundleId)
+	return err == nil, err
+}
+
+func (ud *uiaDriver) AppActivate(bundleId string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) AppDeactivate(second float64) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) AppAuthReset(resource ProtectedResource) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Tap(x, y int, options ...DataOption) error {
+	return ud.TapFloat(float64(x), float64(y), options...)
+}
+
+func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) {
+	// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap")
+	return
+}
+
+func (ud *uiaDriver) DoubleTap(x, y int) error {
+	return ud.DoubleTapFloat(float64(x), float64(y))
+}
+
+func (ud *uiaDriver) DoubleTapFloat(x, y float64) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) TouchAndHold(x, y int, second ...float64) (err error) {
+	return ud.TouchAndHoldFloat(float64(x), float64(y), second...)
+}
+
+func (ud *uiaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err error) {
+	if len(second) == 0 {
+		second = []float64{1.0}
+	}
+	// register(postHandler, new TouchLongClick("/wd/hub/session/:sessionId/touch/longclick"))
+	data := map[string]interface{}{
+		"params": map[string]interface{}{
+			"x":        x,
+			"y":        y,
+			"duration": int(second[0] * 1000),
+		},
+	}
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/longclick")
+	return
+}
+
+func (ud *uiaDriver) _drag(data map[string]interface{}) (err error) {
+	// register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/drag")
+	return
+}
+
+// Drag performs a swipe from one coordinate to another coordinate. You can control
+// the smoothness and speed of the swipe by specifying the number of steps.
+// Each step execution is throttled to 5 milliseconds per step, so for a 100
+// steps, the swipe will take around 0.5 seconds to complete.
+func (ud *uiaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) error {
+	return ud.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
+}
+
+func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) {
+	data := map[string]interface{}{
+		"startX": fromX,
+		"startY": fromY,
+		"endX":   toX,
+		"endY":   toY,
+	}
+
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["steps"]; !ok {
+		data["steps"] = 12 // default steps
+	}
+
+	return ud._drag(data)
+}
+
+func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...DataOption) (err error) {
+	// register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform"))
+	data := map[string]interface{}{
+		"startX": startX,
+		"startY": startY,
+		"endX":   endX,
+		"endY":   endY,
+	}
+
+	// append options in post data for extra uiautomator configurations
+	// e.g. use WithPressDuration to set pressForDuration
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["steps"]; !ok {
+		data["steps"] = 12 // default steps
+	}
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/perform")
+	return
+}
+
+// Swipe performs a swipe from one coordinate to another using the number of steps
+// to determine smoothness and speed. Each step execution is throttled to 5ms
+// per step. So for a 100 steps, the swipe will take about 1/2 second to complete.
+//  `steps` is the number of move steps sent to the system
+func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error {
+	options = append(options, WithSteps(12))
+	return ud.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
+}
+
+func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error {
+	return ud._swipe(fromX, fromY, toX, toY, options...)
+}
+
+func (ud *uiaDriver) ForceTouch(x, y int, pressure float64, second ...float64) error {
+	return ud.ForceTouchFloat(float64(x), float64(y), pressure, second...)
+}
+
+func (ud *uiaDriver) ForceTouchFloat(x, y, pressure float64, second ...float64) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) PerformW3CActions(actions *W3CActions) (err error) {
+	data := map[string]interface{}{
+		"actions": actions,
+	}
+	// register(postHandler, new W3CActions("/wd/hub/session/:sessionId/actions"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "/actions")
+	return
+}
+
+func (ud *uiaDriver) PerformAppiumTouchActions(touchActs *TouchActions) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) SetPasteboard(contentType PasteboardType, content string) (err error) {
+	lbl := content
+
+	const defaultLabelLen = 10
+	if len(lbl) > defaultLabelLen {
+		lbl = lbl[:defaultLabelLen]
+	}
+
+	data := map[string]interface{}{
+		"contentType": contentType,
+		"label":       lbl,
+		"content":     base64.StdEncoding.EncodeToString([]byte(content)),
+	}
+	// register(postHandler, new SetClipboard("/wd/hub/session/:sessionId/appium/device/set_clipboard"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/device/set_clipboard")
+	return
+}
+
+func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error) {
+	if len(contentType) == 0 {
+		contentType = PasteboardTypePlaintext
+	}
+	// register(postHandler, new GetClipboard("/wd/hub/session/:sessionId/appium/device/get_clipboard"))
+	data := map[string]interface{}{
+		"contentType": contentType[0],
+	}
+	var rawResp rawResponse
+	if rawResp, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/device/get_clipboard"); err != nil {
+		return
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return
+	}
+
+	if data, err := base64.StdEncoding.DecodeString(reply.Value); err != nil {
+		raw.Write([]byte(reply.Value))
+	} else {
+		raw.Write(data)
+	}
+	return
+}
+
+func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) {
+	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
+	// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
+	data := map[string]interface{}{
+		"text": text,
+	}
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["isReplace"]; !ok {
+		data["isReplace"] = true // default true
+	}
+
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "keys")
+	return
+}
+
+func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) {
+	data := map[string]interface{}{
+		"view": text,
+	}
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	var element WebElement
+	if valuetext, ok := data["text"]; ok {
+		element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()})
+	} else if valueid, ok := data["id"]; ok {
+		element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)})
+	} else if valuedesc, ok := data["description"]; ok {
+		element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().Description(fmt.Sprintf("%v", valuedesc)).String()})
+	} else {
+		element, err = ud.FindElement(BySelector{ClassName: ElementType{EditText: true}})
+	}
+	if err != nil {
+		return err
+	}
+	return element.SendKeys(text, options...)
+}
+
+func (ud *uiaDriver) KeyboardDismiss(keyNames ...string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) PressButton(devBtn DeviceButton) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) IOHIDEvent(pageID EventPageID, usageID EventUsageID, duration ...float64) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) ExpectNotification(notifyName string, notifyType NotificationType, second ...int) (err error) {
+	// register(postHandler, new OpenNotification("/wd/hub/session/:sessionId/appium/device/open_notifications"))
+	_, err = ud.httpPOST(nil, "/session", ud.sessionId, "appium/device/open_notifications")
+	return
+}
+
+func (ud *uiaDriver) SiriActivate(text string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) SiriOpenUrl(url string) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Orientation() (orientation Orientation, err error) {
+	// register(getHandler, new GetOrientation("/wd/hub/session/:sessionId/orientation"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "orientation"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value Orientation })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+
+	orientation = reply.Value
+	return
+}
+
+func (ud *uiaDriver) SetOrientation(orientation Orientation) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) Rotation() (rotation Rotation, err error) {
+	// register(getHandler, new GetRotation("/wd/hub/session/:sessionId/rotation"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "rotation"); err != nil {
+		return Rotation{}, err
+	}
+	reply := new(struct{ Value Rotation })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Rotation{}, err
+	}
+
+	rotation = reply.Value
+	return
+}
+
+func (ud *uiaDriver) SetRotation(rotation Rotation) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) MatchTouchID(isMatch bool) (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) _findElements(method, selector string, elementID ...string) (elements []WebElement, err error) {
+	// register(postHandler, new FindElements("/wd/hub/session/:sessionId/elements"))
+	data := map[string]interface{}{
+		"strategy": method,
+		"selector": selector,
+	}
+	if len(elementID) != 0 {
+		data["context"] = elementID[0]
+	}
+	var rawResp rawResponse
+	if rawResp, err = ud.httpPOST(data, "/session", ud.sessionId, "/elements"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value []map[string]string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	if len(reply.Value) == 0 {
+		return nil, fmt.Errorf("no such element: unable to find an element using '%s', value '%s'", method, selector)
+	}
+	elements = make([]WebElement, len(reply.Value))
+	for i, elem := range reply.Value {
+		var id string
+		if id = elementIDFromValue(elem); id == "" {
+			return nil, fmt.Errorf("invalid element returned: %+v", reply)
+		}
+		uie := WebElement(uiaElement{parent: ud, id: id})
+		elements[i] = uie
+	}
+	return
+}
+
+func (ud *uiaDriver) _findElement(method, selector string, elementID ...string) (elem *uiaElement, err error) {
+	// register(postHandler, new FindElement("/wd/hub/session/:sessionId/element"))
+	data := map[string]interface{}{
+		"strategy": method,
+		"selector": selector,
+	}
+	if len(elementID) != 0 {
+		data["context"] = elementID[0]
+	}
+	var rawResp rawResponse
+	if rawResp, err = ud.httpPOST(data, "/session", ud.sessionId, "/element"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value map[string]string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	if len(reply.Value) == 0 {
+		return nil, fmt.Errorf("no such element: unable to find an element using '%s', value '%s'", method, selector)
+	}
+	var id string
+	if id = elementIDFromValue(reply.Value); id == "" {
+		return nil, fmt.Errorf("invalid element returned: %+v", reply)
+	}
+	elem = &uiaElement{parent: ud, id: id}
+	return
+}
+
+func (ud *uiaDriver) ActiveElement() (element WebElement, err error) {
+	// TODO
+	return element, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) FindElement(by BySelector) (element WebElement, err error) {
+	return ud._findElement(by.getUsingAndValue())
+}
+
+func (ud *uiaDriver) FindElements(by BySelector) (elements []WebElement, err error) {
+	// [[FBRoute POST:@"/elements"] respondWithTarget:self action:@selector(handleFindElements:)]
+	using, value := by.getUsingAndValue()
+	data := map[string]interface{}{
+		"using": using,
+		"value": value,
+	}
+	var rawResp rawResponse
+	if rawResp, err = ud.httpPOST(data, "/session", ud.sessionId, "/elements"); err != nil {
+		return nil, err
+	}
+	var elementIDs []string
+	if elementIDs, err = rawResp.valueConvertToElementIDs(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find an element using '%s', value '%s'", err, using, value)
+		}
+		return nil, err
+	}
+	elements = make([]WebElement, len(elementIDs))
+	for i := range elementIDs {
+		elements[i] = WebElement(uiaElement{parent: ud, id: elementIDs[i]})
+	}
+	return
+}
+
+func (ud *uiaDriver) Screenshot() (raw *bytes.Buffer, err error) {
+	// register(getHandler, new CaptureScreenshot("/wd/hub/session/:sessionId/screenshot"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "screenshot"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+
+	var decodeStr []byte
+	if decodeStr, err = base64.StdEncoding.DecodeString(reply.Value); err != nil {
+		return nil, err
+	}
+
+	raw = bytes.NewBuffer(decodeStr)
+	return
+}
+
+func (ud *uiaDriver) Source(srcOpt ...SourceOption) (source string, err error) {
+	// register(getHandler, new Source("/wd/hub/session/:sessionId/source"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "source"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+
+	source = reply.Value
+	return
+}
+
+func (ud *uiaDriver) AccessibleSource() (source string, err error) {
+	// TODO
+	return source, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) HealthCheck() (err error) {
+	// TODO
+	return errDriverNotImplemented
+}
+
+func (ud *uiaDriver) GetAppiumSettings() (settings map[string]interface{}, err error) {
+	// register(getHandler, new GetSettings("/wd/hub/session/:sessionId/appium/settings"))
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "appium/settings"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value map[string]interface{} })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+
+	settings = reply.Value
+	return
+}
+
+func (ud *uiaDriver) SetAppiumSettings(settings map[string]interface{}) (ret map[string]interface{}, err error) {
+	data := map[string]interface{}{
+		"settings": settings,
+	}
+	// register(postHandler, new UpdateSettings("/wd/hub/session/:sessionId/appium/settings"))
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/settings")
+	return
+}
+
+func (ud *uiaDriver) IsHealthy() (healthy bool, err error) {
+	// TODO
+	return healthy, errDriverNotImplemented
+}
+
+func (ud *uiaDriver) WaitWithTimeoutAndInterval(condition Condition, timeout, interval time.Duration) error {
+	startTime := time.Now()
+	for {
+		done, err := condition(ud)
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+
+		if elapsed := time.Since(startTime); elapsed > timeout {
+			return fmt.Errorf("timeout after %v", elapsed)
+		}
+		time.Sleep(interval)
+	}
+}
+
+func (ud *uiaDriver) WaitWithTimeout(condition Condition, timeout time.Duration) error {
+	return ud.WaitWithTimeoutAndInterval(condition, timeout, DefaultWaitInterval)
+}
+
+func (ud *uiaDriver) Wait(condition Condition) error {
+	return ud.WaitWithTimeoutAndInterval(condition, DefaultWaitTimeout, DefaultWaitInterval)
+}
+
+func (ud *uiaDriver) StartCaptureLog(identifier ...string) (err error) {
+	log.Info().Msg("start adb log recording")
+	err = ud.logcat.CatchLogcat()
+	return
+}
+
+func (ud *uiaDriver) StopCaptureLog() (result interface{}, err error) {
+	log.Info().Msg("stop adb log recording")
+	err = ud.logcat.Stop()
+	if err != nil {
+		log.Error().Err(err).Msg("failed to get adb log recording")
+		return "", err
+	}
+	content := ud.logcat.logBuffer.String()
+	return ConvertPoints(content), nil
+}

hrp/pkg/uixt/android_elment.go

@@ -0,0 +1,312 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+var errElementNotImplemented = errors.New("element method not implemented")
+
+type uiaElement struct {
+	parent *uiaDriver
+	id     string
+}
+
+func (ue uiaElement) Click() (err error) {
+	// register(postHandler, new Click("/wd/hub/session/:sessionId/element/:id/click"))
+	_, err = ue.parent.httpPOST(nil, "/session", ue.parent.sessionId, "/element", ue.id, "/click")
+	return
+}
+
+func (ue uiaElement) SendKeys(text string, options ...DataOption) (err error) {
+	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/element/:id/value"))
+	// https://github.com/appium/appium-uiutomator2-server/blob/master/app/src/main/java/io/appium/uiutomator2/handler/SendKeysToElement.java#L76-L85
+	data := map[string]interface{}{
+		"text": text,
+	}
+
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["isReplace"]; !ok {
+		data["isReplace"] = true // default true
+	}
+
+	_, err = ue.parent.httpPOST(data, "/session", ue.parent.sessionId, "/element", ue.id, "/value")
+	return
+}
+
+func (ue uiaElement) Clear() (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) Tap(x, y int) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) TapFloat(x, y float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) DoubleTap() (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) TouchAndHold(second ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) TwoFingerTap() (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) TapWithNumberOfTaps(numberOfTaps, numberOfTouches int) (err error) {
+	// Todo: implement
+	log.Fatal().Msg("not support")
+	return
+}
+
+func (ue uiaElement) ForceTouch(pressure float64, second ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) ForceTouchFloat(x, y, pressure float64, second ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) Drag(fromX, fromY, toX, toY int, steps ...float64) (err error) {
+	return ue.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), steps...)
+}
+
+func (ue uiaElement) DragFloat(fromX, fromY, toX, toY float64, steps ...float64) (err error) {
+	if len(steps) == 0 {
+		steps = []float64{12 * 10}
+	} else {
+		steps[0] = 12 * 10
+	}
+	data := map[string]interface{}{
+		"elementId": ue.id,
+		"endX":      toX,
+		"endY":      toY,
+		"steps":     steps[0],
+	}
+	return ue.parent._drag(data)
+}
+
+func (ue uiaElement) Swipe(fromX, fromY, toX, toY int) error {
+	return ue.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY))
+}
+
+func (ue uiaElement) SwipeFloat(fromX, fromY, toX, toY float64) error {
+	options := []DataOption{
+		WithSteps(12),
+		WithCustomOption("elementId", ue.id),
+	}
+	return ue.parent._swipe(fromX, fromY, toX, toY, options...)
+}
+
+func (ue uiaElement) SwipeDirection(direction Direction, velocity ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) Pinch(scale, velocity float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) PinchToZoomOutByW3CAction(scale ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) Rotate(rotation float64, velocity ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) PickerWheelSelect(order PickerWheelOrder, offset ...int) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) scroll(data interface{}) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) ScrollElementByName(name string) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) ScrollElementByPredicate(predicate string) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) ScrollToVisible() (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) ScrollDirection(direction Direction, distance ...float64) (err error) {
+	// TODO
+	return errElementNotImplemented
+}
+
+func (ue uiaElement) FindElement(by BySelector) (element WebElement, err error) {
+	method, selector := by.getMethodAndSelector()
+	return ue.parent._findElement(method, selector, ue.id)
+}
+
+func (ue uiaElement) FindElements(by BySelector) (elements []WebElement, err error) {
+	method, selector := by.getMethodAndSelector()
+	return ue.parent._findElements(method, selector, ue.id)
+}
+
+func (ue uiaElement) FindVisibleCells() (elements []WebElement, err error) {
+	// TODO
+	return elements, errElementNotImplemented
+}
+
+func (ue uiaElement) Rect() (rect Rect, err error) {
+	// register(getHandler, new GetRect("/wd/hub/session/:sessionId/element/:id/rect"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/rect"); err != nil {
+		return Rect{}, err
+	}
+	reply := new(struct{ Value Rect })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Rect{}, err
+	}
+	rect = reply.Value
+	return
+}
+
+func (ue uiaElement) Location() (point Point, err error) {
+	// register(getHandler, new Location("/wd/hub/session/:sessionId/element/:id/location"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/location"); err != nil {
+		return Point{-1, -1}, err
+	}
+	reply := new(struct{ Value Point })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Point{-1, -1}, err
+	}
+	point = reply.Value
+	return
+}
+
+func (ue uiaElement) Size() (size Size, err error) {
+	// register(getHandler, new GetSize("/wd/hub/session/:sessionId/element/:id/size"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/size"); err != nil {
+		return Size{-1, -1}, err
+	}
+	reply := new(struct{ Value Size })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Size{-1, -1}, err
+	}
+	size = reply.Value
+	return
+}
+
+func (ue uiaElement) Text() (text string, err error) {
+	// register(getHandler, new GetText("/wd/hub/session/:sessionId/element/:id/text"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/text"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+	text = reply.Value
+	return
+}
+
+func (ue uiaElement) Type() (elemType string, err error) {
+	// TODO
+	return elemType, errElementNotImplemented
+}
+
+func (ue uiaElement) IsEnabled() (enabled bool, err error) {
+	// TODO
+	return enabled, errElementNotImplemented
+}
+
+func (ue uiaElement) IsDisplayed() (displayed bool, err error) {
+	// TODO
+	return displayed, errElementNotImplemented
+}
+
+func (ue uiaElement) IsSelected() (selected bool, err error) {
+	// TODO
+	return selected, errElementNotImplemented
+}
+
+func (ue uiaElement) IsAccessible() (accessible bool, err error) {
+	// TODO
+	return accessible, errElementNotImplemented
+}
+
+func (ue uiaElement) IsAccessibilityContainer() (isAccessibilityContainer bool, err error) {
+	// TODO
+	return isAccessibilityContainer, errElementNotImplemented
+}
+
+func (ue uiaElement) GetAttribute(attr ElementAttribute) (value string, err error) {
+	// register(getHandler, new GetElementAttribute("/wd/hub/session/:sessionId/element/:id/attribute/:name"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/attribute", attr.getAttributeName()); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+	value = reply.Value
+	return
+}
+
+func (ue uiaElement) UID() (uid string) {
+	return ue.id
+}
+
+func (ue uiaElement) Screenshot() (raw *bytes.Buffer, err error) {
+	// W3C endpoint
+	// register(getHandler, new GetElementScreenshot("/wd/hub/session/:sessionId/element/:id/screenshot"))
+	// JSONWP endpoint
+	// register(getHandler, new GetElementScreenshot("/wd/hub/session/:sessionId/screenshot/:id"))
+	var rawResp rawResponse
+	if rawResp, err = ue.parent.httpGET("/session", ue.parent.sessionId, "/element", ue.id, "/screenshot"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+
+	var decodeStr []byte
+	if decodeStr, err = base64.StdEncoding.DecodeString(reply.Value); err != nil {
+		return nil, err
+	}
+
+	raw = bytes.NewBuffer(decodeStr)
+	return
+}

hrp/pkg/uixt/android_key.go

@@ -0,0 +1,879 @@
+package uixt
+
+type KeyMeta int
+
+const (
+	KMEmpty     KeyMeta = 0     // As a `null`
+	KMCapLocked KeyMeta = 0x100 // SHIFT key locked in CAPS mode.
+	KMAltLocked KeyMeta = 0x200 // ALT key locked.
+	KMSymLocked KeyMeta = 0x400 // SYM key locked.
+	KMSelecting KeyMeta = 0x800 // Text is in selection mode.
+	// KMAltOn        KeyMeta = 0x02     // This mask is used to check whether one of the ALT meta keys is pressed.
+	// KMAltLeftOn    KeyMeta = 0x10     // This mask is used to check whether the left ALT meta key is pressed.
+	// KMAltRightOn   KeyMeta = 0x20     // This mask is used to check whether the right the ALT meta key is pressed.
+	// KMShiftOn      KeyMeta = 0x1      // This mask is used to check whether one of the SHIFT meta keys is pressed.
+	// KMShiftLeftOn  KeyMeta = 0x40     // This mask is used to check whether the left SHIFT meta key is pressed.
+	// KMShiftRightOn KeyMeta = 0x80     // This mask is used to check whether the right SHIFT meta key is pressed.
+	// KMSymOn        KeyMeta = 0x4      // This mask is used to check whether the SYM meta key is pressed.
+	// KMFunctionOn   KeyMeta = 0x8      // This mask is used to check whether the FUNCTION meta key is pressed.
+	// KMCtrlOn       KeyMeta = 0x1000   // This mask is used to check whether one of the CTRL meta keys is pressed.
+	// KMCtrlLeftOn   KeyMeta = 0x2000   // This mask is used to check whether the left CTRL meta key is pressed.
+	// KMCtrlRightOn  KeyMeta = 0x4000   // This mask is used to check whether the right CTRL meta key is pressed.
+	// KMMetaOn       KeyMeta = 0x10000  // This mask is used to check whether one of the META meta keys is pressed.
+	// KMMetaLeftOn   KeyMeta = 0x20000  // This mask is used to check whether the left META meta key is pressed.
+	// KMMetaRightOn  KeyMeta = 0x40000  // This mask is used to check whether the right META meta key is pressed.
+	// KMCapsLockOn   KeyMeta = 0x100000 // This mask is used to check whether the CAPS LOCK meta key is on.
+	// KMNumLockOn    KeyMeta = 0x200000 // This mask is used to check whether the NUM LOCK meta key is on.
+	// KMScrollLockOn KeyMeta = 0x400000 // This mask is used to check whether the SCROLL LOCK meta key is on.
+	// KMShiftMask            = KMShiftOn | KMShiftLeftOn | KMShiftRightOn
+	// KMAltMask              = KMAltOn | KMAltLeftOn | KMAltRightOn
+	// KMCtrlMask             = KMCtrlOn | KMCtrlLeftOn | KMCtrlRightOn
+	// KMMetaMask             = KMMetaOn | KMMetaLeftOn | KMMetaRightOn
+)
+
+type KeyFlag int
+
+const (
+	// KFWokeHere This mask is set if the device woke because of this key event.
+	// Deprecated
+	KFWokeHere KeyFlag = 0x1
+
+	// KFSoftKeyboard This mask is set if the key event was generated by a software keyboard.
+	KFSoftKeyboard KeyFlag = 0x2
+
+	// KFKeepTouchMode This mask is set if we don't want the key event to cause us to leave touch mode.
+	KFKeepTouchMode KeyFlag = 0x4
+
+	// KFFromSystem This mask is set if an event was known to come from a trusted part
+	// of the system.  That is, the event is known to come from the user,
+	// and could not have been spoofed by a third party component.
+	KFFromSystem KeyFlag = 0x8
+
+	// KFEditorAction This mask is used for compatibility, to identify enter keys that are
+	// coming from an IME whose enter key has been auto-labelled "next" or
+	// "done".  This allows TextView to dispatch these as normal enter keys
+	// for old applications, but still do the appropriate action when receiving them.
+	KFEditorAction KeyFlag = 0x10
+
+	// KFCanceled When associated with up key events, this indicates that the key press
+	// has been canceled.  Typically this is used with virtual touch screen
+	// keys, where the user can slide from the virtual key area on to the
+	// display: in that case, the application will receive a canceled up
+	// event and should not perform the action normally associated with the
+	// key.  Note that for this to work, the application can not perform an
+	// action for a key until it receives an up or the long press timeout has expired.
+	KFCanceled KeyFlag = 0x20
+
+	// KFVirtualHardKey This key event was generated by a virtual (on-screen) hard key area.
+	// Typically this is an area of the touchscreen, outside of the regular
+	// display, dedicated to "hardware" buttons.
+	KFVirtualHardKey KeyFlag = 0x40
+
+	// KFLongPress This flag is set for the first key repeat that occurs after the long press timeout.
+	KFLongPress KeyFlag = 0x80
+
+	// KFCanceledLongPress Set when a key event has `KFCanceled` set because a long
+	// press action was executed while it was down.
+	KFCanceledLongPress KeyFlag = 0x100
+
+	// KFTracking Set for `ACTION_UP` when this event's key code is still being
+	// tracked from its initial down.  That is, somebody requested that tracking
+	// started on the key down and a long press has not caused
+	// the tracking to be canceled.
+	KFTracking KeyFlag = 0x200
+
+	// KFFallback Set when a key event has been synthesized to implement default behavior
+	// for an event that the application did not handle.
+	// Fallback key events are generated by unhandled trackball motions
+	// (to emulate a directional keypad) and by certain unhandled key presses
+	// that are declared in the key map (such as special function numeric keypad
+	// keys when numlock is off).
+	KFFallback KeyFlag = 0x400
+
+	// KFPredispatch Signifies that the key is being predispatched.
+	// KFPredispatch KeyFlag = 0x20000000
+
+	// KFStartTracking Private control to determine when an app is tracking a key sequence.
+	// KFStartTracking KeyFlag = 0x40000000
+
+	// KFTainted Private flag that indicates when the system has detected that this key event
+	// may be inconsistent with respect to the sequence of previously delivered key events,
+	// such as when a key up event is sent but the key was not down.
+	// KFTainted KeyFlag = 0x80000000
+)
+
+type KeyCode int
+
+const (
+	_ KeyCode = 0 // Unknown key code.
+
+	// KCSoftLeft Soft Left key
+	// Usually situated below the display on phones and used as a multi-function
+	// feature key for selecting a software defined function shown on the bottom left
+	// of the display.
+	KCSoftLeft KeyCode = 1
+
+	// KCSoftRight Soft Right key.
+	// Usually situated below the display on phones and used as a multi-function
+	// feature key for selecting a software defined function shown on the bottom right
+	// of the display.
+	KCSoftRight KeyCode = 2
+
+	// KCHome Home key.
+	// This key is handled by the framework and is never delivered to applications.
+	KCHome KeyCode = 3
+
+	KCBack    KeyCode = 4  // Back key
+	KCCall    KeyCode = 5  // Call key
+	KCEndCall KeyCode = 6  // End Call key
+	KC0       KeyCode = 7  // '0' key
+	KC1       KeyCode = 8  // '1' key
+	KC2       KeyCode = 9  // '2' key
+	KC3       KeyCode = 10 // '3' key
+	KC4       KeyCode = 11 // '4' key
+	KC5       KeyCode = 12 // '5' key
+	KC6       KeyCode = 13 // '6' key
+	KC7       KeyCode = 14 // '7' key
+	KC8       KeyCode = 15 // '8' key
+	KC9       KeyCode = 16 // '9' key
+	KCStar    KeyCode = 17 // '*' key
+	KCPound   KeyCode = 18 // '#' key
+
+	// KCDPadUp KeycodeDPadUp Directional Pad Up key.
+	// May also be synthesized from trackball motions.
+	KCDPadUp KeyCode = 19
+
+	// KCDPadDown Directional Pad Down key.
+	// May also be synthesized from trackball motions.
+	KCDPadDown KeyCode = 20
+
+	// KCDPadLeft Directional Pad Left key.
+	// May also be synthesized from trackball motions.
+	KCDPadLeft KeyCode = 21
+
+	// KCDPadRight Directional Pad Right key.
+	// May also be synthesized from trackball motions.
+	KCDPadRight KeyCode = 22
+
+	// KCDPadCenter Directional Pad Center key.
+	// May also be synthesized from trackball motions.
+	KCDPadCenter KeyCode = 23
+
+	// KCVolumeUp Volume Up key.
+	// Adjusts the speaker volume up.
+	KCVolumeUp KeyCode = 24
+
+	// KCVolumeDown Volume Down key.
+	// Adjusts the speaker volume down.
+	KCVolumeDown KeyCode = 25
+
+	// KCPower Power key.
+	KCPower KeyCode = 26
+
+	// KCCamera Camera key.
+	// Used to launch a camera application or take pictures.
+	KCCamera KeyCode = 27
+
+	KCClear      KeyCode = 28 // Clear key
+	KCa          KeyCode = 29 // 'a' key
+	KCb          KeyCode = 30 // 'b' key
+	KCc          KeyCode = 31 // 'c' key
+	KCd          KeyCode = 32 // 'd' key
+	KCe          KeyCode = 33 // 'e' key
+	KCf          KeyCode = 34 // 'f' key
+	KCg          KeyCode = 35 // 'g' key
+	KCh          KeyCode = 36 // 'h' key
+	KCi          KeyCode = 37 // 'i' key
+	KCj          KeyCode = 38 // 'j' key
+	KCk          KeyCode = 39 // 'k' key
+	KCl          KeyCode = 40 // 'l' key
+	KCm          KeyCode = 41 // 'm' key
+	KCn          KeyCode = 42 // 'n' key
+	KCo          KeyCode = 43 // 'o' key
+	KCp          KeyCode = 44 // 'p' key
+	KCq          KeyCode = 45 // 'q' key
+	KCr          KeyCode = 46 // 'r' key
+	KCs          KeyCode = 47 // 's' key
+	KCt          KeyCode = 48 // 't' key
+	KCu          KeyCode = 49 // 'u' key
+	KCv          KeyCode = 50 // 'v' key
+	KCw          KeyCode = 51 // 'w' key
+	KCx          KeyCode = 52 // 'x' key
+	KCy          KeyCode = 53 // 'y' key
+	KCz          KeyCode = 54 // 'z' key
+	KCComma      KeyCode = 55 // ',' key
+	KCPeriod     KeyCode = 56 // '.' key
+	KCAltLeft    KeyCode = 57 // Left Alt modifier key
+	KCAltRight   KeyCode = 58 // Right Alt modifier key
+	KCShiftLeft  KeyCode = 59 // Left Shift modifier key
+	KCShiftRight KeyCode = 60 // Right Shift modifier key
+	KCTab        KeyCode = 61 // Tab key
+	KCSpace      KeyCode = 62 // Space key
+
+	// KCSym Symbol modifier key.
+	// Used to enter alternate symbols.
+	KCSym KeyCode = 63
+
+	// KCExplorer Explorer special function key.
+	// Used to launch a browser application.
+	KCExplorer KeyCode = 64
+
+	// KCEnvelope Envelope special function key.
+	// Used to launch a mail application.
+	KCEnvelope KeyCode = 65
+
+	// KCEnter Enter key.
+	KCEnter KeyCode = 66
+
+	// KCDel Backspace key.
+	// Deletes characters before the insertion point, unlike `KCForwardDel`.
+	KCDel KeyCode = 67
+
+	KCGrave        KeyCode = 68 // '`' (backtick) key
+	KCMinus        KeyCode = 69 // '-'
+	KCEquals       KeyCode = 70 // '=' key
+	KCLeftBracket  KeyCode = 71 // '[' key
+	KCRightBracket KeyCode = 72 // ']' key
+	KCBackslash    KeyCode = 73 // '\' key
+	KCSemicolon    KeyCode = 74 // '' key
+	KCApostrophe   KeyCode = 75 // ''' (apostrophe) key
+	KCSlash        KeyCode = 76 // '/' key
+	KCAt           KeyCode = 77 // '@' key
+
+	// KCNum Number modifier key.
+	// Used to enter numeric symbols.
+	// This key is not Num Lock; it is more like `KCAltLeft` and is
+	// interpreted as an ALT key by {@link android.text.method.MetaKeyKeyListener}.
+	KCNum KeyCode = 78
+
+	// KCHeadsetHook Headset Hook key.
+	// Used to hang up calls and stop media.
+	KCHeadsetHook KeyCode = 79
+
+	// KCFocus Camera Focus key.
+	// Used to focus the camera.
+	// *Camera* focus
+	KCFocus KeyCode = 80
+
+	KCPlus             KeyCode = 81 // '+' key.
+	KCMenu             KeyCode = 82 // Menu key.
+	KCNotification     KeyCode = 83 // Notification key.
+	KCSearch           KeyCode = 84 // Search key.
+	KCMediaPlayPause   KeyCode = 85 // Play/Pause media key.
+	KCMediaStop        KeyCode = 86 // Stop media key.
+	KCMediaNext        KeyCode = 87 // Play Next media key.
+	KCMediaPrevious    KeyCode = 88 // Play Previous media key.
+	KCMediaRewind      KeyCode = 89 // Rewind media key.
+	KCMediaFastForward KeyCode = 90 // Fast Forward media key.
+
+	// KCMute Mute key.
+	// Mutes the microphone, unlike `KCVolumeMute`
+	KCMute KeyCode = 91
+
+	// KCPageUp Page Up key.
+	KCPageUp KeyCode = 92
+
+	// KCPageDown Page Down key.
+	KCPageDown KeyCode = 93
+
+	// KCPictSymbols Picture Symbols modifier key.
+	// Used to switch symbol sets (Emoji, Kao-moji).
+	// switch symbol-sets (Emoji,Kao-moji)
+	KCPictSymbols KeyCode = 94
+
+	// KCSwitchCharset Switch Charset modifier key.
+	// Used to switch character sets (Kanji, Katakana).
+	// switch char-sets (Kanji,Katakana)
+	KCSwitchCharset KeyCode = 95
+
+	// KCButtonA A Button key.
+	// On a game controller, the A button should be either the button labeled A
+	// or the first button on the bottom row of controller buttons.
+	KCButtonA KeyCode = 96
+
+	// KCButtonB B Button key.
+	// On a game controller, the B button should be either the button labeled B
+	// or the second button on the bottom row of controller buttons.
+	KCButtonB KeyCode = 97
+
+	// KCButtonC C Button key.
+	// On a game controller, the C button should be either the button labeled C
+	// or the third button on the bottom row of controller buttons.
+	KCButtonC KeyCode = 98
+
+	// KCButtonX X Button key.
+	// On a game controller, the X button should be either the button labeled X
+	// or the first button on the upper row of controller buttons.
+	KCButtonX KeyCode = 99
+
+	// KCButtonY Y Button key.
+	// On a game controller, the Y button should be either the button labeled Y
+	// or the second button on the upper row of controller buttons.
+	KCButtonY KeyCode = 100
+
+	// KCButtonZ Z Button key.
+	// On a game controller, the Z button should be either the button labeled Z
+	// or the third button on the upper row of controller buttons.
+	KCButtonZ KeyCode = 101
+
+	// KCButtonL1 L1 Button key.
+	// On a game controller, the L1 button should be either the button labeled L1 (or L)
+	// or the top left trigger button.
+	KCButtonL1 KeyCode = 102
+
+	// KCButtonR1 R1 Button key.
+	// On a game controller, the R1 button should be either the button labeled R1 (or R)
+	// or the top right trigger button.
+	KCButtonR1 KeyCode = 103
+
+	// KCButtonL2 L2 Button key.
+	// On a game controller, the L2 button should be either the button labeled L2
+	// or the bottom left trigger button.
+	KCButtonL2 KeyCode = 104
+
+	// KCButtonR2 R2 Button key.
+	// On a game controller, the R2 button should be either the button labeled R2
+	// or the bottom right trigger button.
+	KCButtonR2 KeyCode = 105
+
+	// KCButtonTHUMBL Left Thumb Button key.
+	// On a game controller, the left thumb button indicates that the left (or only)
+	// joystick is pressed.
+	KCButtonTHUMBL KeyCode = 106
+
+	// KCButtonTHUMBR Right Thumb Button key.
+	// On a game controller, the right thumb button indicates that the right
+	// joystick is pressed.
+	KCButtonTHUMBR KeyCode = 107
+
+	// KCButtonStart Start Button key.
+	// On a game controller, the button labeled Start.
+	KCButtonStart KeyCode = 108
+
+	// KCButtonSelect Select Button key.
+	// On a game controller, the button labeled Select.
+	KCButtonSelect KeyCode = 109
+
+	// KCButtonMode Mode Button key.
+	// On a game controller, the button labeled Mode.
+	KCButtonMode KeyCode = 110
+
+	// KCEscape Escape key.
+	KCEscape KeyCode = 111
+
+	// KCForwardDel Forward Delete key.
+	// Deletes characters ahead of the insertion point, unlike `KCDel`.
+	KCForwardDel KeyCode = 112
+
+	KCCtrlLeft   KeyCode = 113 // Left Control modifier key
+	KCCtrlRight  KeyCode = 114 // Right Control modifier key
+	KCCapsLock   KeyCode = 115 // Caps Lock key
+	KCScrollLock KeyCode = 116 // Scroll Lock key
+	KCMetaLeft   KeyCode = 117 // Left Meta modifier key
+	KCMetaRight  KeyCode = 118 // Right Meta modifier key
+	KCFunction   KeyCode = 119 // Function modifier key
+	KCSysRq      KeyCode = 120 // System Request / Print Screen key
+	KCBreak      KeyCode = 121 // Break / Pause key
+
+	// KCMoveHome Home Movement key.
+	// Used for scrolling or moving the cursor around to the start of a line
+	// or to the top of a list.
+	KCMoveHome KeyCode = 122
+
+	// KCMoveEnd End Movement key.
+	// Used for scrolling or moving the cursor around to the end of a line
+	// or to the bottom of a list.
+	KCMoveEnd KeyCode = 123
+
+	// KCInsert Insert key.
+	// Toggles insert / overwrite edit mode.
+	KCInsert KeyCode = 124
+
+	// KCForward Forward key.
+	// Navigates forward in the history stack.  Complement of `KCBack`.
+	KCForward KeyCode = 125
+
+	// KCMediaPlay Play media key.
+	KCMediaPlay KeyCode = 126
+
+	// KCMediaPause Pause media key.
+	KCMediaPause KeyCode = 127
+
+	// KCMediaClose Close media key.
+	// May be used to close a CD tray, for example.
+	KCMediaClose KeyCode = 128
+
+	// KCMediaEject Eject media key.
+	// May be used to eject a CD tray, for example.
+	KCMediaEject KeyCode = 129
+
+	// KCMediaRecord Record media key.
+	KCMediaRecord KeyCode = 130
+
+	KCF1  KeyCode = 131 // F1 key.
+	KCF2  KeyCode = 132 // F2 key.
+	KCF3  KeyCode = 133 // F3 key.
+	KCF4  KeyCode = 134 // F4 key.
+	KCF5  KeyCode = 135 // F5 key.
+	KCF6  KeyCode = 136 // F6 key.
+	KCF7  KeyCode = 137 // F7 key.
+	KCF8  KeyCode = 138 // F8 key.
+	KCF9  KeyCode = 139 // F9 key.
+	KCF10 KeyCode = 140 // F10 key.
+	KCF11 KeyCode = 141 // F11 key.
+	KCF12 KeyCode = 142 // F12 key.
+
+	// KCNumLock Num Lock key.
+	// This is the Num Lock key; it is different from `KCNum`.
+	// This key alters the behavior of other keys on the numeric keypad.
+	KCNumLock KeyCode = 143
+
+	KCNumpad0          KeyCode = 144 // Numeric keypad '0' key
+	KCNumpad1          KeyCode = 145 // Numeric keypad '1' key
+	KCNumpad2          KeyCode = 146 // Numeric keypad '2' key
+	KCNumpad3          KeyCode = 147 // Numeric keypad '3' key
+	KCNumpad4          KeyCode = 148 // Numeric keypad '4' key
+	KCNumpad5          KeyCode = 149 // Numeric keypad '5' key
+	KCNumpad6          KeyCode = 150 // Numeric keypad '6' key
+	KCNumpad7          KeyCode = 151 // Numeric keypad '7' key
+	KCNumpad8          KeyCode = 152 // Numeric keypad '8' key
+	KCNumpad9          KeyCode = 153 // Numeric keypad '9' key
+	KCNumpadDivide     KeyCode = 154 // Numeric keypad '/' key (for division)
+	KCNumpadMultiply   KeyCode = 155 // Numeric keypad '*' key (for multiplication)
+	KCNumpadSubtract   KeyCode = 156 // Numeric keypad '-' key (for subtraction)
+	KCNumpadAdd        KeyCode = 157 // Numeric keypad '+' key (for addition)
+	KCNumpadDot        KeyCode = 158 // Numeric keypad '.' key (for decimals or digit grouping)
+	KCNumpadComma      KeyCode = 159 // Numeric keypad ',' key (for decimals or digit grouping)
+	KCNumpadEnter      KeyCode = 160 // Numeric keypad Enter key
+	KCNumpadEquals     KeyCode = 161 // Numeric keypad 'KeyCode =' key
+	KCNumpadLeftParen  KeyCode = 162 // Numeric keypad '(' key
+	KCNumpadRightParen KeyCode = 163 // Numeric keypad ')' key
+
+	// KCVolumeMute Volume Mute key.
+	// Mutes the speaker, unlike `KCMute`.
+	// This key should normally be implemented as a toggle such that the first press
+	// mutes the speaker and the second press restores the original volume.
+	KCVolumeMute KeyCode = 164
+
+	// KCInfo Info key.
+	// Common on TV remotes to show additional information related to what is
+	// currently being viewed.
+	KCInfo KeyCode = 165
+
+	// KCChannelUp Channel up key.
+	// On TV remotes, increments the television channel.
+	KCChannelUp KeyCode = 166
+
+	// KCChannelDown Channel down key.
+	// On TV remotes, decrements the television channel.
+	KCChannelDown KeyCode = 167
+
+	// KCZoomIn Zoom in key.
+	KCZoomIn KeyCode = 168
+
+	// KCZoomOut Zoom out key.
+	KCZoomOut KeyCode = 169
+
+	// KCTv TV key.
+	// On TV remotes, switches to viewing live TV.
+	KCTv KeyCode = 170
+
+	// KCWindow Window key.
+	// On TV remotes, toggles picture-in-picture mode or other windowing functions.
+	// On Android Wear devices, triggers a display offset.
+	KCWindow KeyCode = 171
+
+	// KCGuide Guide key.
+	// On TV remotes, shows a programming guide.
+	KCGuide KeyCode = 172
+
+	// KCDvr DVR key.
+	// On some TV remotes, switches to a DVR mode for recorded shows.
+	KCDvr KeyCode = 173
+
+	// KCBookmark Bookmark key.
+	// On some TV remotes, bookmarks content or web pages.
+	KCBookmark KeyCode = 174
+
+	// KCCaptions Toggle captions key.
+	// Switches the mode for closed-captioning text, for example during television shows.
+	KCCaptions KeyCode = 175
+
+	// KCSettings Settings key.
+	// Starts the system settings activity.
+	KCSettings KeyCode = 176
+
+	// KCTvPower TV power key.
+	// On TV remotes, toggles the power on a television screen.
+	KCTvPower KeyCode = 177
+
+	// KCTvInput TV input key.
+	// On TV remotes, switches the input on a television screen.
+	KCTvInput KeyCode = 178
+
+	// KCStbPower Set-top-box power key.
+	// On TV remotes, toggles the power on an external Set-top-box.
+	KCStbPower KeyCode = 179
+
+	// KCStbInput Set-top-box input key.
+	// On TV remotes, switches the input mode on an external Set-top-box.
+	KCStbInput KeyCode = 180
+
+	// KCAvrPower A/V Receiver power key.
+	// On TV remotes, toggles the power on an external A/V Receiver.
+	KCAvrPower KeyCode = 181
+
+	// KCAvrInput A/V Receiver input key.
+	// On TV remotes, switches the input mode on an external A/V Receiver.
+	KCAvrInput KeyCode = 182
+
+	// KCProgRed Red "programmable" key.
+	// On TV remotes, acts as a contextual/programmable key.
+	KCProgRed KeyCode = 183
+
+	// KCProgGreen Green "programmable" key.
+	// On TV remotes, actsas a contextual/programmable key.
+	KCProgGreen KeyCode = 184
+
+	// KCProgYellow Yellow "programmable" key.
+	// On TV remotes, acts as a contextual/programmable key.
+	KCProgYellow KeyCode = 185
+
+	// KCProgBlue Blue "programmable" key.
+	// On TV remotes, acts as a contextual/programmable key.
+	KCProgBlue KeyCode = 186
+
+	// KCAppSwitch App switch key.
+	// Should bring up the application switcher dialog.
+	KCAppSwitch KeyCode = 187
+
+	KCButton1  KeyCode = 188 // Generic Game Pad Button #1
+	KCButton2  KeyCode = 189 // Generic Game Pad Button #2
+	KCButton3  KeyCode = 190 // Generic Game Pad Button #3
+	KCButton4  KeyCode = 191 // Generic Game Pad Button #4
+	KCButton5  KeyCode = 192 // Generic Game Pad Button #5
+	KCButton6  KeyCode = 193 // Generic Game Pad Button #6
+	KCButton7  KeyCode = 194 // Generic Game Pad Button #7
+	KCButton8  KeyCode = 195 // Generic Game Pad Button #8
+	KCButton9  KeyCode = 196 // Generic Game Pad Button #9
+	KCButton10 KeyCode = 197 // Generic Game Pad Button #10
+	KCButton11 KeyCode = 198 // Generic Game Pad Button #11
+	KCButton12 KeyCode = 199 // Generic Game Pad Button #12
+	KCButton13 KeyCode = 200 // Generic Game Pad Button #13
+	KCButton14 KeyCode = 201 // Generic Game Pad Button #14
+	KCButton15 KeyCode = 202 // Generic Game Pad Button #15
+	KCButton16 KeyCode = 203 // Generic Game Pad Button #16
+
+	// KCLanguageSwitch Language Switch key.
+	// Toggles the current input language such as switching between English and Japanese on
+	// a QWERTY keyboard.  On some devices, the same function may be performed by
+	// pressing Shift+Spacebar.
+	KCLanguageSwitch KeyCode = 204
+
+	// Manner Mode key.
+	// Toggles silent or vibrate mode on and off to make the device behave more politely
+	// in certain settings such as on a crowded train.  On some devices, the key may only
+	// operate when long-pressed.
+	KCMannerMode KeyCode = 205
+
+	// 3D Mode key.
+	// Toggles the display between 2D and 3D mode.
+	KC3dMode KeyCode = 206
+
+	// Contacts special function key.
+	// Used to launch an address book application.
+	KCContacts KeyCode = 207
+
+	// Calendar special function key.
+	// Used to launch a calendar application.
+	KCCalendar KeyCode = 208
+
+	// Music special function key.
+	// Used to launch a music player application.
+	KCMusic KeyCode = 209
+
+	// Calculator special function key.
+	// Used to launch a calculator application.
+	KCCalculator KeyCode = 210
+
+	// Japanese full-width / half-width key.
+	KCZenkakuHankaku KeyCode = 211
+
+	// Japanese alphanumeric key.
+	KCEisu KeyCode = 212
+
+	// Japanese non-conversion key.
+	KCMuhenkan KeyCode = 213
+
+	// Japanese conversion key.
+	KCHenkan KeyCode = 214
+
+	// Japanese katakana / hiragana key.
+	KCKatakanaHiragana KeyCode = 215
+
+	// Japanese Yen key.
+	KCYen KeyCode = 216
+
+	// Japanese Ro key.
+	KCRo KeyCode = 217
+
+	// Japanese kana key.
+	KCKana KeyCode = 218
+
+	// Assist key.
+	// Launches the global assist activity.  Not delivered to applications.
+	KCAssist KeyCode = 219
+
+	// Brightness Down key.
+	// Adjusts the screen brightness down.
+	KCBrightnessDown KeyCode = 220
+
+	// Brightness Up key.
+	// Adjusts the screen brightness up.
+	KCBrightnessUp KeyCode = 221
+
+	// Audio Track key.
+	// Switches the audio tracks.
+	KCMediaAudioTrack KeyCode = 222
+
+	// Sleep key.
+	// Puts the device to sleep.  Behaves somewhat like {@link #KEYCODE_POWER} but it
+	// has no effect if the device is already asleep.
+	KCSleep KeyCode = 223
+
+	// Wakeup key.
+	// Wakes up the device.  Behaves somewhat like {@link #KEYCODE_POWER} but it
+	// has no effect if the device is already awake.
+	KCWakeup KeyCode = 224
+
+	// Pairing key.
+	// Initiates peripheral pairing mode. Useful for pairing remote control
+	// devices or game controllers, especially if no other input mode is
+	// available.
+	KCPairing KeyCode = 225
+
+	// Media Top Menu key.
+	// Goes to the top of media menu.
+	KCMediaTopMenu KeyCode = 226
+
+	// '11' key.
+	KC11 KeyCode = 227
+
+	// '12' key.
+	KC12 KeyCode = 228
+
+	// Last Channel key.
+	// Goes to the last viewed channel.
+	KCLastChannel KeyCode = 229
+
+	// TV data service key.
+	// Displays data services like weather, sports.
+	KCTvDataService KeyCode = 230
+
+	// Voice Assist key.
+	// Launches the global voice assist activity. Not delivered to applications.
+	KCVoiceAssist KeyCode = 231
+
+	// Radio key.
+	// Toggles TV service / Radio service.
+	KCTvRadioService KeyCode = 232
+
+	// Teletext key.
+	// Displays Teletext service.
+	KCTvTeletext KeyCode = 233
+
+	// Number entry key.
+	// Initiates to enter multi-digit channel nubmber when each digit key is assigned
+	// for selecting separate channel. Corresponds to Number Entry Mode (0x1D) of CEC
+	// User Control Code.
+	KCTvNumberEntry KeyCode = 234
+
+	// Analog Terrestrial key.
+	// Switches to analog terrestrial broadcast service.
+	KCTvTerrestrialAnalog KeyCode = 235
+
+	// Digital Terrestrial key.
+	// Switches to digital terrestrial broadcast service.
+	KCTvTerrestrialDigital KeyCode = 236
+
+	// Satellite key.
+	// Switches to digital satellite broadcast service.
+	KCTvSatellite KeyCode = 237
+
+	// BS key.
+	// Switches to BS digital satellite broadcasting service available in Japan.
+	KCTvSatelliteBs KeyCode = 238
+
+	// CS key.
+	// Switches to CS digital satellite broadcasting service available in Japan.
+	KCTvSatelliteCs KeyCode = 239
+
+	// BS/CS key.
+	// Toggles between BS and CS digital satellite services.
+	KCTvSatelliteService KeyCode = 240
+
+	// Toggle Network key.
+	// Toggles selecting broacast services.
+	KCTvNetwork KeyCode = 241
+
+	// Antenna/Cable key.
+	// Toggles broadcast input source between antenna and cable.
+	KCTvAntennaCable KeyCode = 242
+
+	// HDMI #1 key.
+	// Switches to HDMI input #1.
+	KCTvInputHdmi1 KeyCode = 243
+
+	// HDMI #2 key.
+	// Switches to HDMI input #2.
+	KCTvInputHdmi2 KeyCode = 244
+
+	// HDMI #3 key.
+	// Switches to HDMI input #3.
+	KCTvInputHdmi3 KeyCode = 245
+
+	// HDMI #4 key.
+	// Switches to HDMI input #4.
+	KCTvInputHdmi4 KeyCode = 246
+
+	// Composite #1 key.
+	// Switches to composite video input #1.
+	KCTvInputComposite1 KeyCode = 247
+
+	// Composite #2 key.
+	// Switches to composite video input #2.
+	KCTvInputComposite2 KeyCode = 248
+
+	// Component #1 key.
+	// Switches to component video input #1.
+	KCTvInputComponent1 KeyCode = 249
+
+	// Component #2 key.
+	// Switches to component video input #2.
+	KCTvInputComponent2 KeyCode = 250
+
+	// VGA #1 key.
+	// Switches to VGA (analog RGB) input #1.
+	KCTvInputVga1 KeyCode = 251
+
+	// Audio description key.
+	// Toggles audio description off / on.
+	KCTvAudioDescription KeyCode = 252
+
+	// Audio description mixing volume up key.
+	// Louden audio description volume as compared with normal audio volume.
+	KCTvAudioDescriptionMixUp KeyCode = 253
+
+	// Audio description mixing volume down key.
+	// Lessen audio description volume as compared with normal audio volume.
+	KCTvAudioDescriptionMixDown KeyCode = 254
+
+	// Zoom mode key.
+	// Changes Zoom mode (Normal, Full, Zoom, Wide-zoom, etc.)
+	KCTvZoomMode KeyCode = 255
+
+	// Contents menu key.
+	// Goes to the title list. Corresponds to Contents Menu (0x0B) of CEC User Control
+	// Code
+	KCTvContentsMenu KeyCode = 256
+
+	// Media context menu key.
+	// Goes to the context menu of media contents. Corresponds to Media Context-sensitive
+	// Menu (0x11) of CEC User Control Code.
+	KCTvMediaContextMenu KeyCode = 257
+
+	// Timer programming key.
+	// Goes to the timer recording menu. Corresponds to Timer Programming (0x54) of
+	// CEC User Control Code.
+	KCTvTimerProgramming KeyCode = 258
+
+	// Help key.
+	KCHelp KeyCode = 259
+
+	// Navigate to previous key.
+	// Goes backward by one item in an ordered collection of items.
+	KCNavigatePrevious KeyCode = 260
+
+	// Navigate to next key.
+	// Advances to the next item in an ordered collection of items.
+	KCNavigateNext KeyCode = 261
+
+	// Navigate in key.
+	// Activates the item that currently has focus or expands to the next level of a navigation
+	// hierarchy.
+	KCNavigateIn KeyCode = 262
+
+	// Navigate out key.
+	// Backs out one level of a navigation hierarchy or collapses the item that currently has
+	// focus.
+	KCNavigateOut KeyCode = 263
+
+	// Primary stem key for Wear
+	// Main power/reset button on watch.
+	KCStemPrimary KeyCode = 264
+
+	// Generic stem key 1 for Wear
+	KCStem1 KeyCode = 265
+
+	// Generic stem key 2 for Wear
+	KCStem2 KeyCode = 266
+
+	// Generic stem key 3 for Wear
+	KCStem3 KeyCode = 267
+
+	// Directional Pad Up-Left
+	KCDPadUpLeft KeyCode = 268
+
+	// Directional Pad Down-Left
+	KCDPadDownLeft KeyCode = 269
+
+	// Directional Pad Up-Right
+	KCDPadUpRight KeyCode = 270
+
+	// Directional Pad Down-Right
+	KCDPadDownRight KeyCode = 271
+
+	// Skip forward media key.
+	KCMediaSkipForward KeyCode = 272
+
+	// Skip backward media key.
+	KCMediaSkipBackward KeyCode = 273
+
+	// Step forward media key.
+	// Steps media forward, one frame at a time.
+	KCMediaStepForward KeyCode = 274
+
+	// Step backward media key.
+	// Steps media backward, one frame at a time.
+	KCMediaStepBackward KeyCode = 275
+
+	// put device to sleep unless a wakelock is held.
+	KCSoftSleep KeyCode = 276
+
+	// Cut key.
+	KCCut KeyCode = 277
+
+	// Copy key.
+	KCCopy KeyCode = 278
+
+	// Paste key.
+	KCPaste KeyCode = 279
+
+	// Consumed by the system for navigation up
+	KCSystemNavigationUp KeyCode = 280
+
+	// Consumed by the system for navigation down
+	KCSystemNavigationDown KeyCode = 281
+
+	// Consumed by the system for navigation left*/
+	KCSystemNavigationLeft KeyCode = 282
+
+	// Consumed by the system for navigation right
+	KCSystemNavigationRight KeyCode = 283
+
+	// Show all apps
+	KCAllApps KeyCode = 284
+
+	// Refresh key.
+	KCRefresh KeyCode = 285
+)

hrp/pkg/uixt/client.go

@@ -0,0 +1,104 @@
+package uixt
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+type Driver struct {
+	urlPrefix *url.URL
+	sessionId string
+	client    *http.Client
+}
+
+func (wd *Driver) concatURL(u *url.URL, elem ...string) string {
+	var tmp *url.URL
+	if u == nil {
+		u = wd.urlPrefix
+	}
+	tmp, _ = url.Parse(u.String())
+	tmp.Path = path.Join(append([]string{u.Path}, elem...)...)
+	return tmp.String()
+}
+
+func (wd *Driver) httpGET(pathElem ...string) (rawResp rawResponse, err error) {
+	return wd.httpRequest(http.MethodGet, wd.concatURL(nil, pathElem...), nil)
+}
+
+func (wd *Driver) httpPOST(data interface{}, pathElem ...string) (rawResp rawResponse, err error) {
+	var bsJSON []byte = nil
+	if data != nil {
+		if bsJSON, err = json.Marshal(data); err != nil {
+			return nil, err
+		}
+	}
+	return wd.httpRequest(http.MethodPost, wd.concatURL(nil, pathElem...), bsJSON)
+}
+
+func (wd *Driver) httpDELETE(pathElem ...string) (rawResp rawResponse, err error) {
+	return wd.httpRequest(http.MethodDelete, wd.concatURL(nil, pathElem...), nil)
+}
+
+func (wd *Driver) httpRequest(method string, rawURL string, rawBody []byte) (rawResp rawResponse, err error) {
+	log.Debug().Str("method", method).Str("url", rawURL).Str("body", string(rawBody)).Msg("request driver agent")
+
+	var req *http.Request
+	if req, err = http.NewRequest(method, rawURL, bytes.NewBuffer(rawBody)); err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json;charset=UTF-8")
+	req.Header.Set("Accept", "application/json")
+
+	start := time.Now()
+	var resp *http.Response
+	if resp, err = wd.client.Do(req); err != nil {
+		return nil, err
+	}
+	defer func() {
+		// https://github.com/etcd-io/etcd/blob/v3.3.25/pkg/httputil/httputil.go#L16-L22
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
+		_ = resp.Body.Close()
+	}()
+
+	rawResp, err = ioutil.ReadAll(resp.Body)
+	logger := log.Debug().Int("statusCode", resp.StatusCode).Str("duration", time.Since(start).String())
+	if !strings.HasSuffix(rawURL, "screenshot") {
+		// avoid printing screenshot data
+		logger.Str("response", string(rawResp))
+	}
+	logger.Msg("get driver agent response")
+	if err != nil {
+		return nil, err
+	}
+
+	if err = rawResp.checkErr(); err != nil {
+		if resp.StatusCode == http.StatusOK {
+			return rawResp, nil
+		}
+		return nil, err
+	}
+
+	return
+}
+
+func convertToHTTPClient(conn net.Conn) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return conn, nil
+			},
+		},
+		Timeout: 0,
+	}
+}

hrp/pkg/uixt/drag.go

@@ -0,0 +1,30 @@
+package uixt
+
+func (dExt *DriverExt) Drag(pathname string, toX, toY int, pressForDuration ...float64) (err error) {
+	return dExt.DragFloat(pathname, float64(toX), float64(toY), pressForDuration...)
+}
+
+func (dExt *DriverExt) DragFloat(pathname string, toX, toY float64, pressForDuration ...float64) (err error) {
+	return dExt.DragOffsetFloat(pathname, toX, toY, 0.5, 0.5, pressForDuration...)
+}
+
+func (dExt *DriverExt) DragOffset(pathname string, toX, toY int, xOffset, yOffset float64, pressForDuration ...float64) (err error) {
+	return dExt.DragOffsetFloat(pathname, float64(toX), float64(toY), xOffset, yOffset, pressForDuration...)
+}
+
+func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffset float64, pressForDuration ...float64) (err error) {
+	if len(pressForDuration) == 0 {
+		pressForDuration = []float64{1.0}
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	fromX := x + width*xOffset
+	fromY := y + height*yOffset
+
+	return dExt.Driver.DragFloat(fromX, fromY, toX, toY,
+		WithPressDuration(pressForDuration[0]))
+}

hrp/pkg/uixt/drag_test.go

@@ -0,0 +1,20 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"testing"
+)
+
+func TestDriverExt_Drag(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_map.png"
+
+	// err = driverExt.Drag(pathSearch, 300, 500, 2)
+	// checkErr(t, err)
+
+	err = driverExt.DragOffset(pathSearch, 300, 500, 2.1, 0.5, 2)
+	checkErr(t, err)
+}

hrp/pkg/uixt/ext.go

@@ -0,0 +1,593 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"image/png"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+type MobileMethod string
+
+const (
+	AppInstall          MobileMethod = "install"
+	AppUninstall        MobileMethod = "uninstall"
+	AppStart            MobileMethod = "app_start"
+	AppLaunch           MobileMethod = "app_launch"            // 等待 app 打开并堵塞到 app 首屏加载完成，可以传入 app 的启动参数、环境变量
+	AppLaunchUnattached MobileMethod = "app_launch_unattached" // 只负责通知打开 app，不堵塞等待，不可传入启动参数
+	AppTerminate        MobileMethod = "app_terminate"
+	AppStop             MobileMethod = "app_stop"
+	CtlScreenShot       MobileMethod = "screenshot"
+	CtlSleep            MobileMethod = "sleep"
+	CtlStartCamera      MobileMethod = "camera_start" // alias for app_launch camera
+	CtlStopCamera       MobileMethod = "camera_stop"  // alias for app_terminate camera
+	RecordStart         MobileMethod = "record_start"
+	RecordStop          MobileMethod = "record_stop"
+
+	// UI validation
+	SelectorName       string = "ui_name"
+	SelectorLabel      string = "ui_label"
+	SelectorOCR        string = "ui_ocr"
+	SelectorImage      string = "ui_image"
+	AssertionExists    string = "exists"
+	AssertionNotExists string = "not_exists"
+
+	// UI handling
+	ACTION_Home        MobileMethod = "home"
+	ACTION_TapXY       MobileMethod = "tap_xy"
+	ACTION_TapAbsXY    MobileMethod = "tap_abs_xy"
+	ACTION_TapByOCR    MobileMethod = "tap_ocr"
+	ACTION_TapByCV     MobileMethod = "tap_cv"
+	ACTION_Tap         MobileMethod = "tap"
+	ACTION_DoubleTapXY MobileMethod = "double_tap_xy"
+	ACTION_DoubleTap   MobileMethod = "double_tap"
+	ACTION_Swipe       MobileMethod = "swipe"
+	ACTION_Input       MobileMethod = "input"
+
+	// custom actions
+	ACTION_SwipeToTapApp  MobileMethod = "swipe_to_tap_app"  // swipe left & right to find app and tap
+	ACTION_SwipeToTapText MobileMethod = "swipe_to_tap_text" // swipe up & down to find text and tap
+)
+
+type MobileAction struct {
+	Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"`
+	Params interface{}  `json:"params,omitempty" yaml:"params,omitempty"`
+
+	Identifier          string `json:"identifier,omitempty" yaml:"identifier,omitempty"`                     // used to identify the action in log
+	MaxRetryTimes       int    `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"`           // max retry times
+	Index               int    `json:"index,omitempty" yaml:"index,omitempty"`                               // index of the target element, should start from 1
+	Timeout             int    `json:"timeout,omitempty" yaml:"timeout,omitempty"`                           // TODO: wait timeout in seconds for mobile action
+	IgnoreNotFoundError bool   `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
+	Text                string `json:"text,omitempty" yaml:"text,omitempty"`
+	ID                  string `json:"id,omitempty" yaml:"id,omitempty"`
+	Description         string `json:"description,omitempty" yaml:"description,omitempty"`
+}
+
+type ActionOption func(o *MobileAction)
+
+func WithIdentifier(identifier string) ActionOption {
+	return func(o *MobileAction) {
+		o.Identifier = identifier
+	}
+}
+
+func WithIndex(index int) ActionOption {
+	return func(o *MobileAction) {
+		o.Index = index
+	}
+}
+
+func WithText(text string) ActionOption {
+	return func(o *MobileAction) {
+		o.Text = text
+	}
+}
+
+func WithID(id string) ActionOption {
+	return func(o *MobileAction) {
+		o.ID = id
+	}
+}
+
+func WithDescription(description string) ActionOption {
+	return func(o *MobileAction) {
+		o.Description = description
+	}
+}
+
+func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
+	return func(o *MobileAction) {
+		o.MaxRetryTimes = maxRetryTimes
+	}
+}
+
+func WithTimeout(timeout int) ActionOption {
+	return func(o *MobileAction) {
+		o.Timeout = timeout
+	}
+}
+
+func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
+	return func(o *MobileAction) {
+		o.IgnoreNotFoundError = ignoreError
+	}
+}
+
+// TemplateMatchMode is the type of the template matching operation.
+type TemplateMatchMode int
+
+type CVArgs struct {
+	matchMode TemplateMatchMode
+	threshold float64
+}
+
+type CVOption func(*CVArgs)
+
+func WithTemplateMatchMode(mode TemplateMatchMode) CVOption {
+	return func(args *CVArgs) {
+		args.matchMode = mode
+	}
+}
+
+func WithThreshold(threshold float64) CVOption {
+	return func(args *CVArgs) {
+		args.threshold = threshold
+	}
+}
+
+type DriverExt struct {
+	UUID            string // ios udid or android serial
+	Driver          WebDriver
+	windowSize      Size
+	frame           *bytes.Buffer
+	doneMjpegStream chan bool
+	scale           float64
+	StartTime       time.Time     // used to associate screenshots name
+	ScreenShots     []string      // save screenshots path
+	perfStop        chan struct{} // stop performance monitor
+	perfData        []string      // save perf data
+
+	CVArgs
+}
+
+func extend(driver WebDriver) (dExt *DriverExt, err error) {
+	dExt = &DriverExt{Driver: driver}
+	dExt.doneMjpegStream = make(chan bool, 1)
+
+	// get device window size
+	dExt.windowSize, err = dExt.Driver.WindowSize()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get windows size")
+	}
+
+	if dExt.scale, err = dExt.Driver.Scale(); err != nil {
+		return nil, err
+	}
+
+	return dExt, nil
+}
+
+func (dExt *DriverExt) GetPerfData() []string {
+	if dExt.perfStop == nil {
+		return nil
+	}
+	close(dExt.perfStop)
+	return dExt.perfData
+}
+
+func (dExt *DriverExt) takeScreenShot() (raw *bytes.Buffer, err error) {
+	// wait for action done
+	time.Sleep(500 * time.Millisecond)
+
+	// 优先使用 MJPEG 流进行截图，性能最优
+	// 如果 MJPEG 流未开启，则使用 WebDriver 的截图接口
+	if dExt.frame != nil {
+		return dExt.frame, nil
+	}
+	if raw, err = dExt.Driver.Screenshot(); err != nil {
+		log.Error().Err(err).Msg("takeScreenShot failed")
+		return nil, err
+	}
+	return raw, nil
+}
+
+// saveScreenShot saves image file to $CWD/screenshots/ folder
+func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) {
+	img, format, err := image.Decode(raw)
+	if err != nil {
+		return "", errors.Wrap(err, "decode screenshot image failed")
+	}
+
+	dir, _ := os.Getwd()
+	screenshotsDir := filepath.Join(dir, "screenshots")
+	if err = os.MkdirAll(screenshotsDir, os.ModePerm); err != nil {
+		return "", errors.Wrap(err, "create screenshots directory failed")
+	}
+	screenshotPath := filepath.Join(screenshotsDir,
+		fmt.Sprintf("%s.%s", fileName, format))
+
+	file, err := os.Create(screenshotPath)
+	if err != nil {
+		return "", errors.Wrap(err, "create screenshot image file failed")
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+
+	switch format {
+	case "png":
+		err = png.Encode(file, img)
+	case "jpeg":
+		err = jpeg.Encode(file, img, nil)
+	default:
+		return "", fmt.Errorf("unsupported image format: %s", format)
+	}
+	if err != nil {
+		return "", errors.Wrap(err, "encode screenshot image failed")
+	}
+
+	return screenshotPath, nil
+}
+
+// ScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
+func (dExt *DriverExt) ScreenShot(fileName string) (string, error) {
+	raw, err := dExt.takeScreenShot()
+	if err != nil {
+		return "", errors.Wrap(err, "screenshot by WDA failed")
+	}
+
+	path, err := dExt.saveScreenShot(raw, fileName)
+	if err != nil {
+		return "", errors.Wrap(err, "save screenshot failed")
+	}
+	return path, nil
+}
+
+// isPathExists returns true if path exists, whether path is file or dir
+func isPathExists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
+	var selector BySelector
+	if strings.HasPrefix(param, "/") {
+		// xpath
+		selector = BySelector{
+			XPath: param,
+		}
+	} else if strings.HasPrefix(param, "com.") {
+		// name
+		selector = BySelector{
+			ResourceIdID: param,
+		}
+	} else {
+		// name
+		selector = BySelector{
+			LinkText: NewElementAttribute().WithName(param),
+		}
+	}
+
+	return dExt.Driver.FindElement(selector)
+}
+
+func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
+	// click on text, using OCR
+	if !isPathExists(search) {
+		return dExt.FindTextByOCR(search, index...)
+	}
+	// click on image, using opencv
+	return dExt.FindImageRectInUIKit(search, index...)
+}
+
+func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
+	x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale
+	width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale
+	return
+}
+
+func (dExt *DriverExt) PerformTouchActions(touchActions *TouchActions) error {
+	return dExt.Driver.PerformAppiumTouchActions(touchActions)
+}
+
+func (dExt *DriverExt) PerformActions(actions *W3CActions) error {
+	return dExt.Driver.PerformW3CActions(actions)
+}
+
+func (dExt *DriverExt) IsNameExist(name string) bool {
+	selector := BySelector{
+		LinkText: NewElementAttribute().WithName(name),
+	}
+	_, err := dExt.Driver.FindElement(selector)
+	return err == nil
+}
+
+func (dExt *DriverExt) IsLabelExist(label string) bool {
+	selector := BySelector{
+		LinkText: NewElementAttribute().WithLabel(label),
+	}
+	_, err := dExt.Driver.FindElement(selector)
+	return err == nil
+}
+
+func (dExt *DriverExt) IsOCRExist(text string) bool {
+	_, _, _, _, err := dExt.FindTextByOCR(text)
+	return err == nil
+}
+
+func (dExt *DriverExt) IsImageExist(text string) bool {
+	_, _, _, _, err := dExt.FindImageRectInUIKit(text)
+	return err == nil
+}
+
+var errActionNotImplemented = errors.New("UI action not implemented")
+
+func (dExt *DriverExt) DoAction(action MobileAction) error {
+	log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action")
+
+	switch action.Method {
+	case AppInstall:
+		// TODO
+		return errActionNotImplemented
+	case AppLaunch:
+		if bundleId, ok := action.Params.(string); ok {
+			return dExt.Driver.AppLaunch(bundleId)
+		}
+		return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
+			AppLaunch, action.Params)
+	case AppLaunchUnattached:
+		if bundleId, ok := action.Params.(string); ok {
+			return dExt.Driver.AppLaunchUnattached(bundleId)
+		}
+		return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
+			AppLaunchUnattached, action.Params)
+	case ACTION_SwipeToTapApp:
+		if appName, ok := action.Params.(string); ok {
+			var point PointF
+			findApp := func(d *DriverExt) error {
+				var err error
+				point, err = d.GetTextXY(appName, action.Index)
+				return err
+			}
+			foundAppAction := func(d *DriverExt) error {
+				// click app to launch
+				return d.TapAbsXY(point.X, point.Y-20, action.Identifier)
+			}
+
+			// go to home screen
+			if err := dExt.Driver.Homescreen(); err != nil {
+				return errors.Wrap(err, "go to home screen failed")
+			}
+
+			// swipe to first screen
+			for i := 0; i < 5; i++ {
+				dExt.SwipeRight()
+			}
+
+			// default to retry 5 times
+			if action.MaxRetryTimes == 0 {
+				action.MaxRetryTimes = 5
+			}
+			// swipe next screen until app found
+			return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes)
+		}
+		return fmt.Errorf("invalid %s params, should be app name(string), got %v",
+			ACTION_SwipeToTapApp, action.Params)
+	case ACTION_SwipeToTapText:
+		if text, ok := action.Params.(string); ok {
+			var point PointF
+			findText := func(d *DriverExt) error {
+				var err error
+				point, err = d.GetTextXY(text, action.Index)
+				return err
+			}
+			foundTextAction := func(d *DriverExt) error {
+				// tap text
+				return d.TapAbsXY(point.X, point.Y, action.Identifier)
+			}
+
+			// default to retry 10 times
+			if action.MaxRetryTimes == 0 {
+				action.MaxRetryTimes = 10
+			}
+			// swipe until live room found
+			return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
+		}
+		return fmt.Errorf("invalid %s params, should be app text(string), got %v",
+			ACTION_SwipeToTapText, action.Params)
+	case AppTerminate:
+		if bundleId, ok := action.Params.(string); ok {
+			success, err := dExt.Driver.AppTerminate(bundleId)
+			if err != nil {
+				return errors.Wrap(err, "failed to terminate app")
+			}
+			if !success {
+				log.Warn().Str("bundleId", bundleId).Msg("app was not running")
+			}
+			return nil
+		}
+		return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params)
+	case ACTION_Home:
+		return dExt.Driver.Homescreen()
+	case ACTION_TapXY:
+		if location, ok := action.Params.([]interface{}); ok {
+			// relative x,y of window size: [0.5, 0.5]
+			if len(location) != 2 {
+				return fmt.Errorf("invalid tap location params: %v", location)
+			}
+			x, _ := location[0].(float64)
+			y, _ := location[1].(float64)
+			return dExt.TapXY(x, y, action.Identifier)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
+	case ACTION_TapAbsXY:
+		if location, ok := action.Params.([]interface{}); ok {
+			// absolute coordinates x,y of window size: [100, 300]
+			if len(location) != 2 {
+				return fmt.Errorf("invalid tap location params: %v", location)
+			}
+			x, _ := location[0].(float64)
+			y, _ := location[1].(float64)
+			return dExt.TapAbsXY(x, y, action.Identifier)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
+	case ACTION_Tap:
+		if param, ok := action.Params.(string); ok {
+			return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
+	case ACTION_TapByOCR:
+		if ocrText, ok := action.Params.(string); ok {
+			return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
+	case ACTION_TapByCV:
+		if imagePath, ok := action.Params.(string); ok {
+			return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
+	case ACTION_DoubleTapXY:
+		if location, ok := action.Params.([]interface{}); ok {
+			// relative x,y of window size: [0.5, 0.5]
+			if len(location) != 2 {
+				return fmt.Errorf("invalid tap location params: %v", location)
+			}
+			x, _ := location[0].(float64)
+			y, _ := location[1].(float64)
+			return dExt.DoubleTapXY(x, y)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params)
+	case ACTION_DoubleTap:
+		if param, ok := action.Params.(string); ok {
+			return dExt.DoubleTap(param)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
+	case ACTION_Swipe:
+		if positions, ok := action.Params.([]interface{}); ok {
+			// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
+			if len(positions) != 4 {
+				return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions)
+			}
+			fromX, _ := positions[0].(float64)
+			fromY, _ := positions[1].(float64)
+			toX, _ := positions[2].(float64)
+			toY, _ := positions[3].(float64)
+			return dExt.SwipeRelative(fromX, fromY, toX, toY, action.Identifier)
+		}
+		if direction, ok := action.Params.(string); ok {
+			return dExt.SwipeTo(direction, action.Identifier)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
+	case ACTION_Input:
+		// input text on current active element
+		// append \n to send text with enter
+		// send \b\b\b to delete 3 chars
+		param := fmt.Sprintf("%v", action.Params)
+		options := []DataOption{}
+		if action.Text != "" {
+			options = append(options, WithCustomOption("text", action.Text))
+		}
+		if action.ID != "" {
+			options = append(options, WithCustomOption("id", action.ID))
+		}
+		if action.Description != "" {
+			options = append(options, WithCustomOption("description", action.Description))
+		}
+		if action.Identifier != "" {
+			options = append(options, WithCustomOption("log", map[string]interface{}{
+				"enable": true,
+				"data":   action.Identifier,
+			}))
+		}
+		return dExt.Driver.Input(param, options...)
+	case CtlSleep:
+		if param, ok := action.Params.(json.Number); ok {
+			seconds, _ := param.Float64()
+			time.Sleep(time.Duration(seconds*1000) * time.Millisecond)
+			return nil
+		} else if param, ok := action.Params.(float64); ok {
+			time.Sleep(time.Duration(param*1000) * time.Millisecond)
+			return nil
+		} else if param, ok := action.Params.(int64); ok {
+			time.Sleep(time.Duration(param) * time.Second)
+			return nil
+		}
+		return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
+	case CtlScreenShot:
+		// take snapshot
+		log.Info().Msg("take snapshot for current screen")
+		screenshotPath, err := dExt.ScreenShot(fmt.Sprintf("%d_screenshot_%d",
+			dExt.StartTime.Unix(), time.Now().Unix()))
+		if err != nil {
+			return errors.Wrap(err, "take screenshot failed")
+		}
+		log.Info().Str("path", screenshotPath).Msg("take screenshot")
+		dExt.ScreenShots = append(dExt.ScreenShots, screenshotPath)
+		return err
+	case CtlStartCamera:
+		return dExt.Driver.StartCamera()
+	case CtlStopCamera:
+		return dExt.Driver.StopCamera()
+	}
+	return nil
+}
+
+func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool {
+	var exists bool
+	if assert == AssertionExists {
+		exists = true
+	} else {
+		exists = false
+	}
+	var result bool
+	switch check {
+	case SelectorName:
+		result = (dExt.IsNameExist(expected) == exists)
+	case SelectorLabel:
+		result = (dExt.IsLabelExist(expected) == exists)
+	case SelectorOCR:
+		result = (dExt.IsOCRExist(expected) == exists)
+	case SelectorImage:
+		result = (dExt.IsImageExist(expected) == exists)
+	}
+
+	if !result {
+		if message == nil {
+			message = []string{""}
+		}
+		log.Error().
+			Str("assert", assert).
+			Str("expect", expected).
+			Str("msg", message[0]).
+			Msg("validate UI failed")
+		return false
+	}
+
+	log.Info().
+		Str("assert", assert).
+		Str("expect", expected).
+		Msg("validate UI success")
+	return true
+}
+
+func checkErr(t *testing.T, err error, msg ...string) {
+	if err != nil {
+		if len(msg) == 0 {
+			t.Fatal(err)
+		} else {
+			t.Fatal(msg, err)
+		}
+	}
+}

hrp/pkg/uixt/gesture.go

@@ -0,0 +1,44 @@
+//go:build opencv
+
+package uixt
+
+import (
+	"image"
+	"sort"
+)
+
+func (dExt *DriverExt) GesturePassword(pathname string, password ...int) (err error) {
+	var rects []image.Rectangle
+	if rects, err = dExt.FindAllImageRect(pathname); err != nil {
+		return err
+	}
+
+	sort.Slice(rects, func(i, j int) bool {
+		if rects[i].Min.Y < rects[j].Min.Y {
+			return true
+		} else if rects[i].Min.Y == rects[j].Min.Y {
+			if rects[i].Min.X < rects[j].Min.X {
+				return true
+			}
+		}
+		return false
+	})
+
+	touchActions := NewTouchActions(len(password)*2 + 1)
+	for i := range password {
+		x, y, width, height := dExt.MappingToRectInUIKit(rects[password[i]])
+		x = x + width*0.5
+		y = y + height*0.5
+
+		if i == 0 {
+			touchActions.Press(NewTouchActionPress().WithXYFloat(x, y)).
+				Wait(0.2)
+		} else {
+			touchActions.MoveTo(NewTouchActionMoveTo().WithXYFloat(x, y)).
+				Wait(0.2)
+		}
+	}
+	touchActions.Release()
+
+	return dExt.PerformTouchActions(touchActions)
+}

hrp/pkg/uixt/gesture_test.go

@@ -0,0 +1,25 @@
+//go:build opencv
+
+package uixt
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestDriverExt_GesturePassword(t *testing.T) {
+	split := strings.Split("6304258", "")
+	password := make([]int, len(split))
+	for i := range split {
+		password[i], _ = strconv.Atoi(split[i])
+	}
+
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
+
+	err = driverExt.GesturePassword(pathSearch, password...)
+	checkErr(t, err)
+}

hrp/pkg/uixt/ios_action.go

@@ -0,0 +1,373 @@
+package uixt
+
+import (
+	"strconv"
+	"strings"
+)
+
+type W3CActions []map[string]interface{}
+
+func NewW3CActions(capacity ...int) *W3CActions {
+	if len(capacity) == 0 || capacity[0] <= 0 {
+		capacity = []int{8}
+	}
+	tmp := make(W3CActions, 0, capacity[0])
+	return &tmp
+}
+
+func (act *W3CActions) SendKeys(text string) *W3CActions {
+	keyboard := make(map[string]interface{})
+	keyboard["type"] = "key"
+	keyboard["id"] = "keyboard" + strconv.FormatInt(int64(len(*act)+1), 10)
+
+	ss := strings.Split(text, "")
+	type KeyEvent struct {
+		Type  string `json:"type"`
+		Value string `json:"value"`
+	}
+	actOptKey := make([]KeyEvent, 0, len(ss)+1)
+	for i := range ss {
+		actOptKey = append(
+			actOptKey,
+			KeyEvent{Type: "keyDown", Value: ss[i]},
+			KeyEvent{Type: "keyUp", Value: ss[i]},
+		)
+	}
+	keyboard["actions"] = actOptKey
+	*act = append(*act, keyboard)
+	return act
+}
+
+func (act *W3CActions) _newFinger() map[string]interface{} {
+	pointer := make(map[string]interface{})
+	pointer["type"] = "pointer"
+	pointer["id"] = "finger" + strconv.FormatInt(int64(len(*act)+1), 10)
+	pointer["parameters"] = map[string]string{"pointerType": "touch"}
+	return pointer
+}
+
+func (act *W3CActions) FingerAction(fingerAct *FingerAction, fActs ...*FingerAction) *W3CActions {
+	fActs = append([]*FingerAction{fingerAct}, fActs...)
+	for i := range fActs {
+		pointer := act._newFinger()
+		pointer["actions"] = *fActs[i]
+		*act = append(*act, pointer)
+	}
+	return act
+}
+
+type FingerAction []map[string]interface{}
+
+func NewFingerAction(capacity ...int) *FingerAction {
+	if len(capacity) == 0 || capacity[0] <= 0 {
+		capacity = []int{8}
+	}
+	tmp := make(FingerAction, 0, capacity[0])
+	return &tmp
+}
+
+type FingerMove map[string]interface{}
+
+func NewFingerMove() FingerMove {
+	return map[string]interface{}{"type": "pointerMove"}
+}
+
+func (fm FingerMove) WithXY(x, y int) FingerMove {
+	fm["x"] = x
+	fm["y"] = y
+	return fm
+}
+
+func (fm FingerMove) WithXYFloat(x, y float64) FingerMove {
+	fm["x"] = x
+	fm["y"] = y
+	return fm
+}
+
+func (fm FingerMove) WithOrigin(element WebElement) FingerMove {
+	fm["origin"] = element.UID()
+	return fm
+}
+
+func (fm FingerMove) WithDuration(second float64) FingerMove {
+	fm["duration"] = second
+	return fm
+}
+
+func (fa *FingerAction) Move(fm FingerMove) *FingerAction {
+	*fa = append(*fa, fm)
+	return fa
+}
+
+func (fa *FingerAction) Down() *FingerAction {
+	*fa = append(*fa, map[string]interface{}{"type": "pointerDown"})
+	return fa
+}
+
+func (fa *FingerAction) Up() *FingerAction {
+	*fa = append(*fa, map[string]interface{}{"type": "pointerUp"})
+	return fa
+}
+
+func (fa *FingerAction) Pause(second ...float64) *FingerAction {
+	if len(second) == 0 || second[0] < 0 {
+		second = []float64{0.5}
+	}
+	tmp := map[string]interface{}{
+		"type":     "pause",
+		"duration": second[0] * 1000,
+	}
+	*fa = append(*fa, tmp)
+	return fa
+}
+
+func (act *W3CActions) Tap(x, y int, element ...WebElement) *W3CActions {
+	fm := NewFingerMove().WithXY(x, y)
+	if len(element) != 0 {
+		fm.WithOrigin(element[0])
+	}
+	fingerAction := NewFingerAction().
+		Move(fm).
+		Down().
+		Pause(0.1).
+		Up()
+	return act.FingerAction(fingerAction)
+}
+
+func (act *W3CActions) DoubleTap(x, y int, element ...WebElement) *W3CActions {
+	fm := NewFingerMove().WithXY(x, y)
+	if len(element) != 0 {
+		fm.WithOrigin(element[0])
+	}
+	fingerAction := NewFingerAction().
+		Move(fm).
+		Down().
+		Pause(0.1).
+		Up().
+		Pause(0.04).
+		Down().
+		Pause(0.1).
+		Up()
+	return act.FingerAction(fingerAction)
+}
+
+func (act *W3CActions) Press(x, y int, second float64, element ...WebElement) *W3CActions {
+	fm := NewFingerMove().WithXY(x, y)
+	if len(element) != 0 {
+		fm.WithOrigin(element[0])
+	}
+	fingerAction := NewFingerAction().
+		Move(fm).
+		Down().
+		Pause(second).
+		Up()
+	return act.FingerAction(fingerAction)
+}
+
+func (act *W3CActions) Swipe(fromX, fromY, toX, toY int, element ...WebElement) *W3CActions {
+	fmFrom := NewFingerMove().WithXY(fromX, fromY)
+	fmTo := NewFingerMove().WithXY(toX, toY)
+	if len(element) != 0 {
+		fmFrom.WithOrigin(element[0])
+		fmTo.WithOrigin(element[0])
+	}
+	fingerAction := NewFingerAction().
+		Move(fmFrom).
+		Down().
+		Pause(0.25).
+		Move(fmTo).
+		Pause(0.25).
+		Up()
+	return act.FingerAction(fingerAction)
+}
+
+func (act *W3CActions) SwipeFloat(fromX, fromY, toX, toY float64, element ...WebElement) *W3CActions {
+	fmFrom := NewFingerMove().WithXYFloat(fromX, fromY)
+	fmTo := NewFingerMove().WithXYFloat(toX, toY)
+	if len(element) != 0 {
+		fmFrom.WithOrigin(element[0])
+		fmTo.WithOrigin(element[0])
+	}
+	fingerAction := NewFingerAction().
+		Move(fmFrom).
+		Down().
+		Pause(0.25).
+		Move(fmTo).
+		Pause(0.25).
+		Up()
+	return act.FingerAction(fingerAction)
+}
+
+/* ---------------------------------------------------------------------------------------------------------------- */
+
+type TouchActions []map[string]interface{}
+
+func NewTouchActions(capacity ...int) *TouchActions {
+	if len(capacity) == 0 || capacity[0] <= 0 {
+		capacity = []int{8}
+	}
+	tmp := make(TouchActions, 0, capacity[0])
+	return &tmp
+}
+
+func (act *TouchActions) MoveTo(opt TouchActionMoveTo) *TouchActions {
+	tmp := map[string]interface{}{
+		"action":  "moveTo",
+		"options": opt,
+	}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) Tap(opt TouchActionTap) *TouchActions {
+	tmp := map[string]interface{}{
+		"action":  "tap",
+		"options": opt,
+	}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) Press(opt TouchActionPress) *TouchActions {
+	tmp := map[string]interface{}{
+		"action":  "press",
+		"options": opt,
+	}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) LongPress(opt TouchActionLongPress) *TouchActions {
+	tmp := map[string]interface{}{
+		"action":  "longPress",
+		"options": opt,
+	}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) Wait(second ...float64) *TouchActions {
+	if len(second) == 0 || second[0] < 0 {
+		second = []float64{0.5}
+	}
+	tmp := map[string]interface{}{
+		"action":  "wait",
+		"options": map[string]interface{}{"ms": second[0] * 1000},
+	}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) Release() *TouchActions {
+	tmp := map[string]interface{}{"action": "release"}
+	*act = append(*act, tmp)
+	return act
+}
+
+func (act *TouchActions) Cancel() *TouchActions {
+	tmp := map[string]interface{}{"action": "cancel"}
+	*act = append(*act, tmp)
+	return act
+}
+
+type TouchActionMoveTo map[string]interface{}
+
+func NewTouchActionMoveTo() TouchActionMoveTo {
+	return make(map[string]interface{})
+}
+
+func (opt TouchActionMoveTo) WithXY(x, y int) TouchActionMoveTo {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionMoveTo) WithXYFloat(x, y float64) TouchActionMoveTo {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionMoveTo) WithElement(element WebElement) TouchActionMoveTo {
+	opt["element"] = element.UID()
+	return opt
+}
+
+type TouchActionTap map[string]interface{}
+
+func NewTouchActionTap() TouchActionTap {
+	return make(map[string]interface{})
+}
+
+func (opt TouchActionTap) WithXY(x, y int) TouchActionTap {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionTap) WithXYFloat(x, y float64) TouchActionTap {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionTap) WithElement(element WebElement) TouchActionTap {
+	opt["element"] = element.UID()
+	return opt
+}
+
+func (opt TouchActionTap) WithCount(count int) TouchActionTap {
+	opt["count"] = count
+	return opt
+}
+
+type TouchActionPress map[string]interface{}
+
+func NewTouchActionPress() TouchActionPress {
+	return make(map[string]interface{})
+}
+
+func (opt TouchActionPress) WithXY(x, y int) TouchActionPress {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionPress) WithXYFloat(x, y float64) TouchActionPress {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionPress) WithElement(element WebElement) TouchActionPress {
+	opt["element"] = element.UID()
+	return opt
+}
+
+func (opt TouchActionPress) WithPressure(pressure float64) TouchActionPress {
+	opt["pressure"] = pressure
+	return opt
+}
+
+type TouchActionLongPress map[string]interface{}
+
+func NewTouchActionLongPress() TouchActionLongPress {
+	return make(map[string]interface{})
+}
+
+func (opt TouchActionLongPress) WithXY(x, y int) TouchActionLongPress {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionLongPress) WithXYFloat(x, y float64) TouchActionLongPress {
+	opt["x"] = x
+	opt["y"] = y
+	return opt
+}
+
+func (opt TouchActionLongPress) WithElement(element WebElement) TouchActionLongPress {
+	opt["element"] = element.UID()
+	return opt
+}

hrp/pkg/uixt/ios_device.go

@@ -0,0 +1,560 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	builtinJSON "encoding/json"
+	"fmt"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	giDevice "github.com/electricbubble/gidevice"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+const (
+	// Changes the value of maximum depth for traversing elements source tree.
+	// It may help to prevent out of memory or timeout errors while getting the elements source tree,
+	// but it might restrict the depth of source tree.
+	// A part of elements source tree might be lost if the value was too small. Defaults to 50
+	snapshotMaxDepth = 10
+	// Allows to customize accept/dismiss alert button selector.
+	// It helps you to handle an arbitrary element as accept button in accept alert command.
+	// The selector should be a valid class chain expression, where the search root is the alert element itself.
+	// The default button location algorithm is used if the provided selector is wrong or does not match any element.
+	// e.g. **/XCUIElementTypeButton[`label CONTAINS[c] ‘accept’`]
+	acceptAlertButtonSelector  = "**/XCUIElementTypeButton[`label IN {'允许','好','仅在使用应用期间','稍后再说'}`]"
+	dismissAlertButtonSelector = "**/XCUIElementTypeButton[`label IN {'不允许','暂不'}`]"
+)
+
+const (
+	defaultWDAPort   = 8100
+	defaultMjpegPort = 9100
+)
+
+func InitWDAClient(device *IOSDevice) (*DriverExt, error) {
+	// init wda device
+	iosDevice, err := NewIOSDevice(device.opitons()...)
+	if err != nil {
+		return nil, err
+	}
+
+	// init WDA driver
+	capabilities := NewCapabilities()
+	capabilities.WithDefaultAlertAction(AlertActionAccept)
+	var driver WebDriver
+
+	if os.Getenv("WDA_USB_DRIVER") == "" {
+		// default use http driver
+		driver, err = iosDevice.NewHTTPDriver(capabilities)
+	} else {
+		driver, err = iosDevice.NewUSBDriver(capabilities)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to init WDA driver")
+	}
+
+	// switch to iOS springboard before init WDA session
+	// aviod getting stuck when some super app is activate such as douyin or wexin
+	log.Info().Msg("go back to home screen")
+	if err = driver.Homescreen(); err != nil {
+		return nil, errors.Wrap(err, "failed to go back to home screen")
+	}
+
+	driverExt, err := Extend(driver)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to extend WebDriver")
+	}
+	settings, err := driverExt.Driver.SetAppiumSettings(map[string]interface{}{
+		"snapshotMaxDepth":          snapshotMaxDepth,
+		"acceptAlertButtonSelector": acceptAlertButtonSelector,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to set appium WDA settings")
+	}
+	log.Info().Interface("appiumWDASettings", settings).Msg("set appium WDA settings")
+
+	if device.LogOn {
+		err = driverExt.Driver.StartCaptureLog("hrp_wda_log")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if device.PerfOptions != nil {
+		data, err := iosDevice.d.PerfStart(device.perfOpitons()...)
+		if err != nil {
+			return nil, err
+		}
+
+		driverExt.perfStop = make(chan struct{})
+		// start performance monitor
+		go func() {
+			for {
+				select {
+				case <-driverExt.perfStop:
+					iosDevice.d.PerfStop()
+					return
+				case d := <-data:
+					fmt.Println(string(d))
+					driverExt.perfData = append(driverExt.perfData, string(d))
+				}
+			}
+		}()
+	}
+
+	driverExt.UUID = iosDevice.UUID()
+	return driverExt, nil
+}
+
+type IOSDeviceOption func(*IOSDevice)
+
+func WithUDID(udid string) IOSDeviceOption {
+	return func(device *IOSDevice) {
+		device.UDID = udid
+	}
+}
+
+func WithWDAPort(port int) IOSDeviceOption {
+	return func(device *IOSDevice) {
+		device.Port = port
+	}
+}
+
+func WithWDAMjpegPort(port int) IOSDeviceOption {
+	return func(device *IOSDevice) {
+		device.MjpegPort = port
+	}
+}
+
+func WithLogOn(logOn bool) IOSDeviceOption {
+	return func(device *IOSDevice) {
+		device.LogOn = logOn
+	}
+}
+
+func WithPerfOptions(options ...giDevice.PerfOption) IOSDeviceOption {
+	return func(device *IOSDevice) {
+		device.PerfOptions = &giDevice.PerfOptions{}
+		for _, option := range options {
+			option(device.PerfOptions)
+		}
+	}
+}
+
+func IOSDevices(udid ...string) (devices []giDevice.Device, err error) {
+	var usbmux giDevice.Usbmux
+	if usbmux, err = giDevice.NewUsbmux(); err != nil {
+		return nil, fmt.Errorf("init usbmux failed: %v", err)
+	}
+
+	if devices, err = usbmux.Devices(); err != nil {
+		return nil, fmt.Errorf("list ios devices failed: %v", err)
+	}
+
+	// filter by udid
+	var deviceList []giDevice.Device
+	for _, d := range devices {
+		for _, u := range udid {
+			if u != "" && u != d.Properties().SerialNumber {
+				continue
+			}
+			deviceList = append(deviceList, d)
+		}
+	}
+
+	return deviceList, nil
+}
+
+func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) {
+	device = &IOSDevice{
+		Port:      defaultWDAPort,
+		MjpegPort: defaultMjpegPort,
+	}
+	for _, option := range options {
+		option(device)
+	}
+
+	deviceList, err := IOSDevices(device.UDID)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(deviceList) > 0 {
+		device.UDID = deviceList[0].Properties().SerialNumber
+		log.Info().Str("udid", device.UDID).Msg("select device")
+		device.d = deviceList[0]
+		return device, nil
+	}
+
+	return nil, fmt.Errorf("device %s not found", device.UDID)
+}
+
+type IOSDevice struct {
+	d           giDevice.Device
+	PerfOptions *giDevice.PerfOptions `json:"perf_options,omitempty" yaml:"perf_options,omitempty"`
+	UDID        string                `json:"udid,omitempty" yaml:"udid,omitempty"`
+	Port        int                   `json:"port,omitempty" yaml:"port,omitempty"`             // WDA remote port
+	MjpegPort   int                   `json:"mjpeg_port,omitempty" yaml:"mjpeg_port,omitempty"` // WDA remote MJPEG port
+	LogOn       bool                  `json:"log_on,omitempty" yaml:"log_on,omitempty"`
+}
+
+func (dev *IOSDevice) UUID() string {
+	return dev.UDID
+}
+
+func (dev *IOSDevice) forward(localPort, remotePort int) error {
+	log.Info().Int("localPort", localPort).Int("remotePort", remotePort).
+		Str("udid", dev.UDID).Msg("forward tcp port")
+
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", localPort))
+	if err != nil {
+		log.Error().Err(err).Msg("listen tcp error")
+		return err
+	}
+
+	go func(listener net.Listener, device giDevice.Device) {
+		for {
+			accept, err := listener.Accept()
+			if err != nil {
+				log.Error().Err(err).Msg("accept error")
+				continue
+			}
+
+			rInnerConn, err := device.NewConnect(remotePort)
+			if err != nil {
+				log.Error().Err(err).Msg("connect to device failed")
+				os.Exit(1)
+			}
+
+			rConn := rInnerConn.RawConn()
+			_ = rConn.SetDeadline(time.Time{})
+
+			go func(lConn net.Conn) {
+				go func(lConn, rConn net.Conn) {
+					if _, err := io.Copy(lConn, rConn); err != nil {
+						log.Error().Err(err).Msg("copy local -> remote")
+					}
+				}(lConn, rConn)
+				go func(lConn, rConn net.Conn) {
+					if _, err := io.Copy(rConn, lConn); err != nil {
+						log.Error().Err(err).Msg("copy local <- remote")
+					}
+				}(lConn, rConn)
+			}(accept)
+		}
+	}(listener, dev.d)
+
+	return nil
+}
+
+func (dev *IOSDevice) opitons() (deviceOptions []IOSDeviceOption) {
+	if dev.UDID != "" {
+		deviceOptions = append(deviceOptions, WithUDID(dev.UDID))
+	}
+	if dev.Port != 0 {
+		deviceOptions = append(deviceOptions, WithWDAPort(dev.Port))
+	}
+	if dev.MjpegPort != 0 {
+		deviceOptions = append(deviceOptions, WithWDAMjpegPort(dev.MjpegPort))
+	}
+	return
+}
+
+func (dev *IOSDevice) perfOpitons() (perfOptions []giDevice.PerfOption) {
+	if dev.PerfOptions == nil {
+		return
+	}
+
+	// system
+	if dev.PerfOptions.SysCPU {
+		perfOptions = append(perfOptions, giDevice.WithPerfSystemCPU(true))
+	}
+	if dev.PerfOptions.SysMem {
+		perfOptions = append(perfOptions, giDevice.WithPerfSystemMem(true))
+	}
+	if dev.PerfOptions.SysDisk {
+		perfOptions = append(perfOptions, giDevice.WithPerfSystemDisk(true))
+	}
+	if dev.PerfOptions.SysNetwork {
+		perfOptions = append(perfOptions, giDevice.WithPerfSystemNetwork(true))
+	}
+	if dev.PerfOptions.FPS {
+		perfOptions = append(perfOptions, giDevice.WithPerfFPS(true))
+	}
+	if dev.PerfOptions.Network {
+		perfOptions = append(perfOptions, giDevice.WithPerfNetwork(true))
+	}
+
+	// process
+	if dev.PerfOptions.BundleID != "" {
+		perfOptions = append(perfOptions,
+			giDevice.WithPerfBundleID(dev.PerfOptions.BundleID))
+	}
+	if dev.PerfOptions.Pid != 0 {
+		perfOptions = append(perfOptions,
+			giDevice.WithPerfPID(dev.PerfOptions.Pid))
+	}
+
+	// config
+	if dev.PerfOptions.OutputInterval != 0 {
+		perfOptions = append(perfOptions,
+			giDevice.WithPerfOutputInterval(dev.PerfOptions.OutputInterval))
+	}
+	if dev.PerfOptions.SystemAttributes != nil {
+		perfOptions = append(perfOptions,
+			giDevice.WithPerfSystemAttributes(dev.PerfOptions.SystemAttributes...))
+	}
+	if dev.PerfOptions.ProcessAttributes != nil {
+		perfOptions = append(perfOptions,
+			giDevice.WithPerfProcessAttributes(dev.PerfOptions.ProcessAttributes...))
+	}
+	return
+}
+
+// NewHTTPDriver creates new remote HTTP client, this will also start a new session.
+func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver WebDriver, err error) {
+	localPort, err := getFreePort()
+	if err != nil {
+		return nil, errors.Wrap(err, "get free port failed")
+	}
+	if err = dev.forward(localPort, dev.Port); err != nil {
+		return nil, errors.Wrap(err, "forward tcp port failed")
+	}
+	localMjpegPort, err := getFreePort()
+	if err != nil {
+		return nil, errors.Wrap(err, "get free port failed")
+	}
+	if err = dev.forward(localMjpegPort, dev.MjpegPort); err != nil {
+		return nil, errors.Wrap(err, "forward tcp port failed")
+	}
+
+	log.Info().Interface("capabilities", capabilities).
+		Int("localPort", localPort).Int("localMjpegPort", localMjpegPort).
+		Msg("init WDA HTTP driver")
+
+	wd := new(wdaDriver)
+	wd.client = http.DefaultClient
+
+	host := "127.0.0.1"
+	if wd.urlPrefix, err = url.Parse(fmt.Sprintf("http://%s:%d", host, localPort)); err != nil {
+		return nil, err
+	}
+	var sessionInfo SessionInfo
+	if sessionInfo, err = wd.NewSession(capabilities); err != nil {
+		return nil, err
+	}
+	wd.sessionId = sessionInfo.SessionId
+
+	if wd.mjpegHTTPConn, err = net.Dial(
+		"tcp",
+		fmt.Sprintf("%s:%d", host, localMjpegPort),
+	); err != nil {
+		return nil, err
+	}
+	wd.mjpegClient = convertToHTTPClient(wd.mjpegHTTPConn)
+
+	return wd, nil
+}
+
+// NewUSBDriver creates new client via USB connected device, this will also start a new session.
+func (dev *IOSDevice) NewUSBDriver(capabilities Capabilities) (driver WebDriver, err error) {
+	log.Info().Interface("capabilities", capabilities).
+		Str("udid", dev.UDID).Msg("init WDA USB driver")
+
+	wd := new(wdaDriver)
+	if wd.defaultConn, err = dev.d.NewConnect(dev.Port, 0); err != nil {
+		return nil, fmt.Errorf("connect port %d failed: %w",
+			dev.Port, err)
+	}
+	wd.client = convertToHTTPClient(wd.defaultConn.RawConn())
+
+	if wd.mjpegUSBConn, err = dev.d.NewConnect(dev.MjpegPort, 0); err != nil {
+		return nil, fmt.Errorf("connect MJPEG port %d failed: %w",
+			dev.MjpegPort, err)
+	}
+	wd.mjpegClient = convertToHTTPClient(wd.mjpegUSBConn.RawConn())
+
+	if wd.urlPrefix, err = url.Parse("http://" + dev.UDID); err != nil {
+		return nil, err
+	}
+	_, err = wd.NewSession(capabilities)
+
+	return wd, err
+}
+
+func (dExt *DriverExt) ConnectMjpegStream(httpClient *http.Client) (err error) {
+	if httpClient == nil {
+		return errors.New(`'httpClient' can't be nil`)
+	}
+
+	var req *http.Request
+	if req, err = http.NewRequest(http.MethodGet, "http://*", nil); err != nil {
+		return err
+	}
+
+	var resp *http.Response
+	if resp, err = httpClient.Do(req); err != nil {
+		return err
+	}
+	// defer func() { _ = resp.Body.Close() }()
+
+	var boundary string
+	if _, param, err := mime.ParseMediaType(resp.Header.Get("Content-Type")); err != nil {
+		return err
+	} else {
+		boundary = strings.Trim(param["boundary"], "-")
+	}
+
+	mjpegReader := multipart.NewReader(resp.Body, boundary)
+
+	go func() {
+		for {
+			select {
+			case <-dExt.doneMjpegStream:
+				_ = resp.Body.Close()
+				return
+			default:
+				var part *multipart.Part
+				if part, err = mjpegReader.NextPart(); err != nil {
+					dExt.frame = nil
+					continue
+				}
+
+				raw := new(bytes.Buffer)
+				if _, err = raw.ReadFrom(part); err != nil {
+					dExt.frame = nil
+					continue
+				}
+				dExt.frame = raw
+			}
+		}
+	}()
+
+	return
+}
+
+func (dExt *DriverExt) CloseMjpegStream() {
+	dExt.doneMjpegStream <- true
+}
+
+type rawResponse []byte
+
+func (r rawResponse) checkErr() (err error) {
+	reply := new(struct {
+		Value struct {
+			Err        string `json:"error"`
+			Message    string `json:"message"`
+			Traceback  string `json:"traceback"`  // wda
+			Stacktrace string `json:"stacktrace"` // uia
+		}
+	})
+	if err = json.Unmarshal(r, reply); err != nil {
+		return err
+	}
+	if reply.Value.Err != "" {
+		errText := reply.Value.Message
+		re := regexp.MustCompile(`{.+?=(.+?)}`)
+		if re.MatchString(reply.Value.Message) {
+			subMatch := re.FindStringSubmatch(reply.Value.Message)
+			errText = subMatch[len(subMatch)-1]
+		}
+		return fmt.Errorf("%s: %s", reply.Value.Err, errText)
+	}
+	return
+}
+
+func (r rawResponse) valueConvertToString() (s string, err error) {
+	reply := new(struct{ Value string })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return "", errors.Wrapf(err, "json.Unmarshal failed, rawResponse: %s", string(r))
+	}
+	s = reply.Value
+	return
+}
+
+func (r rawResponse) valueConvertToBool() (b bool, err error) {
+	reply := new(struct{ Value bool })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return false, err
+	}
+	b = reply.Value
+	return
+}
+
+func (r rawResponse) valueConvertToSessionInfo() (sessionInfo SessionInfo, err error) {
+	reply := new(struct{ Value struct{ SessionInfo } })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return SessionInfo{}, err
+	}
+	sessionInfo = reply.Value.SessionInfo
+	return
+}
+
+func (r rawResponse) valueConvertToJsonRawMessage() (raw builtinJSON.RawMessage, err error) {
+	reply := new(struct{ Value builtinJSON.RawMessage })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return nil, err
+	}
+	raw = reply.Value
+	return
+}
+
+func (r rawResponse) valueDecodeAsBase64() (raw *bytes.Buffer, err error) {
+	str, err := r.valueConvertToString()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to convert value to string")
+	}
+	decodeString, err := base64.StdEncoding.DecodeString(str)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to decode base64 string")
+	}
+	raw = bytes.NewBuffer(decodeString)
+	return
+}
+
+var errNoSuchElement = errors.New("no such element")
+
+func (r rawResponse) valueConvertToElementID() (id string, err error) {
+	reply := new(struct{ Value map[string]string })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return "", err
+	}
+	if len(reply.Value) == 0 {
+		return "", errNoSuchElement
+	}
+	if id = elementIDFromValue(reply.Value); id == "" {
+		return "", fmt.Errorf("invalid element returned: %+v", reply)
+	}
+	return
+}
+
+func (r rawResponse) valueConvertToElementIDs() (IDs []string, err error) {
+	reply := new(struct{ Value []map[string]string })
+	if err = json.Unmarshal(r, reply); err != nil {
+		return nil, err
+	}
+	if len(reply.Value) == 0 {
+		return nil, errNoSuchElement
+	}
+	IDs = make([]string, len(reply.Value))
+	for i, elem := range reply.Value {
+		var id string
+		if id = elementIDFromValue(elem); id == "" {
+			return nil, fmt.Errorf("invalid element returned: %+v", reply)
+		}
+		IDs[i] = id
+	}
+	return
+}

hrp/pkg/uixt/ios_driver.go

@@ -0,0 +1,898 @@
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	builtinJSON "encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	giDevice "github.com/electricbubble/gidevice"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+type wdaDriver struct {
+	Driver
+
+	// default port
+	defaultConn giDevice.InnerConn
+
+	// mjpeg port
+	mjpegUSBConn  giDevice.InnerConn // via USB
+	mjpegHTTPConn net.Conn           // via HTTP
+	mjpegClient   *http.Client
+}
+
+func (wd *wdaDriver) GetMjpegClient() *http.Client {
+	return wd.mjpegClient
+}
+
+func (wd *wdaDriver) Close() error {
+	if wd.defaultConn != nil {
+		wd.defaultConn.Close()
+	}
+	if wd.mjpegUSBConn != nil {
+		wd.mjpegUSBConn.Close()
+	}
+
+	if wd.mjpegClient != nil {
+		wd.mjpegClient.CloseIdleConnections()
+	}
+	return wd.mjpegHTTPConn.Close()
+}
+
+func (wd *wdaDriver) NewSession(capabilities Capabilities) (sessionInfo SessionInfo, err error) {
+	// [[FBRoute POST:@"/session"].withoutSession respondWithTarget:self action:@selector(handleCreateSession:)]
+	data := make(map[string]interface{})
+	if len(capabilities) == 0 {
+		data["capabilities"] = make(map[string]interface{})
+	} else {
+		data["capabilities"] = map[string]interface{}{"alwaysMatch": capabilities}
+	}
+
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session"); err != nil {
+		return SessionInfo{}, err
+	}
+	if sessionInfo, err = rawResp.valueConvertToSessionInfo(); err != nil {
+		return SessionInfo{}, err
+	}
+	wd.sessionId = sessionInfo.SessionId
+	return
+}
+
+func (wd *wdaDriver) ActiveSession() (sessionInfo SessionInfo, err error) {
+	// [[FBRoute GET:@""] respondWithTarget:self action:@selector(handleGetActiveSession:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId); err != nil {
+		return SessionInfo{}, err
+	}
+	if sessionInfo, err = rawResp.valueConvertToSessionInfo(); err != nil {
+		return SessionInfo{}, err
+	}
+	return
+}
+
+func (wd *wdaDriver) DeleteSession() (err error) {
+	// [[FBRoute DELETE:@""] respondWithTarget:self action:@selector(handleDeleteSession:)]
+	_, err = wd.httpDELETE("/session", wd.sessionId)
+	return
+}
+
+func (wd *wdaDriver) Status() (deviceStatus DeviceStatus, err error) {
+	// [[FBRoute GET:@"/status"].withoutSession respondWithTarget:self action:@selector(handleGetStatus:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/status"); err != nil {
+		return DeviceStatus{}, err
+	}
+	reply := new(struct{ Value struct{ DeviceStatus } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return DeviceStatus{}, err
+	}
+	deviceStatus = reply.Value.DeviceStatus
+	return
+}
+
+func (wd *wdaDriver) DeviceInfo() (deviceInfo DeviceInfo, err error) {
+	// [[FBRoute GET:@"/wda/device/info"] respondWithTarget:self action:@selector(handleGetDeviceInfo:)]
+	// [[FBRoute GET:@"/wda/device/info"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/device/info"); err != nil {
+		return DeviceInfo{}, err
+	}
+	reply := new(struct{ Value struct{ DeviceInfo } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return DeviceInfo{}, err
+	}
+	deviceInfo = reply.Value.DeviceInfo
+	return
+}
+
+func (wd *wdaDriver) Location() (location Location, err error) {
+	// [[FBRoute GET:@"/wda/device/location"] respondWithTarget:self action:@selector(handleGetLocation:)]
+	// [[FBRoute GET:@"/wda/device/location"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/device/location"); err != nil {
+		return Location{}, err
+	}
+	reply := new(struct{ Value struct{ Location } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Location{}, err
+	}
+	location = reply.Value.Location
+	return
+}
+
+func (wd *wdaDriver) BatteryInfo() (batteryInfo BatteryInfo, err error) {
+	// [[FBRoute GET:@"/wda/batteryInfo"] respondWithTarget:self action:@selector(handleGetBatteryInfo:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/batteryInfo"); err != nil {
+		return BatteryInfo{}, err
+	}
+	reply := new(struct{ Value struct{ BatteryInfo } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return BatteryInfo{}, err
+	}
+	batteryInfo = reply.Value.BatteryInfo
+	return
+}
+
+func (wd *wdaDriver) WindowSize() (size Size, err error) {
+	// [[FBRoute GET:@"/window/size"] respondWithTarget:self action:@selector(handleGetWindowSize:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/window/size"); err != nil {
+		return Size{}, err
+	}
+	reply := new(struct{ Value struct{ Size } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Size{}, err
+	}
+	size = reply.Value.Size
+	return
+}
+
+func (wd *wdaDriver) Screen() (screen Screen, err error) {
+	// [[FBRoute GET:@"/wda/screen"] respondWithTarget:self action:@selector(handleGetScreen:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/screen"); err != nil {
+		return Screen{}, err
+	}
+	reply := new(struct{ Value struct{ Screen } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Screen{}, err
+	}
+	screen = reply.Value.Screen
+	return
+}
+
+func (wd *wdaDriver) Scale() (float64, error) {
+	screen, err := wd.Screen()
+	if err != nil {
+		return 0, err
+	}
+	return screen.Scale, nil
+}
+
+func (wd *wdaDriver) ActiveAppInfo() (info AppInfo, err error) {
+	// [[FBRoute GET:@"/wda/activeAppInfo"] respondWithTarget:self action:@selector(handleActiveAppInfo:)]
+	// [[FBRoute GET:@"/wda/activeAppInfo"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/activeAppInfo"); err != nil {
+		return AppInfo{}, err
+	}
+	reply := new(struct{ Value struct{ AppInfo } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return AppInfo{}, err
+	}
+	info = reply.Value.AppInfo
+	return
+}
+
+func (wd *wdaDriver) ActiveAppsList() (appsList []AppBaseInfo, err error) {
+	// [[FBRoute GET:@"/wda/apps/list"] respondWithTarget:self action:@selector(handleGetActiveAppsList:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/apps/list"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value []AppBaseInfo })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	appsList = reply.Value
+	return
+}
+
+func (wd *wdaDriver) AppState(bundleId string) (runState AppState, err error) {
+	// [[FBRoute POST:@"/wda/apps/state"] respondWithTarget:self action:@selector(handleSessionAppState:)]
+	data := map[string]interface{}{"bundleId": bundleId}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/state"); err != nil {
+		return 0, err
+	}
+	reply := new(struct{ Value AppState })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return 0, err
+	}
+	runState = reply.Value
+	_ = rawResp
+	return
+}
+
+func (wd *wdaDriver) IsLocked() (locked bool, err error) {
+	// [[FBRoute GET:@"/wda/locked"] respondWithTarget:self action:@selector(handleIsLocked:)]
+	// [[FBRoute GET:@"/wda/locked"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/locked"); err != nil {
+		return false, err
+	}
+	if locked, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (wd *wdaDriver) Unlock() (err error) {
+	// [[FBRoute POST:@"/wda/unlock"] respondWithTarget:self action:@selector(handleUnlock:)]
+	// [[FBRoute POST:@"/wda/unlock"].withoutSession
+	_, err = wd.httpPOST(nil, "/session", wd.sessionId, "/wda/unlock")
+	return
+}
+
+func (wd *wdaDriver) Lock() (err error) {
+	// [[FBRoute POST:@"/wda/lock"] respondWithTarget:self action:@selector(handleLock:)]
+	// [[FBRoute POST:@"/wda/lock"].withoutSession
+	_, err = wd.httpPOST(nil, "/session", wd.sessionId, "/wda/lock")
+	return
+}
+
+func (wd *wdaDriver) Homescreen() (err error) {
+	// [[FBRoute POST:@"/wda/homescreen"].withoutSession respondWithTarget:self action:@selector(handleHomescreenCommand:)]
+	_, err = wd.httpPOST(nil, "/wda/homescreen")
+	return
+}
+
+func (wd *wdaDriver) AlertText() (text string, err error) {
+	// [[FBRoute GET:@"/alert/text"] respondWithTarget:self action:@selector(handleAlertGetTextCommand:)]
+	// [[FBRoute GET:@"/alert/text"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/alert/text"); err != nil {
+		return "", err
+	}
+	if text, err = rawResp.valueConvertToString(); err != nil {
+		return "", err
+	}
+	return
+}
+
+func (wd *wdaDriver) AlertButtons() (btnLabels []string, err error) {
+	// [[FBRoute GET:@"/wda/alert/buttons"] respondWithTarget:self action:@selector(handleGetAlertButtonsCommand:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/alert/buttons"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value []string })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	btnLabels = reply.Value
+	return
+}
+
+func (wd *wdaDriver) AlertAccept(label ...string) (err error) {
+	// [[FBRoute POST:@"/alert/accept"] respondWithTarget:self action:@selector(handleAlertAcceptCommand:)]
+	// [[FBRoute POST:@"/alert/accept"].withoutSession
+	data := make(map[string]interface{})
+	if len(label) != 0 && label[0] != "" {
+		data["name"] = label[0]
+	}
+	_, err = wd.httpPOST(data, "/alert/accept")
+	return
+}
+
+func (wd *wdaDriver) AlertDismiss(label ...string) (err error) {
+	// [[FBRoute POST:@"/alert/dismiss"] respondWithTarget:self action:@selector(handleAlertDismissCommand:)]
+	// [[FBRoute POST:@"/alert/dismiss"].withoutSession
+	data := make(map[string]interface{})
+	if len(label) != 0 && label[0] != "" {
+		data["name"] = label[0]
+	}
+	_, err = wd.httpPOST(data, "/alert/dismiss")
+	return
+}
+
+func (wd *wdaDriver) AlertSendKeys(text string) (err error) {
+	// [[FBRoute POST:@"/alert/text"] respondWithTarget:self action:@selector(handleAlertSetTextCommand:)]
+	data := map[string]interface{}{"value": strings.Split(text, "")}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/alert/text")
+	return
+}
+
+func (wd *wdaDriver) AppLaunch(bundleId string, launchOpt ...AppLaunchOption) (err error) {
+	// [[FBRoute POST:@"/wda/apps/launch"] respondWithTarget:self action:@selector(handleSessionAppLaunch:)]
+	data := make(map[string]interface{})
+	if len(launchOpt) != 0 {
+		data = launchOpt[0]
+	}
+	data["bundleId"] = bundleId
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/launch")
+	return
+}
+
+func (wd *wdaDriver) AppLaunchUnattached(bundleId string) (err error) {
+	// [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)]
+	data := map[string]interface{}{"bundleId": bundleId}
+	_, err = wd.httpPOST(data, "/wda/apps/launchUnattached")
+	return
+}
+
+func (wd *wdaDriver) AppTerminate(bundleId string) (successful bool, err error) {
+	// [[FBRoute POST:@"/wda/apps/terminate"] respondWithTarget:self action:@selector(handleSessionAppTerminate:)]
+	data := map[string]interface{}{"bundleId": bundleId}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/terminate"); err != nil {
+		return false, err
+	}
+	if successful, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (wd *wdaDriver) AppActivate(bundleId string) (err error) {
+	// [[FBRoute POST:@"/wda/apps/activate"] respondWithTarget:self action:@selector(handleSessionAppActivate:)]
+	data := map[string]interface{}{"bundleId": bundleId}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/activate")
+	return
+}
+
+func (wd *wdaDriver) AppDeactivate(second float64) (err error) {
+	// [[FBRoute POST:@"/wda/deactivateApp"] respondWithTarget:self action:@selector(handleDeactivateAppCommand:)]
+	if second < 3 {
+		second = 3.0
+	}
+	data := map[string]interface{}{"duration": second}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/deactivateApp")
+	return
+}
+
+func (wd *wdaDriver) AppAuthReset(resource ProtectedResource) (err error) {
+	// [[FBRoute POST:@"/wda/resetAppAuth"] respondWithTarget:self action:@selector(handleResetAppAuth:)]
+	data := map[string]interface{}{"resource": resource}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/resetAppAuth")
+	return
+}
+
+func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error {
+	return wd.TapFloat(float64(x), float64(y), options...)
+}
+
+func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) {
+	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	// append options in post data for extra WDA configurations
+	// e.g. add identifier in tap event logs
+	for _, option := range options {
+		option(data)
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0")
+	return
+}
+
+func (wd *wdaDriver) DoubleTap(x, y int) error {
+	return wd.DoubleTapFloat(float64(x), float64(y))
+}
+
+func (wd *wdaDriver) DoubleTapFloat(x, y float64) (err error) {
+	// [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)]
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/doubleTap")
+	return
+}
+
+func (wd *wdaDriver) TouchAndHold(x, y int, second ...float64) error {
+	return wd.TouchAndHoldFloat(float64(x), float64(y), second...)
+}
+
+func (wd *wdaDriver) TouchAndHoldFloat(x, y float64, second ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/touchAndHold"] respondWithTarget:self action:@selector(handleTouchAndHoldCoordinate:)]
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	if len(second) == 0 || second[0] <= 0 {
+		second = []float64{1.0}
+	}
+	data["duration"] = second[0]
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/touchAndHold")
+	return
+}
+
+func (wd *wdaDriver) Drag(fromX, fromY, toX, toY int, options ...DataOption) error {
+	return wd.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
+}
+
+func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) (err error) {
+	// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
+	data := map[string]interface{}{
+		"fromX": fromX,
+		"fromY": fromY,
+		"toX":   toX,
+		"toY":   toY,
+	}
+
+	// append options in post data for extra WDA configurations
+	// e.g. use WithPressDuration to set pressForDuration
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["duration"]; !ok {
+		data["duration"] = 1.0 // default duration
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
+	return
+}
+
+func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error {
+	options = append(options, WithPressDuration(0))
+	return wd.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
+}
+
+func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error {
+	options = append(options, WithPressDuration(0))
+	return wd.DragFloat(fromX, fromY, toX, toY, options...)
+}
+
+func (wd *wdaDriver) ForceTouch(x, y int, pressure float64, second ...float64) error {
+	return wd.ForceTouchFloat(float64(x), float64(y), pressure, second...)
+}
+
+func (wd *wdaDriver) ForceTouchFloat(x, y, pressure float64, second ...float64) error {
+	if len(second) == 0 || second[0] <= 0 {
+		second = []float64{1.0}
+	}
+	actions := NewTouchActions().
+		Press(
+			NewTouchActionPress().WithXYFloat(x, y).WithPressure(pressure)).
+		Wait(second[0]).
+		Release()
+	return wd.PerformAppiumTouchActions(actions)
+}
+
+func (wd *wdaDriver) PerformW3CActions(actions *W3CActions) (err error) {
+	// [[FBRoute POST:@"/actions"] respondWithTarget:self action:@selector(handlePerformW3CTouchActions:)]
+	data := map[string]interface{}{"actions": actions}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/actions")
+	return
+}
+
+func (wd *wdaDriver) PerformAppiumTouchActions(touchActs *TouchActions) (err error) {
+	// [[FBRoute POST:@"/wda/touch/perform"] respondWithTarget:self action:@selector(handlePerformAppiumTouchActions:)]
+	// [[FBRoute POST:@"/wda/touch/multi/perform"]
+	data := map[string]interface{}{"actions": touchActs}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/touch/multi/perform")
+	return
+}
+
+func (wd *wdaDriver) SetPasteboard(contentType PasteboardType, content string) (err error) {
+	// [[FBRoute POST:@"/wda/setPasteboard"] respondWithTarget:self action:@selector(handleSetPasteboard:)]
+	data := map[string]interface{}{
+		"contentType": contentType,
+		"content":     base64.StdEncoding.EncodeToString([]byte(content)),
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/setPasteboard")
+	return
+}
+
+func (wd *wdaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error) {
+	// [[FBRoute POST:@"/wda/getPasteboard"] respondWithTarget:self action:@selector(handleGetPasteboard:)]
+	data := map[string]interface{}{"contentType": contentType}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/getPasteboard"); err != nil {
+		return nil, err
+	}
+	if raw, err = rawResp.valueDecodeAsBase64(); err != nil {
+		return nil, err
+	}
+	return
+}
+
+func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) {
+	// [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)]
+	data := map[string]interface{}{"value": strings.Split(text, "")}
+
+	// append options in post data for extra WDA configurations
+	// e.g. use WithFrequency to set frequency of typing
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["frequency"]; !ok {
+		data["frequency"] = 60 // default frequency
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys")
+	return
+}
+
+func (wd *wdaDriver) Input(text string, options ...DataOption) (err error) {
+	return wd.SendKeys(text, options...)
+}
+
+func (wd *wdaDriver) KeyboardDismiss(keyNames ...string) (err error) {
+	// [[FBRoute POST:@"/wda/keyboard/dismiss"] respondWithTarget:self action:@selector(handleDismissKeyboardCommand:)]
+	if len(keyNames) == 0 {
+		keyNames = []string{"return"}
+	}
+	data := map[string]interface{}{"keyNames": keyNames}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keyboard/dismiss")
+	return
+}
+
+func (wd *wdaDriver) PressButton(devBtn DeviceButton) (err error) {
+	// [[FBRoute POST:@"/wda/pressButton"] respondWithTarget:self action:@selector(handlePressButtonCommand:)]
+	data := map[string]interface{}{"name": devBtn}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/pressButton")
+	return
+}
+
+func (wd *wdaDriver) IOHIDEvent(pageID EventPageID, usageID EventUsageID, duration ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/performIoHidEvent"] respondWithTarget:self action:@selector(handlePeformIOHIDEvent:)]
+	if len(duration) == 0 || duration[0] <= 0 {
+		duration = []float64{0.005}
+	}
+	data := map[string]interface{}{
+		"page":     pageID,
+		"usage":    usageID,
+		"duration": duration[0],
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/performIoHidEvent")
+	return
+}
+
+func (wd *wdaDriver) StartCamera() (err error) {
+	// start camera, alias for app_launch com.apple.camera
+	return wd.AppLaunch("com.apple.camera")
+}
+
+func (wd *wdaDriver) StopCamera() (err error) {
+	// stop camera, alias for app_terminate com.apple.camera
+	success, err := wd.AppTerminate("com.apple.camera")
+	if err != nil {
+		return errors.Wrap(err, "failed to terminate camera")
+	}
+	if !success {
+		log.Warn().Msg("camera was not running")
+	}
+	return nil
+}
+
+func (wd *wdaDriver) ExpectNotification(notifyName string, notifyType NotificationType, second ...int) (err error) {
+	// [[FBRoute POST:@"/wda/expectNotification"] respondWithTarget:self action:@selector(handleExpectNotification:)]
+	if len(second) == 0 {
+		second = []int{60}
+	}
+	data := map[string]interface{}{
+		"name":    notifyName,
+		"type":    notifyType,
+		"timeout": second[0],
+	}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/expectNotification")
+	return
+}
+
+func (wd *wdaDriver) SiriActivate(text string) (err error) {
+	// [[FBRoute POST:@"/wda/siri/activate"] respondWithTarget:self action:@selector(handleActivateSiri:)]
+	data := map[string]interface{}{"text": text}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/siri/activate")
+	return
+}
+
+func (wd *wdaDriver) SiriOpenUrl(url string) (err error) {
+	// [[FBRoute POST:@"/url"] respondWithTarget:self action:@selector(handleOpenURL:)]
+	data := map[string]interface{}{"url": url}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/url")
+	return
+}
+
+func (wd *wdaDriver) Orientation() (orientation Orientation, err error) {
+	// [[FBRoute GET:@"/orientation"] respondWithTarget:self action:@selector(handleGetOrientation:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/orientation"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value Orientation })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+	orientation = reply.Value
+	return
+}
+
+func (wd *wdaDriver) SetOrientation(orientation Orientation) (err error) {
+	// [[FBRoute POST:@"/orientation"] respondWithTarget:self action:@selector(handleSetOrientation:)]
+	data := map[string]interface{}{"orientation": orientation}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/orientation")
+	return
+}
+
+func (wd *wdaDriver) Rotation() (rotation Rotation, err error) {
+	// [[FBRoute GET:@"/rotation"] respondWithTarget:self action:@selector(handleGetRotation:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/rotation"); err != nil {
+		return Rotation{}, err
+	}
+	reply := new(struct{ Value Rotation })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Rotation{}, err
+	}
+	rotation = reply.Value
+	return
+}
+
+func (wd *wdaDriver) SetRotation(rotation Rotation) (err error) {
+	// [[FBRoute POST:@"/rotation"] respondWithTarget:self action:@selector(handleSetRotation:)]
+	_, err = wd.httpPOST(rotation, "/session", wd.sessionId, "/rotation")
+	return
+}
+
+func (wd *wdaDriver) MatchTouchID(isMatch bool) (err error) {
+	// [FBRoute POST:@"/wda/touch_id"]
+	data := map[string]interface{}{"match": isMatch}
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/touch_id")
+	return
+}
+
+func (wd *wdaDriver) ActiveElement() (element WebElement, err error) {
+	// [[FBRoute GET:@"/element/active"] respondWithTarget:self action:@selector(handleGetActiveElement:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/element/active"); err != nil {
+		return nil, err
+	}
+	var elementID string
+	if elementID, err = rawResp.valueConvertToElementID(); err != nil {
+		return nil, err
+	}
+	element = &wdaElement{parent: wd, id: elementID}
+	return
+}
+
+func (wd *wdaDriver) FindElement(by BySelector) (element WebElement, err error) {
+	// [[FBRoute POST:@"/element"] respondWithTarget:self action:@selector(handleFindElement:)]
+	using, value := by.getUsingAndValue()
+	data := map[string]interface{}{
+		"using": using,
+		"value": value,
+	}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/element"); err != nil {
+		return nil, err
+	}
+	var elementID string
+	if elementID, err = rawResp.valueConvertToElementID(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find an element using '%s', value '%s'", err, using, value)
+		}
+		return nil, err
+	}
+	element = &wdaElement{parent: wd, id: elementID}
+	return
+}
+
+func (wd *wdaDriver) FindElements(by BySelector) (elements []WebElement, err error) {
+	// [[FBRoute POST:@"/elements"] respondWithTarget:self action:@selector(handleFindElements:)]
+	using, value := by.getUsingAndValue()
+	data := map[string]interface{}{
+		"using": using,
+		"value": value,
+	}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/elements"); err != nil {
+		return nil, err
+	}
+	var elementIDs []string
+	if elementIDs, err = rawResp.valueConvertToElementIDs(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find an element using '%s', value '%s'", err, using, value)
+		}
+		return nil, err
+	}
+	elements = make([]WebElement, len(elementIDs))
+	for i := range elementIDs {
+		elements[i] = &wdaElement{parent: wd, id: elementIDs[i]}
+	}
+	return
+}
+
+func (wd *wdaDriver) Screenshot() (raw *bytes.Buffer, err error) {
+	// [[FBRoute GET:@"/screenshot"] respondWithTarget:self action:@selector(handleGetScreenshot:)]
+	// [[FBRoute GET:@"/screenshot"].withoutSession respondWithTarget:self action:@selector(handleGetScreenshot:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/screenshot"); err != nil {
+		return nil, errors.Wrap(err, "get WDA screenshot data failed")
+	}
+
+	if raw, err = rawResp.valueDecodeAsBase64(); err != nil {
+		return nil, errors.Wrap(err, "decode WDA screenshot data failed")
+	}
+	return
+}
+
+func (wd *wdaDriver) Source(srcOpt ...SourceOption) (source string, err error) {
+	// [[FBRoute GET:@"/source"] respondWithTarget:self action:@selector(handleGetSourceCommand:)]
+	// [[FBRoute GET:@"/source"].withoutSession
+	tmp, _ := url.Parse(wd.concatURL(nil, "/session", wd.sessionId))
+	toJsonRaw := false
+	if len(srcOpt) != 0 {
+		q := tmp.Query()
+		for k, val := range srcOpt[0] {
+			v := val.(string)
+			q.Set(k, v)
+			if k == "format" && v == "json" {
+				toJsonRaw = true
+			}
+		}
+		tmp.RawQuery = q.Encode()
+	}
+
+	var rawResp rawResponse
+	if rawResp, err = wd.httpRequest(http.MethodGet, wd.concatURL(tmp, "/source"), nil); err != nil {
+		return "", nil
+	}
+	if toJsonRaw {
+		var jr builtinJSON.RawMessage
+		if jr, err = rawResp.valueConvertToJsonRawMessage(); err != nil {
+			return "", err
+		}
+		return string(jr), nil
+	}
+	if source, err = rawResp.valueConvertToString(); err != nil {
+		return "", err
+	}
+	return
+}
+
+func (wd *wdaDriver) AccessibleSource() (source string, err error) {
+	// [[FBRoute GET:@"/wda/accessibleSource"] respondWithTarget:self action:@selector(handleGetAccessibleSourceCommand:)]
+	// [[FBRoute GET:@"/wda/accessibleSource"].withoutSession
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/wda/accessibleSource"); err != nil {
+		return "", err
+	}
+	var jr builtinJSON.RawMessage
+	if jr, err = rawResp.valueConvertToJsonRawMessage(); err != nil {
+		return "", err
+	}
+	source = string(jr)
+	return
+}
+
+func (wd *wdaDriver) HealthCheck() (err error) {
+	// [[FBRoute GET:@"/wda/healthcheck"].withoutSession respondWithTarget:self action:@selector(handleGetHealthCheck:)]
+	_, err = wd.httpGET("/wda/healthcheck")
+	return
+}
+
+func (wd *wdaDriver) GetAppiumSettings() (settings map[string]interface{}, err error) {
+	// [[FBRoute GET:@"/appium/settings"] respondWithTarget:self action:@selector(handleGetSettings:)]
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/session", wd.sessionId, "/appium/settings"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value map[string]interface{} })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	settings = reply.Value
+	return
+}
+
+func (wd *wdaDriver) SetAppiumSettings(settings map[string]interface{}) (ret map[string]interface{}, err error) {
+	// [[FBRoute POST:@"/appium/settings"] respondWithTarget:self action:@selector(handleSetSettings:)]
+	data := map[string]interface{}{"settings": settings}
+	var rawResp rawResponse
+	if rawResp, err = wd.httpPOST(data, "/session", wd.sessionId, "/appium/settings"); err != nil {
+		return nil, err
+	}
+	reply := new(struct{ Value map[string]interface{} })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return nil, err
+	}
+	ret = reply.Value
+	return
+}
+
+func (wd *wdaDriver) IsHealthy() (healthy bool, err error) {
+	var rawResp rawResponse
+	if rawResp, err = wd.httpGET("/health"); err != nil {
+		return false, err
+	}
+	if string(rawResp) != "I-AM-ALIVE" {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (wd *wdaDriver) WdaShutdown() (err error) {
+	_, err = wd.httpGET("/wda/shutdown")
+	return
+}
+
+func (wd *wdaDriver) WaitWithTimeoutAndInterval(condition Condition, timeout, interval time.Duration) error {
+	startTime := time.Now()
+	for {
+		done, err := condition(wd)
+		if err != nil {
+			return err
+		}
+		if done {
+			return nil
+		}
+
+		if elapsed := time.Since(startTime); elapsed > timeout {
+			return fmt.Errorf("timeout after %v", elapsed)
+		}
+		time.Sleep(interval)
+	}
+}
+
+func (wd *wdaDriver) WaitWithTimeout(condition Condition, timeout time.Duration) error {
+	return wd.WaitWithTimeoutAndInterval(condition, timeout, DefaultWaitInterval)
+}
+
+func (wd *wdaDriver) Wait(condition Condition) error {
+	return wd.WaitWithTimeoutAndInterval(condition, DefaultWaitTimeout, DefaultWaitInterval)
+}
+
+func (wd *wdaDriver) triggerWDALog(data map[string]interface{}) (rawResp []byte, err error) {
+	// [[FBRoute POST:@"/gtf/automation/log"].withoutSession respondWithTarget:self action:@selector(handleAutomationLog:)]
+	return wd.httpPOST(data, "/gtf/automation/log")
+}
+
+func (wd *wdaDriver) StartCaptureLog(identifier ...string) error {
+	log.Info().Msg("start WDA log recording")
+	if identifier == nil {
+		identifier = []string{""}
+	}
+	data := map[string]interface{}{"action": "start", "type": 2, "identifier": identifier[0]}
+	_, err := wd.triggerWDALog(data)
+	if err != nil {
+		return errors.Wrap(err, "failed to start WDA log recording")
+	}
+
+	return nil
+}
+
+type wdaResponse struct {
+	Value     interface{} `json:"value"`
+	SessionID string      `json:"sessionId"`
+}
+
+func (wd *wdaDriver) StopCaptureLog() (result interface{}, err error) {
+	log.Info().Msg("stop log recording")
+	data := map[string]interface{}{"action": "stop"}
+	rawResp, err := wd.triggerWDALog(data)
+	if err != nil {
+		log.Error().Err(err).Bytes("rawResp", rawResp).Msg("failed to get WDA logs")
+		return "", errors.Wrap(err, "failed to get WDA logs")
+	}
+	reply := new(wdaResponse)
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		log.Error().Err(err).Bytes("rawResp", rawResp).Msg("failed to json.Unmarshal WDA logs")
+		return reply, err
+	}
+	log.Info().Interface("value", reply.Value).Msg("get WDA log response")
+	return reply.Value, nil
+}

hrp/pkg/uixt/ios_element.go

@@ -0,0 +1,484 @@
+package uixt
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+// All elements returned by search endpoints have assigned element_id.
+// Given element_id you can query properties like:
+// enabled, rect, size, location, text, displayed, accessible, name
+type wdaElement struct {
+	parent *wdaDriver
+	id     string // element_id
+}
+
+func (we wdaElement) Click() (err error) {
+	// [[FBRoute POST:@"/element/:uuid/click"] respondWithTarget:self action:@selector(handleClick:)]
+	_, err = we.parent.httpPOST(nil, "/session", we.parent.sessionId, "/element", we.id, "/click")
+	return
+}
+
+func (we wdaElement) SendKeys(text string, options ...DataOption) (err error) {
+	// [[FBRoute POST:@"/element/:uuid/value"] respondWithTarget:self action:@selector(handleSetValue:)]
+	data := map[string]interface{}{
+		"value": strings.Split(text, ""),
+	}
+	// append options in post data for extra uiautomator configurations
+	for _, option := range options {
+		option(data)
+	}
+
+	if _, ok := data["frequency"]; !ok {
+		data["frequency"] = 60
+	}
+
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/element", we.id, "/value")
+	return
+}
+
+func (we wdaElement) Clear() (err error) {
+	// [[FBRoute POST:@"/element/:uuid/clear"] respondWithTarget:self action:@selector(handleClear:)]
+	_, err = we.parent.httpPOST(nil, "/session", we.parent.sessionId, "/element", we.id, "/clear")
+	return
+}
+
+func (we wdaElement) Tap(x, y int) error {
+	return we.TapFloat(float64(x), float64(y))
+}
+
+func (we wdaElement) TapFloat(x, y float64) (err error) {
+	// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
+	data := map[string]interface{}{
+		"x": x,
+		"y": y,
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/tap/", we.id)
+	return
+}
+
+func (we wdaElement) DoubleTap() (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTap:)]
+	_, err = we.parent.httpPOST(nil, "/session", we.parent.sessionId, "/wda/element", we.id, "/doubleTap")
+	return
+}
+
+func (we wdaElement) TouchAndHold(second ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/touchAndHold"] respondWithTarget:self action:@selector(handleTouchAndHold:)]
+	data := make(map[string]interface{})
+	if len(second) == 0 || second[0] <= 0 {
+		second = []float64{1.0}
+	}
+	data["duration"] = second[0]
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/touchAndHold")
+	return
+}
+
+func (we wdaElement) TwoFingerTap() (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/twoFingerTap"] respondWithTarget:self action:@selector(handleTwoFingerTap:)]
+	_, err = we.parent.httpPOST(nil, "/session", we.parent.sessionId, "/wda/element", we.id, "/twoFingerTap")
+	return
+}
+
+func (we wdaElement) TapWithNumberOfTaps(numberOfTaps, numberOfTouches int) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/tapWithNumberOfTaps"] respondWithTarget:self action:@selector(handleTapWithNumberOfTaps:)]
+	if numberOfTouches <= 0 {
+		return errors.New("'numberOfTouches' must be greater than zero")
+	}
+	if numberOfTouches > 5 {
+		return errors.New("'numberOfTouches' cannot be greater than 5")
+	}
+	if numberOfTaps <= 0 {
+		return errors.New("'numberOfTaps' must be greater than zero")
+	}
+	if numberOfTaps > 10 {
+		return errors.New("'numberOfTaps' cannot be greater than 10")
+	}
+	data := map[string]interface{}{
+		"numberOfTaps":    numberOfTaps,
+		"numberOfTouches": numberOfTouches,
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/tapWithNumberOfTaps")
+	return
+}
+
+func (we wdaElement) ForceTouch(pressure float64, second ...float64) (err error) {
+	return we.ForceTouchFloat(-1, -1, pressure, second...)
+}
+
+func (we wdaElement) ForceTouchFloat(x, y, pressure float64, second ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/forceTouch"] respondWithTarget:self action:@selector(handleForceTouch:)]
+	data := make(map[string]interface{})
+	if x != -1 && y != -1 {
+		data["x"] = x
+		data["y"] = y
+	}
+	if len(second) == 0 || second[0] <= 0 {
+		second = []float64{1.0}
+	}
+	data["pressure"] = pressure
+	data["duration"] = second[0]
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/forceTouch")
+	return
+}
+
+func (we wdaElement) Drag(fromX, fromY, toX, toY int, pressForDuration ...float64) error {
+	return we.DragFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), pressForDuration...)
+}
+
+func (we wdaElement) DragFloat(fromX, fromY, toX, toY float64, pressForDuration ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDrag:)]
+	data := map[string]interface{}{
+		"fromX": fromX,
+		"fromY": fromY,
+		"toX":   toX,
+		"toY":   toY,
+	}
+	if len(pressForDuration) == 0 || pressForDuration[0] < 0 {
+		pressForDuration = []float64{1.0}
+	}
+	data["duration"] = pressForDuration[0]
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/dragfromtoforduration")
+	return
+}
+
+func (we wdaElement) Swipe(fromX, fromY, toX, toY int) error {
+	return we.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY))
+}
+
+func (we wdaElement) SwipeFloat(fromX, fromY, toX, toY float64) error {
+	return we.DragFloat(fromX, fromY, toX, toY, 0)
+}
+
+func (we wdaElement) SwipeDirection(direction Direction, velocity ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/swipe"] respondWithTarget:self action:@selector(handleSwipe:)]
+	data := map[string]interface{}{"direction": direction}
+	if len(velocity) != 0 && velocity[0] > 0 {
+		data["velocity"] = velocity[0]
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/swipe")
+	return
+}
+
+func (we wdaElement) Pinch(scale, velocity float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/pinch"] respondWithTarget:self action:@selector(handlePinch:)]
+	if scale <= 0 {
+		return errors.New("'scale' must be greater than zero")
+	}
+	if scale == 1 {
+		return errors.New("'scale' must be greater or less than 1")
+	}
+	if scale < 1 && velocity > 0 {
+		return errors.New("'velocity' must be less than zero when 'scale' is less than 1")
+	}
+	if scale > 1 && velocity <= 0 {
+		return errors.New("'velocity' must be greater than zero when 'scale' is greater than 1")
+	}
+	data := map[string]interface{}{
+		"scale":    scale,
+		"velocity": velocity,
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/pinch")
+	return
+}
+
+func (we wdaElement) PinchToZoomOutByW3CAction(scale ...float64) (err error) {
+	if len(scale) == 0 {
+		scale = []float64{1.0}
+	} else if scale[0] > 23 {
+		scale[0] = 23
+	}
+	var size Size
+	if size, err = we.Size(); err != nil {
+		return err
+	}
+	r := scale[0] * 2 / 100.0
+	offsetX, offsetY := float64(size.Width)*r, float64(size.Height)*r
+
+	actions := NewW3CActions().SwipeFloat(0-offsetX, 0-offsetY, 0, 0, we).SwipeFloat(offsetX, offsetY, 0, 0, we)
+	return we.parent.PerformW3CActions(actions)
+}
+
+func (we wdaElement) Rotate(rotation float64, velocity ...float64) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/rotate"] respondWithTarget:self action:@selector(handleRotate:)]
+	if rotation > math.Pi*2 || rotation < math.Pi*-2 {
+		return errors.New("'rotation' must not be more than 2π or less than -2π")
+	}
+	if len(velocity) == 0 || velocity[0] == 0 {
+		velocity = []float64{rotation}
+	}
+	if rotation > 0 && velocity[0] < 0 || rotation < 0 && velocity[0] > 0 {
+		return errors.New("'rotation' and 'velocity' must have the same sign")
+	}
+	data := map[string]interface{}{
+		"rotation": rotation,
+		"velocity": velocity[0],
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/rotate")
+	return
+}
+
+func (we wdaElement) PickerWheelSelect(order PickerWheelOrder, offset ...int) (err error) {
+	// [[FBRoute POST:@"/wda/pickerwheel/:uuid/select"] respondWithTarget:self action:@selector(handleWheelSelect:)]
+	if len(offset) == 0 {
+		offset = []int{2}
+	} else if offset[0] <= 0 || offset[0] > 5 {
+		return fmt.Errorf("'offset' value is expected to be in range (0, 5]. '%d' was given instead", offset[0])
+	}
+	data := map[string]interface{}{
+		"order":  order,
+		"offset": float64(offset[0]) * 0.1,
+	}
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/pickerwheel", we.id, "/select")
+	return
+}
+
+func (we wdaElement) scroll(data interface{}) (err error) {
+	// [[FBRoute POST:@"/wda/element/:uuid/scroll"] respondWithTarget:self action:@selector(handleScroll:)]
+	_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/wda/element", we.id, "/scroll")
+	return
+}
+
+func (we wdaElement) ScrollElementByName(name string) error {
+	data := map[string]interface{}{"name": name}
+	return we.scroll(data)
+}
+
+func (we wdaElement) ScrollElementByPredicate(predicate string) error {
+	data := map[string]interface{}{"predicateString": predicate}
+	return we.scroll(data)
+}
+
+func (we wdaElement) ScrollToVisible() error {
+	data := map[string]interface{}{"toVisible": true}
+	return we.scroll(data)
+}
+
+func (we wdaElement) ScrollDirection(direction Direction, distance ...float64) error {
+	if len(distance) == 0 || distance[0] <= 0 {
+		distance = []float64{0.5}
+	}
+	data := map[string]interface{}{
+		"direction": direction,
+		"distance":  distance[0],
+	}
+	return we.scroll(data)
+}
+
+func (we wdaElement) FindElement(by BySelector) (element WebElement, err error) {
+	// [[FBRoute POST:@"/element/:uuid/element"] respondWithTarget:self action:@selector(handleFindSubElement:)]
+	using, value := by.getUsingAndValue()
+	data := map[string]interface{}{
+		"using": using,
+		"value": value,
+	}
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/element", we.id, "/element"); err != nil {
+		return nil, err
+	}
+	var elementID string
+	if elementID, err = rawResp.valueConvertToElementID(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find an element using '%s', value '%s'", err, using, value)
+		}
+		return nil, err
+	}
+	element = &wdaElement{parent: we.parent, id: elementID}
+	return
+}
+
+func (we wdaElement) FindElements(by BySelector) (elements []WebElement, err error) {
+	// [[FBRoute POST:@"/element/:uuid/elements"] respondWithTarget:self action:@selector(handleFindSubElements:)]
+	using, value := by.getUsingAndValue()
+	data := map[string]interface{}{
+		"using": using,
+		"value": value,
+	}
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/element", we.id, "/elements"); err != nil {
+		return nil, err
+	}
+	var elementIDs []string
+	if elementIDs, err = rawResp.valueConvertToElementIDs(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find an element using '%s', value '%s'", err, using, value)
+		}
+		return nil, err
+	}
+	elements = make([]WebElement, len(elementIDs))
+	for i := range elementIDs {
+		elements[i] = &wdaElement{parent: we.parent, id: elementIDs[i]}
+	}
+	return
+}
+
+func (we wdaElement) FindVisibleCells() (elements []WebElement, err error) {
+	// [[FBRoute GET:@"/wda/element/:uuid/getVisibleCells"] respondWithTarget:self action:@selector(handleFindVisibleCells:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/wda/element", we.id, "/getVisibleCells"); err != nil {
+		return nil, err
+	}
+	var elementIDs []string
+	if elementIDs, err = rawResp.valueConvertToElementIDs(); err != nil {
+		if errors.Is(err, errNoSuchElement) {
+			return nil, fmt.Errorf("%w: unable to find a cell element in this element", err)
+		}
+		return nil, err
+	}
+	elements = make([]WebElement, len(elementIDs))
+	for i := range elementIDs {
+		elements[i] = &wdaElement{parent: we.parent, id: elementIDs[i]}
+	}
+	return
+}
+
+func (we wdaElement) Rect() (rect Rect, err error) {
+	// [[FBRoute GET:@"/element/:uuid/rect"] respondWithTarget:self action:@selector(handleGetRect:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/rect"); err != nil {
+		return Rect{}, err
+	}
+	reply := new(struct{ Value struct{ Rect } })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return Rect{}, err
+	}
+	rect = reply.Value.Rect
+	return
+}
+
+func (we wdaElement) Location() (Point, error) {
+	rect, err := we.Rect()
+	if err != nil {
+		return Point{}, err
+	}
+	return rect.Point, nil
+}
+
+func (we wdaElement) Size() (Size, error) {
+	rect, err := we.Rect()
+	if err != nil {
+		return Size{}, err
+	}
+	return rect.Size, nil
+}
+
+func (we wdaElement) Text() (text string, err error) {
+	// [[FBRoute GET:@"/element/:uuid/text"] respondWithTarget:self action:@selector(handleGetText:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/text"); err != nil {
+		return "", err
+	}
+	if text, err = rawResp.valueConvertToString(); err != nil {
+		return "", err
+	}
+	return
+}
+
+func (we wdaElement) Type() (elemType string, err error) {
+	// [[FBRoute GET:@"/element/:uuid/name"] respondWithTarget:self action:@selector(handleGetName:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/name"); err != nil {
+		return "", err
+	}
+	if elemType, err = rawResp.valueConvertToString(); err != nil {
+		return "", err
+	}
+	return
+}
+
+func (we wdaElement) IsEnabled() (enabled bool, err error) {
+	// [[FBRoute GET:@"/element/:uuid/enabled"] respondWithTarget:self action:@selector(handleGetEnabled:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/enabled"); err != nil {
+		return false, err
+	}
+	if enabled, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (we wdaElement) IsDisplayed() (displayed bool, err error) {
+	// [[FBRoute GET:@"/element/:uuid/displayed"] respondWithTarget:self action:@selector(handleGetDisplayed:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/displayed"); err != nil {
+		return false, err
+	}
+	if displayed, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (we wdaElement) IsSelected() (selected bool, err error) {
+	// [[FBRoute GET:@"/element/:uuid/selected"] respondWithTarget:self action:@selector(handleGetSelected:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/selected"); err != nil {
+		return false, err
+	}
+	if selected, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (we wdaElement) IsAccessible() (accessible bool, err error) {
+	// [[FBRoute GET:@"/wda/element/:uuid/accessible"] respondWithTarget:self action:@selector(handleGetAccessible:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/wda/element", we.id, "/accessible"); err != nil {
+		return false, err
+	}
+	if accessible, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (we wdaElement) IsAccessibilityContainer() (isAccessibilityContainer bool, err error) {
+	// [[FBRoute GET:@"/wda/element/:uuid/accessibilityContainer"] respondWithTarget:self action:@selector(handleGetIsAccessibilityContainer:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/wda/element", we.id, "/accessibilityContainer"); err != nil {
+		return false, err
+	}
+	if isAccessibilityContainer, err = rawResp.valueConvertToBool(); err != nil {
+		return false, err
+	}
+	return
+}
+
+func (we wdaElement) GetAttribute(attr ElementAttribute) (value string, err error) {
+	// [[FBRoute GET:@"/element/:uuid/attribute/:name"] respondWithTarget:self action:@selector(handleGetAttribute:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/attribute", attr.getAttributeName()); err != nil {
+		return "", err
+	}
+	if value, err = rawResp.valueConvertToString(); err != nil {
+		return "", err
+	}
+	return
+}
+
+func (we wdaElement) UID() (uid string) {
+	return we.id
+}
+
+func (we wdaElement) Screenshot() (raw *bytes.Buffer, err error) {
+	// W3C element screenshot
+	// [[FBRoute GET:@"/element/:uuid/screenshot"] respondWithTarget:self action:@selector(handleElementScreenshot:)]
+	// JSONWP element screenshot
+	// [[FBRoute GET:@"/screenshot/:uuid"] respondWithTarget:self action:@selector(handleElementScreenshot:)]
+	var rawResp rawResponse
+	if rawResp, err = we.parent.httpGET("/session", we.parent.sessionId, "/element", we.id, "/screenshot"); err != nil {
+		return nil, err
+	}
+	if raw, err = rawResp.valueDecodeAsBase64(); err != nil {
+		return nil, err
+	}
+	return
+}

hrp/pkg/uixt/ocr_off.go

@@ -0,0 +1,10 @@
+//go:build !ocr
+
+package uixt
+
+import "github.com/rs/zerolog/log"
+
+func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
+	log.Fatal().Msg("OCR is not supported")
+	return
+}

hrp/pkg/uixt/ocr_on.go

@@ -0,0 +1,179 @@
+//go:build ocr
+
+package uixt
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"image"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
+)
+
+var client = &http.Client{
+	Timeout: time.Second * 10,
+}
+
+type OCRResult struct {
+	Text   string   `json:"text"`
+	Points []PointF `json:"points"`
+}
+
+type ResponseOCR struct {
+	Code      int         `json:"code"`
+	Message   string      `json:"message"`
+	OCRResult []OCRResult `json:"ocrResult"`
+}
+
+type veDEMOCRService struct{}
+
+func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) {
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	bodyWriter.WriteField("withDet", "true")
+	// bodyWriter.WriteField("timestampOnly", "true")
+
+	formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png")
+	if err != nil {
+		return nil, fmt.Errorf("create form file error: %v", err)
+	}
+	_, err = formWriter.Write(imageBuf)
+	if err != nil {
+		return nil, fmt.Errorf("write form error: %v", err)
+	}
+
+	err = bodyWriter.Close()
+	if err != nil {
+		return nil, fmt.Errorf("close body writer error: %v", err)
+	}
+
+	url, _ := base64.StdEncoding.DecodeString("aHR0cHM6Ly9ndGZ0YXNrLmJ5dGVkYW5jZS5jb20vYXBpL3YxL2FsZ29yaXRobS9vY3I=")
+	req, err := http.NewRequest("POST", string(url), bodyBuf)
+	if err != nil {
+		return nil, fmt.Errorf("construct request error: %v", err)
+	}
+
+	req.Header.Add("Content-Type", bodyWriter.FormDataContentType())
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http reqeust OCR server error: %v", err)
+	}
+	defer resp.Body.Close()
+
+	results, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response body error: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response status code: %d, results: %v", resp.StatusCode, string(results))
+	}
+
+	var ocrResult ResponseOCR
+	err = json.Unmarshal(results, &ocrResult)
+	if err != nil {
+		return nil, fmt.Errorf("json unmarshal response body error: %v", err)
+	}
+
+	return ocrResult.OCRResult, nil
+}
+
+func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) {
+	if len(index) == 0 {
+		index = []int{0} // index not specified
+	}
+
+	ocrResults, err := s.getOCRResult(imageBuf)
+	if err != nil {
+		log.Error().Err(err).Msg("getOCRResult failed")
+		return
+	}
+
+	var rects []image.Rectangle
+	var ocrTexts []string
+	for _, ocrResult := range ocrResults {
+		ocrTexts = append(ocrTexts, ocrResult.Text)
+
+		// not contains text
+		if !strings.Contains(ocrResult.Text, text) {
+			continue
+		}
+
+		rect = image.Rectangle{
+			// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
+			Min: image.Point{
+				X: int(ocrResult.Points[0].X),
+				Y: int(ocrResult.Points[0].Y),
+			},
+			Max: image.Point{
+				X: int(ocrResult.Points[2].X),
+				Y: int(ocrResult.Points[2].Y),
+			},
+		}
+		rects = append(rects, rect)
+
+		// contains text while not match exactly
+		if ocrResult.Text != text {
+			continue
+		}
+
+		// match exactly, and not specify index, return the first one
+		if index[0] == 0 {
+			return rect, nil
+		}
+	}
+
+	if len(rects) == 0 {
+		return image.Rectangle{},
+			fmt.Errorf("text %s not found in %v", text, ocrTexts)
+	}
+
+	// get index
+	idx := index[0]
+	if idx > 0 {
+		// NOTICE: index start from 1
+		idx = idx - 1
+	} else if idx < 0 {
+		idx = len(rects) + idx
+	}
+
+	// index out of range
+	if idx >= len(rects) {
+		return image.Rectangle{}, fmt.Errorf("text %s found %d, index %d out of range",
+			text, len(rects), idx)
+	}
+
+	return rects[idx], nil
+}
+
+type OCRService interface {
+	FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
+}
+
+func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
+	var bufSource *bytes.Buffer
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		err = fmt.Errorf("takeScreenShot error: %v", err)
+		return
+	}
+
+	service := &veDEMOCRService{}
+	rect, err := service.FindText(ocrText, bufSource.Bytes(), index...)
+	if err != nil {
+		log.Warn().Msgf("FindText failed: %s", err.Error())
+		err = fmt.Errorf("FindText failed: %v", err)
+		return
+	}
+
+	log.Info().Str("ocrText", ocrText).Msgf("FindText success")
+	x, y, width, height = dExt.MappingToRectInUIKit(rect)
+	return
+}

hrp/pkg/uixt/ocr_test.go

@@ -0,0 +1,18 @@
+//go:build ocr
+
+package uixt
+
+import (
+	"testing"
+)
+
+func TestDriverExtOCR(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	x, y, width, height, err := driverExt.FindTextByOCR("抖音")
+	checkErr(t, err)
+
+	t.Logf("x: %v, y: %v, width: %v, height: %v", x, y, width, height)
+	driverExt.Driver.TapFloat(x+width*0.5, y+height*0.5-20)
+}

hrp/pkg/uixt/opencv_off.go

@@ -0,0 +1,23 @@
+//go:build !opencv
+
+package uixt
+
+import (
+	"image"
+
+	"github.com/rs/zerolog/log"
+)
+
+func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) {
+	return extend(driver)
+}
+
+func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
+	log.Fatal().Msg("opencv is not supported")
+	return
+}
+
+func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
+	log.Fatal().Msg("opencv is not supported")
+	return
+}

hrp/pkg/uixt/opencv_on.go

@@ -0,0 +1,146 @@
+//go:build opencv
+
+package uixt
+
+import (
+	"bytes"
+	"image"
+	"io/ioutil"
+	"os"
+
+	cvHelper "github.com/electricbubble/opencv-helper"
+)
+
+const (
+	// TmCcoeffNormed maps to TM_CCOEFF_NORMED
+	TmCcoeffNormed TemplateMatchMode = iota
+	// TmSqdiff maps to TM_SQDIFF
+	TmSqdiff
+	// TmSqdiffNormed maps to TM_SQDIFF_NORMED
+	TmSqdiffNormed
+	// TmCcorr maps to TM_CCORR
+	TmCcorr
+	// TmCcorrNormed maps to TM_CCORR_NORMED
+	TmCcorrNormed
+	// TmCcoeff maps to TM_CCOEFF
+	TmCcoeff
+)
+
+type DebugMode int
+
+const (
+	// DmOff no output
+	DmOff DebugMode = iota
+	// DmEachMatch output matched and mismatched values
+	DmEachMatch
+	// DmNotMatch output only values that do not match
+	DmNotMatch
+)
+
+// Extend 获得扩展后的 Driver，
+// 并指定匹配阀值，
+// 获取当前设备的 Scale，
+// 默认匹配模式为 TmCcoeffNormed，
+// 默认关闭 OpenCV 匹配值计算后的输出
+func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) {
+	dExt, err = extend(driver)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, option := range options {
+		option(&dExt.CVArgs)
+	}
+
+	if dExt.threshold == 0 {
+		dExt.threshold = 0.95 // default threshold
+	}
+	if dExt.matchMode == 0 {
+		dExt.matchMode = TmCcoeffNormed // default match mode
+	}
+	cvHelper.Debug(cvHelper.DebugMode(DmOff))
+	return
+}
+
+func (dExt *DriverExt) Debug(dm DebugMode) {
+	cvHelper.Debug(cvHelper.DebugMode(dm))
+}
+
+func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
+	newExt = new(DriverExt)
+	newExt.Driver = dExt.Driver
+	newExt.scale = dExt.scale
+	newExt.matchMode = dExt.matchMode
+	newExt.threshold = threshold
+	return
+}
+
+func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
+	newExt = new(DriverExt)
+	newExt.Driver = dExt.Driver
+	newExt.scale = dExt.scale
+	newExt.matchMode = matchMode
+	newExt.threshold = dExt.threshold
+	return
+}
+
+// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
+// 	pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename())
+// 	if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
+// 		return image.Rectangle{}, err
+// 	}
+//
+// 	if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil {
+// 		return image.Rectangle{}, err
+// 	}
+// 	return
+// }
+
+func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
+	var bufSource, bufSearch *bytes.Buffer
+	if bufSearch, err = getBufFromDisk(search); err != nil {
+		return nil, err
+	}
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		return nil, err
+	}
+
+	if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil {
+		return nil, err
+	}
+	return
+}
+
+func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
+	var bufSource, bufSearch *bytes.Buffer
+	if bufSearch, err = getBufFromDisk(imagePath); err != nil {
+		return 0, 0, 0, 0, err
+	}
+	if bufSource, err = dExt.takeScreenShot(); err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	var rect image.Rectangle
+	if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil {
+		return 0, 0, 0, 0, err
+	}
+
+	// if rect, err = dExt.findImgRect(search); err != nil {
+	// 	return 0, 0, 0, 0, err
+	// }
+	x, y, width, height = dExt.MappingToRectInUIKit(rect)
+	return
+}
+
+func getBufFromDisk(name string) (*bytes.Buffer, error) {
+	var f *os.File
+	var err error
+	if f, err = os.Open(name); err != nil {
+		return nil, err
+	}
+	var all []byte
+	if all, err = ioutil.ReadAll(f); err != nil {
+		return nil, err
+	}
+	return bytes.NewBuffer(all), nil
+}

hrp/pkg/uixt/swipe.go

@@ -0,0 +1,86 @@
+package uixt
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog/log"
+)
+
+func assertRelative(p float64) bool {
+	return p >= 0 && p <= 1
+}
+
+// SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY]
+func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, identifier ...string) error {
+	width := dExt.windowSize.Width
+	height := dExt.windowSize.Height
+
+	if !assertRelative(fromX) || !assertRelative(fromY) ||
+		!assertRelative(toX) || !assertRelative(toY) {
+		return fmt.Errorf("fromX(%f), fromY(%f), toX(%f), toY(%f) must be less than 1",
+			fromX, fromY, toX, toY)
+	}
+
+	fromX = float64(width) * fromX
+	fromY = float64(height) * fromY
+	toX = float64(width) * toX
+	toY = float64(height) * toY
+
+	if len(identifier) > 0 && identifier[0] != "" {
+		option := WithCustomOption("log", map[string]interface{}{
+			"enable": true,
+			"data":   identifier[0],
+		})
+		return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, option)
+	}
+	return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY)
+}
+
+func (dExt *DriverExt) SwipeTo(direction string, identifier ...string) (err error) {
+	switch direction {
+	case "up":
+		return dExt.SwipeUp(identifier...)
+	case "down":
+		return dExt.SwipeDown(identifier...)
+	case "left":
+		return dExt.SwipeLeft(identifier...)
+	case "right":
+		return dExt.SwipeRight(identifier...)
+	}
+	return fmt.Errorf("unexpected direction: %s", direction)
+}
+
+func (dExt *DriverExt) SwipeUp(identifier ...string) (err error) {
+	return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, identifier...)
+}
+
+func (dExt *DriverExt) SwipeDown(identifier ...string) (err error) {
+	return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, identifier...)
+}
+
+func (dExt *DriverExt) SwipeLeft(identifier ...string) (err error) {
+	return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, identifier...)
+}
+
+func (dExt *DriverExt) SwipeRight(identifier ...string) (err error) {
+	return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, identifier...)
+}
+
+// FindCondition indicates the condition to find a UI element
+type FindCondition func(driver *DriverExt) error
+
+// FoundAction indicates the action to do after a UI element is found
+type FoundAction func(driver *DriverExt) error
+
+func (dExt *DriverExt) SwipeUntil(direction string, condition FindCondition, action FoundAction, maxTimes int) error {
+	for i := 0; i < maxTimes; i++ {
+		if err := condition(dExt); err == nil {
+			// do action after found
+			return action(dExt)
+		}
+		if err := dExt.SwipeTo(direction); err != nil {
+			log.Error().Err(err).Msgf("swipe %s failed", direction)
+		}
+	}
+	return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes)
+}

hrp/pkg/uixt/swipe_test.go

@@ -0,0 +1,48 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"testing"
+)
+
+func TestSwipeUntil(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	var point PointF
+	findApp := func(d *DriverExt) error {
+		var err error
+		point, err = d.GetTextXY("抖音")
+		return err
+	}
+	foundAppAction := func(d *DriverExt) error {
+		// click app, launch douyin
+		return d.TapAbsXY(point.X, point.Y, "")
+	}
+
+	driverExt.Driver.Homescreen()
+
+	// swipe to first screen
+	for i := 0; i < 5; i++ {
+		driverExt.SwipeRight()
+	}
+
+	// swipe until app found
+	err = driverExt.SwipeUntil("left", findApp, foundAppAction, 10)
+	checkErr(t, err)
+
+	findLive := func(d *DriverExt) error {
+		var err error
+		point, err = d.GetTextXY("点击进入直播间")
+		return err
+	}
+	foundLiveAction := func(d *DriverExt) error {
+		// enter live room
+		return d.TapAbsXY(point.X, point.Y, "")
+	}
+
+	// swipe until live room found
+	err = driverExt.SwipeUntil("up", findLive, foundLiveAction, 20)
+	checkErr(t, err)
+}

hrp/pkg/uixt/tap.go

@@ -0,0 +1,152 @@
+package uixt
+
+import (
+	"fmt"
+)
+
+func (dExt *DriverExt) TapAbsXY(x, y float64, identifier string) error {
+	// tap on absolute coordinate [x, y]
+	if len(identifier) > 0 {
+		option := WithCustomOption("log", map[string]interface{}{
+			"enable": true,
+			"data":   identifier,
+		})
+		return dExt.Driver.TapFloat(x, y, option)
+	}
+	return dExt.Driver.TapFloat(x, y)
+}
+
+func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
+	// tap on [x, y] percent of window size
+	if x > 1 || y > 1 {
+		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
+	}
+
+	x = x * float64(dExt.windowSize.Width)
+	y = y * float64(dExt.windowSize.Height)
+
+	return dExt.TapAbsXY(x, y, identifier)
+}
+
+func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, err error) {
+	x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
+	if err != nil {
+		return PointF{}, err
+	}
+
+	point = PointF{
+		X: x + width*0.5,
+		Y: y + height*0.5,
+	}
+	return point, nil
+}
+
+func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) {
+	x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...)
+	if err != nil {
+		return PointF{}, err
+	}
+
+	point = PointF{
+		X: x + width*0.5,
+		Y: y + height*0.5,
+	}
+	return point, nil
+}
+
+func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
+	point, err := dExt.GetTextXY(ocrText, index...)
+	if err != nil {
+		if ignoreNotFoundError {
+			return nil
+		}
+		return err
+	}
+
+	return dExt.TapAbsXY(point.X, point.Y, identifier)
+}
+
+func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
+	point, err := dExt.GetImageXY(imagePath, index...)
+	if err != nil {
+		if ignoreNotFoundError {
+			return nil
+		}
+		return err
+	}
+
+	return dExt.TapAbsXY(point.X, point.Y, identifier)
+}
+
+func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
+	return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...)
+}
+
+func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) {
+	// click on element, find by name attribute
+	ele, err := dExt.FindUIElement(param)
+	if err == nil {
+		return ele.Click()
+	}
+
+	x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...)
+	if err != nil {
+		if ignoreNotFoundError {
+			return nil
+		}
+		return err
+	}
+
+	return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, identifier)
+}
+
+func (dExt *DriverExt) DoubleTapXY(x, y float64) error {
+	// double tap on coordinate: [x, y] should be relative
+	if x > 1 || y > 1 {
+		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
+	}
+
+	x = x * float64(dExt.windowSize.Width)
+	y = y * float64(dExt.windowSize.Height)
+	return dExt.Driver.DoubleTapFloat(x, y)
+}
+
+func (dExt *DriverExt) DoubleTap(param string) (err error) {
+	return dExt.DoubleTapOffset(param, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) {
+	// click on element, find by name attribute
+	ele, err := dExt.FindUIElement(param)
+	if err == nil {
+		return ele.DoubleTap()
+	}
+
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
+		return err
+	}
+
+	return dExt.Driver.DoubleTapFloat(x+width*xOffset, y+height*yOffset)
+}
+
+// TapWithNumber sends one or more taps
+func (dExt *DriverExt) TapWithNumber(param string, numberOfTaps int) (err error) {
+	return dExt.TapWithNumberOffset(param, numberOfTaps, 0.5, 0.5)
+}
+
+func (dExt *DriverExt) TapWithNumberOffset(param string, numberOfTaps int, xOffset, yOffset float64) (err error) {
+	if numberOfTaps <= 0 {
+		numberOfTaps = 1
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(param); err != nil {
+		return err
+	}
+
+	x = x + width*xOffset
+	y = y + height*yOffset
+
+	touchActions := NewTouchActions().Tap(NewTouchActionTap().WithXYFloat(x, y).WithCount(numberOfTaps))
+	return dExt.PerformTouchActions(touchActions)
+}

hrp/pkg/uixt/tap_test.go

@@ -0,0 +1,45 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"testing"
+)
+
+func TestDriverExt_TapWithNumber(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/flag7.png"
+
+	err = driverExt.TapWithNumber(pathSearch, 3)
+	checkErr(t, err)
+
+	err = driverExt.TapWithNumberOffset(pathSearch, 3, 0.5, 0.75)
+	checkErr(t, err)
+}
+
+func TestDriverExt_TapXY(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	err = driverExt.TapXY(0.4, 0.5, "")
+	checkErr(t, err)
+}
+
+func TestDriverExt_TapAbsXY(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	err = driverExt.TapAbsXY(100, 300, "")
+	checkErr(t, err)
+}
+
+func TestDriverExt_TapWithOCR(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	// 需要点击文字上方的图标
+	err = driverExt.TapOffset("抖音", 0.5, -1, "", false)
+	checkErr(t, err)
+}

hrp/pkg/uixt/touch.go

@@ -0,0 +1,33 @@
+package uixt
+
+func (dExt *DriverExt) ForceTouch(pathname string, pressure float64, duration ...float64) (err error) {
+	return dExt.ForceTouchOffset(pathname, pressure, 0.5, 0.5, duration...)
+}
+
+func (dExt *DriverExt) ForceTouchOffset(pathname string, pressure, xOffset, yOffset float64, duration ...float64) (err error) {
+	if len(duration) == 0 {
+		duration = []float64{1.0}
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	return dExt.Driver.ForceTouchFloat(x+width*xOffset, y+height*yOffset, pressure, duration[0])
+}
+
+func (dExt *DriverExt) TouchAndHold(pathname string, duration ...float64) (err error) {
+	return dExt.TouchAndHoldOffset(pathname, 0.5, 0.5, duration...)
+}
+
+func (dExt *DriverExt) TouchAndHoldOffset(pathname string, xOffset, yOffset float64, duration ...float64) (err error) {
+	if len(duration) == 0 {
+		duration = []float64{1.0}
+	}
+	var x, y, width, height float64
+	if x, y, width, height, err = dExt.FindUIRectInUIKit(pathname); err != nil {
+		return err
+	}
+
+	return dExt.Driver.TouchAndHoldFloat(x+width*xOffset, y+height*yOffset, duration[0])
+}

hrp/pkg/uixt/touch_test.go

@@ -0,0 +1,39 @@
+//go:build localtest
+
+package uixt
+
+import (
+	"testing"
+)
+
+func TestDriverExt_ForceTouch(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png"
+
+	err = driverExt.ForceTouch(pathSearch, 0.5, 3)
+	checkErr(t, err)
+
+	// err = driverExt.ForceTouchOffset(pathSearch, 0.5, 0.1, 0.9)
+	// checkErr(t, err)
+
+	// err = driverExt.ForceTouchOffset(pathSearch, 0.2, 1.1, -1)
+	// checkErr(t, err)
+}
+
+func TestDriverExt_TouchAndHold(t *testing.T) {
+	driverExt, err := InitWDAClient(nil)
+	checkErr(t, err)
+
+	pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_ft.png"
+
+	// err = driverExt.TouchAndHold(pathSearch)
+	// checkErr(t, err)
+
+	// err = driverExt.TouchAndHold(pathSearch, 3)
+	// checkErr(t, err)
+
+	err = driverExt.TouchAndHoldOffset(pathSearch, 0.8, 0.1)
+	checkErr(t, err)
+}