Feat/yuhongzheng/pre auto install

* fix: fix rotate tap swipe error
* fix: default input frequency from 60 to 10
* fix: error getting window size during screen rotation
* fix: kuake input unicode error
* feat: android input by appium ime
* feat: android swipe and tap with duration
* fix: format import file
* fix: format import file
* feat: 新增按控件点击，获取设备应用，修改日志获取
* feat: 新增ui2控件点击
* fix: format file
* fix: format file
* Merge branch 'video-release' into 'feat/yuhongzheng/pre_auto_install'
* merge
* Merge branch 'feat/yuhongzheng/pre_auto_install' of…
* fix: close reader
* Merge branch 'video-release' into 'feat/yuhongzheng/pre_auto_install'
* fix: format code
* Merge branch 'feat/yuhongzheng/pre_auto_install' of…
* fix: test send key

https://code.byted.org/iesqa/httprunner/merge_requests/34

hrp/pkg/uixt/action.go

@@ -297,7 +297,7 @@ func (o *ActionOptions) updateData(data map[string]interface{}) {
 		data["frequency"] = o.Frequency
 	}
 	if _, ok := data["frequency"]; !ok {
-		data["frequency"] = 60 // default frequency
+		data["frequency"] = 10 // default frequency
 	}
 
 	if _, ok := data["replace"]; !ok {
@@ -320,6 +320,11 @@ func NewActionOptions(options ...ActionOption) *ActionOptions {
 	return actionOptions
 }
 
+type TapTextAction struct {
+	Text    string
+	Options []ActionOption
+}
+
 type ActionOption func(o *ActionOptions)
 
 func WithCustomOption(key string, value interface{}) ActionOption {

hrp/pkg/uixt/android_adb_driver.go

@@ -1,25 +1,32 @@
 package uixt
 
 import (
+	"bufio"
 	"bytes"
+	"encoding/xml"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
+	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/httprunner/funplugin/myexec"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
-	"github.com/httprunner/funplugin/myexec"
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
 	"github.com/httprunner/httprunner/v4/hrp/internal/env"
 	"github.com/httprunner/httprunner/v4/hrp/pkg/gadb"
+	"github.com/httprunner/httprunner/v4/hrp/pkg/utf7"
 )
 
-const AdbKeyBoardPackageName = "com.android.adbkeyboard/.AdbIME"
+const (
+	AdbKeyBoardPackageName = "com.android.adbkeyboard/.AdbIME"
+	UnicodeImePackageName  = "io.appium.settings/.UnicodeIME"
+)
 
 type adbDriver struct {
 	Driver
@@ -91,7 +98,14 @@ func (ad *adbDriver) WindowSize() (size Size, err error) {
 			return Size{Width: width, Height: height}, nil
 		}
 	}
-
+	orientation, err := ad.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("window size get orientation failed, use default orientation")
+		orientation = OrientationPortrait
+	}
+	if orientation != OrientationPortrait {
+		size.Width, size.Height = size.Height, size.Width
+	}
 	err = errors.New("physical window size not found by adb")
 	return
 }
@@ -185,6 +199,24 @@ func (ad *adbDriver) StopCamera() (err error) {
 	return
 }
 
+func (ad *adbDriver) Orientation() (orientation Orientation, err error) {
+	output, err := ad.adbClient.RunShellCommand("dumpsys", "input", "|", "grep", "'SurfaceOrientation'")
+	if err != nil {
+		return
+	}
+	re := regexp.MustCompile(`SurfaceOrientation: (\d)`)
+	matches := re.FindStringSubmatch(output)
+	if len(matches) > 1 { // 确保找到了匹配项
+		if matches[1] == "0" || matches[1] == "2" {
+			return OrientationPortrait, nil
+		} else if matches[1] == "1" || matches[1] == "3" {
+			return OrientationLandscapeLeft, nil
+		}
+	}
+	err = fmt.Errorf("not found SurfaceOrientation value")
+	return
+}
+
 func (ad *adbDriver) Homescreen() (err error) {
 	return ad.PressKeyCode(KCHome, KMEmpty)
 }
@@ -326,6 +358,15 @@ func (ad *adbDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe
 }
 
 func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error) {
+	err = ad.SendUnicodeKeys(text, options...)
+	if err == nil {
+		return
+	}
+	err = ad.InputText(text, options...)
+	return
+}
+
+func (ad *adbDriver) InputText(text string, options ...ActionOption) (err error) {
 	// adb shell input text <text>
 	_, err = ad.adbClient.RunShellCommand("input", "text", text)
 	if err != nil {
@@ -334,6 +375,36 @@ func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error)
 	return nil
 }
 
+func (ad *adbDriver) SendUnicodeKeys(text string, options ...ActionOption) (err error) {
+	// If the Unicode IME is not installed, fall back to the old interface.
+	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
+	// In release version: without the Unicode IME installed, the test cannot execute.
+	if !ad.IsUnicodeIMEInstalled() {
+		return fmt.Errorf("appium unicode ime not installed")
+	}
+	currentIme, err := ad.GetIme()
+	if err != nil {
+		return
+	}
+	if currentIme != UnicodeImePackageName {
+		defer func() {
+			_ = ad.SetIme(currentIme)
+		}()
+		err = ad.SetIme(UnicodeImePackageName)
+		if err != nil {
+			log.Warn().Err(err).Msgf("set Unicode Ime failed")
+			return
+		}
+	}
+	encodedStr, err := utf7.Encoding.NewEncoder().String(text)
+	if err != nil {
+		log.Warn().Err(err).Msgf("encode text with modified utf7 failed")
+		return
+	}
+	err = ad.InputText("\""+strings.ReplaceAll(encodedStr, "\"", "\\\"")+"\"", options...)
+	return
+}
+
 func (ad *adbDriver) IsAdbKeyBoardInstalled() bool {
 	output, err := ad.adbClient.RunShellCommand("ime", "list", "-a")
 	if err != nil {
@@ -342,6 +413,14 @@ func (ad *adbDriver) IsAdbKeyBoardInstalled() bool {
 	return strings.Contains(output, AdbKeyBoardPackageName)
 }
 
+func (ad *adbDriver) IsUnicodeIMEInstalled() bool {
+	output, err := ad.adbClient.RunShellCommand("ime", "list", "-s")
+	if err != nil {
+		return false
+	}
+	return strings.Contains(output, UnicodeImePackageName)
+}
+
 func (ad *adbDriver) SendKeysByAdbKeyBoard(text string) (err error) {
 	defer func() {
 		// Reset to default, don't care which keyboard was chosen before switch:
@@ -404,10 +483,103 @@ func (ad *adbDriver) Screenshot() (raw *bytes.Buffer, err error) {
 }
 
 func (ad *adbDriver) Source(srcOpt ...SourceOption) (source string, err error) {
-	err = errDriverNotImplemented
+	_, err = ad.adbClient.RunShellCommand("rm", "-rf", "/sdcard/window_dump.xml")
+	if err != nil {
+		return
+	}
+	// 高版本报错 ERROR: null root node returned by UiTestAutomationBridge.
+	_, err = ad.adbClient.RunShellCommand("uiautomator", "dump")
+	if err != nil {
+		return
+	}
+	source, err = ad.adbClient.RunShellCommand("cat", "/sdcard/window_dump.xml")
+	if err != nil {
+		return
+	}
 	return
 }
 
+func (ad *adbDriver) sourceTree(srcOpt ...SourceOption) (sourceTree *Hierarchy, err error) {
+	source, err := ad.Source()
+	if err != nil {
+		return
+	}
+	sourceTree = new(Hierarchy)
+	err = xml.Unmarshal([]byte(source), sourceTree)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (ad *adbDriver) TapByText(text string, options ...ActionOption) error {
+	sourceTree, err := ad.sourceTree()
+	if err != nil {
+		return err
+	}
+	return ad.tapByTextUsingHierarchy(sourceTree, text, options...)
+}
+
+func (ad *adbDriver) tapByTextUsingHierarchy(hierarchy *Hierarchy, text string, options ...ActionOption) error {
+	bounds := ad.searchNodes(hierarchy.Layout, text, options...)
+	actionOptions := NewActionOptions(options...)
+	if len(bounds) == 0 {
+		if actionOptions.IgnoreNotFoundError {
+			log.Info().Msg("not found element by text " + text)
+			return nil
+		}
+		return errors.New("not found element by text " + text)
+	}
+	for _, bound := range bounds {
+		width, height := bound.Center()
+		err := ad.TapFloat(width, height, options...)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ad *adbDriver) TapByTexts(actions ...TapTextAction) error {
+	sourceTree, err := ad.sourceTree()
+	if err != nil {
+		return err
+	}
+
+	for _, action := range actions {
+		err := ad.tapByTextUsingHierarchy(sourceTree, action.Text, action.Options...)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ad *adbDriver) searchNodes(nodes []Layout, text string, options ...ActionOption) []Bounds {
+	actionOptions := NewActionOptions(options...)
+	var results []Bounds
+	for _, node := range nodes {
+		result := ad.searchNodes(node.Layout, text, options...)
+		results = append(results, result...)
+		if actionOptions.Regex {
+			// regex on, check if match regex
+			if !regexp.MustCompile(text).MatchString(node.Text) {
+				continue
+			}
+		} else {
+			// regex off, check if match exactly
+			if node.Text != text {
+				ad.searchNodes(node.Layout, text, options...)
+				continue
+			}
+		}
+		if node.Bounds != nil {
+			results = append(results, *node.Bounds)
+		}
+	}
+	return results
+}
+
 func (ad *adbDriver) AccessibleSource() (source string, err error) {
 	err = errDriverNotImplemented
 	return
@@ -435,14 +607,8 @@ func (ad *adbDriver) IsHealthy() (healthy bool, err error) {
 
 func (ad *adbDriver) StartCaptureLog(identifier ...string) (err error) {
 	log.Info().Msg("start adb log recording")
-
-	// clear logcat
-	if _, err = ad.adbClient.RunShellCommand("logcat", "-c"); err != nil {
-		return err
-	}
-
 	// start logcat
-	err = ad.logcat.CatchLogcat()
+	err = ad.logcat.CatchLogcat("iesqaMonitor:V")
 	if err != nil {
 		err = errors.Wrap(code.AndroidCaptureLogError,
 			fmt.Sprintf("start adb log recording failed: %v", err))
@@ -452,17 +618,18 @@ func (ad *adbDriver) StartCaptureLog(identifier ...string) (err error) {
 }
 
 func (ad *adbDriver) StopCaptureLog() (result interface{}, err error) {
-	log.Info().Msg("stop adb log recording")
-	err = ad.logcat.Stop()
+	defer func() {
+		log.Info().Msg("stop adb log recording")
+		err = ad.logcat.Stop()
+		if err != nil {
+			log.Error().Err(err).Msg("failed to get adb log recording")
+		}
+	}()
 	if err != nil {
-		log.Error().Err(err).Msg("failed to get adb log recording")
-		err = errors.Wrap(code.AndroidCaptureLogError,
-			fmt.Sprintf("get adb log recording failed: %v", err))
-		return "", err
+		log.Error().Err(err).Msg("failed to close adb log writer")
 	}
-	content := ad.logcat.logBuffer.String()
-	log.Info().Str("logcat content", content).Msg("display logcat content")
-	pointRes := ConvertPoints(content)
+	pointRes := ConvertPoints(ad.logcat.logs)
+
 	// 没有解析到打点日志，走兜底逻辑
 	if len(pointRes) == 0 {
 		log.Info().Msg("action log is null, use action file >>>")
@@ -476,7 +643,6 @@ func (ad *adbDriver) StopCaptureLog() (result interface{}, err error) {
 			}
 			return nil
 		})
-
 		// 先保持原有状态码不变，这里不return error
 		if err != nil {
 			log.Error().Err(err).Msg("read log file fail")
@@ -488,13 +654,28 @@ func (ad *adbDriver) StopCaptureLog() (result interface{}, err error) {
 			return pointRes, nil
 		}
 
-		data, err := ioutil.ReadFile(files[0])
+		reader, err := os.Open(files[0])
 		if err != nil {
-			log.Info().Msg("read File error")
+			log.Info().Msg("open File error")
+			return pointRes, nil
+		}
+		defer func() {
+			_ = reader.Close()
+		}()
+
+		var lines []string // 创建一个空的字符串数组来存储文件的每一行
+
+		// 使用 bufio.NewScanner 读取文件
+		scanner := bufio.NewScanner(reader)
+		for scanner.Scan() {
+			lines = append(lines, scanner.Text()) // 将每行文本添加到字符串数组
+		}
+
+		if err := scanner.Err(); err != nil {
 			return pointRes, nil
 		}
 
-		pointRes = ConvertPoints(string(data))
+		pointRes = ConvertPoints(lines)
 	}
 	return pointRes, nil
 }
@@ -534,6 +715,30 @@ func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) {
 	return AppInfo{}, errors.Wrap(code.MobileUIAssertForegroundAppError, "get foreground app failed")
 }
 
+func (ad *adbDriver) SetIme(ime string) error {
+	_, err := ad.adbClient.RunShellCommand("ime", "set", ime)
+	if err != nil {
+		return err
+	}
+	// even if the shell command has returned,
+	// as there might be a situation where the input method has not been completely switched yet
+	// Listen to the following message.
+	// InputMethodManagerService: onServiceConnected, name:ComponentInfo{io.appium.settings/io.appium.settings.UnicodeIME}, token:android.os.Binder@44f825
+	// But there is no such log on Vivo.
+	time.Sleep(3 * time.Second)
+	return nil
+}
+
+func (ad *adbDriver) GetIme() (ime string, err error) {
+	currentIme, err := ad.adbClient.RunShellCommand("settings", "get", "secure", "default_input_method")
+	if err != nil {
+		log.Warn().Err(err).Msgf("get default ime failed")
+		return
+	}
+	currentIme = strings.TrimSpace(currentIme)
+	return currentIme, nil
+}
+
 func (ad *adbDriver) AssertForegroundApp(packageName string, activityType ...string) error {
 	log.Debug().Str("package_name", packageName).
 		Strs("activity_type", activityType).

hrp/pkg/uixt/android_device.go

@@ -1,6 +1,7 @@
 package uixt
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"fmt"
@@ -22,7 +23,6 @@ var (
 	AdbServerPort  = gadb.AdbServerPort // 5037
 	UIA2ServerHost = "localhost"
 	UIA2ServerPort = 6790
-	DeviceTempPath = "/data/local/tmp"
 )
 
 const forwardToPrefix = "forward-to-"
@@ -276,27 +276,43 @@ func getFreePort() (int, error) {
 	return l.Addr().(*net.TCPAddr).Port, nil
 }
 
+type LineCallback func(string)
+
 type AdbLogcat struct {
-	serial    string
-	logBuffer *bytes.Buffer
-	errs      []error
-	stopping  chan struct{}
-	done      chan struct{}
-	cmd       *exec.Cmd
+	serial string
+	// logBuffer *bytes.Buffer
+	errs     []error
+	stopping chan struct{}
+	done     chan struct{}
+	cmd      *exec.Cmd
+	callback LineCallback
+	logs     []string
+}
+
+func NewAdbLogcatWithCallback(serial string, callback LineCallback) *AdbLogcat {
+	return &AdbLogcat{
+		serial: serial,
+		// logBuffer: new(bytes.Buffer),
+		stopping: make(chan struct{}),
+		done:     make(chan struct{}),
+		callback: callback,
+		logs:     make([]string, 0),
+	}
 }
 
 func NewAdbLogcat(serial string) *AdbLogcat {
 	return &AdbLogcat{
-		serial:    serial,
-		logBuffer: new(bytes.Buffer),
-		stopping:  make(chan struct{}),
-		done:      make(chan struct{}),
+		serial: serial,
+		// logBuffer: new(bytes.Buffer),
+		stopping: make(chan struct{}),
+		done:     make(chan struct{}),
+		logs:     make([]string, 0),
 	}
 }
 
 // CatchLogcatContext starts logcat with timeout context
 func (l *AdbLogcat) CatchLogcatContext(timeoutCtx context.Context) (err error) {
-	if err = l.CatchLogcat(); err != nil {
+	if err = l.CatchLogcat(""); err != nil {
 		return
 	}
 	go func() {
@@ -331,7 +347,7 @@ func (l *AdbLogcat) Errors() (err error) {
 	return
 }
 
-func (l *AdbLogcat) CatchLogcat() (err error) {
+func (l *AdbLogcat) CatchLogcat(filter string) (err error) {
 	if l.cmd != nil {
 		log.Warn().Msg("logcat already start")
 		return nil
@@ -341,33 +357,43 @@ func (l *AdbLogcat) CatchLogcat() (err error) {
 	if err = myexec.RunCommand("adb", "-s", l.serial, "shell", "logcat", "-c"); err != nil {
 		return
 	}
-
+	args := []string{"-s", l.serial, "logcat", "--format", "time"}
+	if filter != "" {
+		args = append(args, "-s", filter)
+	}
 	// start logcat
-	l.cmd = myexec.Command("adb", "-s", l.serial,
-		"logcat", "--format", "time", "-s", "iesqaMonitor:V")
-	l.cmd.Stderr = l.logBuffer
-	l.cmd.Stdout = l.logBuffer
+	l.cmd = myexec.Command("adb", args...)
+	// l.cmd.Stderr = l.logBuffer
+	// l.cmd.Stdout = l.logBuffer
+	reader, err := l.cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
 	if err = l.cmd.Start(); err != nil {
 		return
 	}
+	go func() {
+		scanner := bufio.NewScanner(reader)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if l.callback != nil {
+				l.callback(line) // Process each line with callback
+			} else {
+				l.logs = append(l.logs, line) // Store line if no callback
+			}
+		}
+	}()
 	go func() {
 		<-l.stopping
+		if e := reader.Close(); e != nil {
+			log.Error().Err(e).Msg("close logcat reader failed")
+		}
 		if e := myexec.KillProcessesByGpid(l.cmd); e != nil {
 			log.Error().Err(e).Msg("kill logcat process failed")
 		}
 		l.done <- struct{}{}
 	}()
-	return
-}
 
-func (l *AdbLogcat) BufferedLogcat() (err error) {
-	// -d: dump the current buffered logcat result and exits
-	cmd := myexec.Command("adb", "-s", l.serial, "logcat", "-d")
-	cmd.Stdout = l.logBuffer
-	cmd.Stderr = l.logBuffer
-	if err = cmd.Run(); err != nil {
-		return
-	}
 	return
 }
 
@@ -381,8 +407,8 @@ type ExportPoint struct {
 	RunTime   int         `json:"run_time,omitempty" yaml:"run_time,omitempty"`
 }
 
-func ConvertPoints(data string) (eps []ExportPoint) {
-	lines := strings.Split(data, "\n")
+func ConvertPoints(lines []string) (eps []ExportPoint) {
+	log.Info().Msg("ConvertPoints")
 	for _, line := range lines {
 		if strings.Contains(line, "ext") {
 			idx := strings.Index(line, "{")
@@ -396,6 +422,7 @@ func ConvertPoints(data string) (eps []ExportPoint) {
 				log.Error().Msg("failed to parse point data")
 				continue
 			}
+			log.Info().Msg(line)
 			eps = append(eps, p)
 		}
 	}
@@ -562,7 +589,7 @@ func (s UiSelectorHelper) Index(index int) UiSelectorHelper {
 //
 // For example, to simulate a user click on
 // the third image that is enabled in a UI screen, you
-// could specify a a search criteria where the instance is
+// could specify a search criteria where the instance is
 // 2, the `className(String)` matches the image
 // widget class, and `enabled(boolean)` is true.
 // The code would look like this:

hrp/pkg/uixt/android_layout.go

@@ -0,0 +1,62 @@
+package uixt
+
+import (
+	"encoding/xml"
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type Attributes struct {
+	Index         int     `xml:"index,attr"`
+	Package       string  `xml:"package,attr"`
+	Class         string  `xml:"class,attr"`
+	Text          string  `xml:"text,attr"`
+	ResourceId    string  `xml:"resource-id,attr"`
+	Checkable     bool    `xml:"checkable,attr"`
+	Checked       bool    `xml:"checked,attr"`
+	Clickable     bool    `xml:"clickable,attr"`
+	Enabled       bool    `xml:"enabled,attr"`
+	Focusable     bool    `xml:"focusable,attr"`
+	Focused       bool    `xml:"focused,attr"`
+	LongClickable bool    `xml:"long-clickable,attr"`
+	Password      bool    `xml:"password,attr"`
+	Scrollable    bool    `xml:"scrollable,attr"`
+	Selected      bool    `xml:"selected,attr"`
+	Bounds        *Bounds `xml:"bounds,attr"`
+	Displayed     bool    `xml:"displayed,attr"`
+}
+
+type Hierarchy struct {
+	XMLName xml.Name `xml:"hierarchy"`
+	Attributes
+	Layout []Layout `xml:",any"`
+}
+
+type Layout struct {
+	Attributes
+	Layout []Layout `xml:",any"`
+}
+
+type Bounds struct {
+	X1, Y1, X2, Y2 int
+}
+
+func (b *Bounds) Center() (float64, float64) {
+	return float64(b.X1+b.X2) / 2, float64(b.Y1+b.Y2) / 2
+}
+
+func (b *Bounds) UnmarshalXMLAttr(attr xml.Attr) error {
+	// 正则表达式用于解析格式为"[x1,y1][x2,y2]"
+	re := regexp.MustCompile(`\[(\d+),(\d+)]\[(\d+),(\d+)]`)
+	matches := re.FindStringSubmatch(attr.Value)
+	if matches == nil {
+		return fmt.Errorf("bounds format is incorrect")
+	}
+	// 转换字符串为整数
+	b.X1, _ = strconv.Atoi(matches[1])
+	b.Y1, _ = strconv.Atoi(matches[2])
+	b.X2, _ = strconv.Atoi(matches[3])
+	b.Y2, _ = strconv.Atoi(matches[4])
+	return nil
+}

hrp/pkg/uixt/android_test.go

@@ -6,18 +6,21 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"strings"
 	"testing"
 	"time"
 )
 
 var (
-	uiaServerURL = "http://localhost:6790/wd/hub"
+	uiaServerURL = "http://forward-to-6790:6790/wd/hub"
 	driverExt    *DriverExt
 )
 
 func setupAndroid(t *testing.T) {
 	device, err := NewAndroidDevice()
 	checkErr(t, err)
+	device.UIA2 = false
 	driverExt, err = device.NewDriver()
 	checkErr(t, err)
 }
@@ -132,6 +135,18 @@ func TestDriver_Source(t *testing.T) {
 	t.Log(source)
 }
 
+func TestDriver_TapByText(t *testing.T) {
+	driver, err := NewUIADriver(nil, uiaServerURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = driver.TapByText("安装")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestDriver_BatteryInfo(t *testing.T) {
 	driver, err := NewUIADriver(nil, uiaServerURL)
 	if err != nil {
@@ -204,7 +219,7 @@ func TestDriver_Swipe(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = driver.Swipe(400, 1000, 400, 500)
+	err = driver.Swipe(400, 1000, 400, 500, WithPressDuration(2000))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -215,6 +230,14 @@ func TestDriver_Swipe(t *testing.T) {
 	}
 }
 
+func TestDriver_Swipe_Relative(t *testing.T) {
+	setupAndroid(t)
+	err := driverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestDriver_Drag(t *testing.T) {
 	driver, err := NewUIADriver(nil, uiaServerURL)
 	if err != nil {
@@ -235,28 +258,26 @@ func TestDriver_Drag(t *testing.T) {
 }
 
 func TestDriver_SendKeys(t *testing.T) {
-	driver, err := NewUIADriver(nil, uiaServerURL)
+	setupAndroid(t)
+
+	err := driverExt.Driver.SendKeys("Android\"输入速度测试", WithIdentifier("test"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = driver.SendKeys("abc")
-	if err != nil {
-		t.Fatal(err)
-	}
 	time.Sleep(time.Second * 2)
 
-	err = driver.SendKeys("def")
-	if err != nil {
-		t.Fatal(err)
-	}
-	time.Sleep(time.Second * 2)
+	//err = driver.SendKeys("def")
+	//if err != nil {
+	//	t.Fatal(err)
+	//}
+	//time.Sleep(time.Second * 2)
 
-	err = driver.SendKeys("\\n")
+	//err = driver.SendKeys("\\n")
 	// err = driver.SendKeys(`\n`, false)
-	if err != nil {
-		t.Fatal(err)
-	}
+	//if err != nil {
+	//	t.Fatal(err)
+	//}
 }
 
 func TestDriver_PressBack(t *testing.T) {
@@ -421,10 +442,44 @@ func TestDriver_AppTerminate(t *testing.T) {
 
 func TestConvertPoints(t *testing.T) {
 	data := "10-09 20:16:48.216 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317808206,\"ext\":\"输入\",\"from\":{\"x\":0.0,\"y\":0.0},\"operation\":\"Gtf-SendKeys\",\"run_time\":627,\"start\":1665317807579,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":0.0,\"y\":0.0}}\n10-09 20:18:22.899 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317902898,\"ext\":\"进入直播间\",\"from\":{\"x\":717.0,\"y\":2117.5},\"operation\":\"Gtf-Tap\",\"run_time\":121,\"start\":1665317902777,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":717.0,\"y\":2117.5}}\n10-09 20:18:32.063 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317912062,\"ext\":\"第一次上划\",\"from\":{\"x\":1437.0,\"y\":2409.9},\"operation\":\"Gtf-Swipe\",\"run_time\":32,\"start\":1665317912030,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":1437.0,\"y\":2409.9}}"
-	eps := ConvertPoints(data)
+
+	eps := ConvertPoints(strings.Split(data, "\n"))
 	if len(eps) != 3 {
 		t.Fatal()
 	}
 	jsons, _ := json.Marshal(eps)
 	println(fmt.Sprintf("%v", string(jsons)))
 }
+
+func TestDriver_ShellInputUnicode(t *testing.T) {
+	device, _ := NewAndroidDevice()
+	driver, err := device.NewAdbDriver()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = driver.SendKeys("test中文输入&")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	raw, err := driver.Screenshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(os.WriteFile("s1.png", raw.Bytes(), 0o600))
+}
+
+func TestTapTexts(t *testing.T) {
+	setupAndroid(t)
+	actions := []TapTextAction{
+		{Text: "^.*无视风险安装$", Options: []ActionOption{WithTapOffset(100, 0), WithRegex(true), WithIgnoreNotFoundError(true)}},
+		{Text: "已了解此应用未经检测.*", Options: []ActionOption{WithTapOffset(-450, 0), WithRegex(true), WithIgnoreNotFoundError(true)}},
+		{Text: "^(.*无视风险安装|确定|继续|完成|点击继续安装|继续安装旧版本|替换|安装|授权本次安装|继续安装|重新安装)$", Options: []ActionOption{WithRegex(true), WithIgnoreNotFoundError(true)}},
+	}
+	err := driverExt.Driver.TapByTexts(actions...)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

hrp/pkg/uixt/android_uia2_driver.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/base64"
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"net"
 	"net/http"
@@ -16,6 +17,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
+	"github.com/httprunner/httprunner/v4/hrp/pkg/utf7"
 )
 
 var errDriverNotImplemented = errors.New("driver method not implemented")
@@ -224,6 +226,14 @@ func (ud *uiaDriver) WindowSize() (size Size, err error) {
 		return Size{}, err
 	}
 	size = reply.Value.Size
+	orientation, err := ud.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("window size get orientation failed, use default orientation")
+		orientation = OrientationPortrait
+	}
+	if orientation != OrientationPortrait {
+		size.Width, size.Height = size.Height, size.Width
+	}
 	return
 }
 
@@ -253,6 +263,20 @@ func (ud *uiaDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta, flags ...K
 	return
 }
 
+func (ud *uiaDriver) Orientation() (orientation Orientation, err error) {
+	// [[FBRoute GET:@"/orientation"] respondWithTarget:self action:@selector(handleGetOrientation:)]
+	var rawResp rawResponse
+	if rawResp, err = ud.httpGET("/session", ud.sessionId, "/orientation"); err != nil {
+		return "", err
+	}
+	reply := new(struct{ Value Orientation })
+	if err = json.Unmarshal(rawResp, reply); err != nil {
+		return "", err
+	}
+	orientation = reply.Value
+	return
+}
+
 func (ud *uiaDriver) Tap(x, y int, options ...ActionOption) error {
 	return ud.TapFloat(float64(x), float64(y), options...)
 }
@@ -268,15 +292,31 @@ func (ud *uiaDriver) TapFloat(x, y float64, options ...ActionOption) (err error)
 	x += actionOptions.getRandomOffset()
 	y += actionOptions.getRandomOffset()
 
-	data := map[string]interface{}{
-		"x": x,
-		"y": y,
+	duration := 100.0
+	if actionOptions.PressDuration > 0 {
+		duration = actionOptions.PressDuration
 	}
+	data := map[string]interface{}{
+		"actions": []interface{}{
+			map[string]interface{}{
+				"type":       "pointer",
+				"parameters": map[string]string{"pointerType": "touch"},
+				"id":         "touch",
+				"actions": []interface{}{
+					map[string]interface{}{"type": "pointerMove", "duration": 0, "x": x, "y": y, "origin": "viewport"},
+					map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
+					map[string]interface{}{"type": "pause", "duration": duration},
+					map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
+				},
+			},
+		},
+	}
+
 	// update data options in post data for extra uiautomator configurations
 	actionOptions.updateData(data)
 
-	_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap")
-	return
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "actions/tap")
+	return err
 }
 
 func (ud *uiaDriver) TouchAndHold(x, y int, second ...float64) (err error) {
@@ -358,17 +398,30 @@ func (ud *uiaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...Actio
 	toX += actionOptions.getRandomOffset()
 	toY += actionOptions.getRandomOffset()
 
+	duration := 200.0
+	if actionOptions.PressDuration > 0 {
+		duration = actionOptions.PressDuration
+	}
 	data := map[string]interface{}{
-		"startX": fromX,
-		"startY": fromY,
-		"endX":   toX,
-		"endY":   toY,
+		"actions": []interface{}{
+			map[string]interface{}{
+				"type":       "pointer",
+				"parameters": map[string]string{"pointerType": "touch"},
+				"id":         "touch",
+				"actions": []interface{}{
+					map[string]interface{}{"type": "pointerMove", "duration": 0, "x": fromX, "y": fromY, "origin": "viewport"},
+					map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
+					map[string]interface{}{"type": "pointerMove", "duration": duration, "x": toX, "y": toY, "origin": "viewport"},
+					map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
+				},
+			},
+		},
 	}
 
 	// update data options in post data for extra uiautomator configurations
 	actionOptions.updateData(data)
 
-	_, err := ud.httpPOST(data, "/session", ud.sessionId, "touch/perform")
+	_, err := ud.httpPOST(data, "/session", ud.sessionId, "actions/swipe")
 	return err
 }
 
@@ -415,25 +468,79 @@ func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe
 	return
 }
 
+// SendKeys Android input does not support setting frequency.
 func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) {
 	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
 	// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
 	actionOptions := NewActionOptions(options...)
-	data := map[string]interface{}{
-		"text": text,
-	}
-	// new data options in post data for extra uiautomator configurations
-	actionOptions.updateData(data)
-
-	_, err = ud.httpPOST(data, "/session", ud.sessionId, "keys")
+	err = ud.SendUnicodeKeys(text, options...)
 	if err != nil {
-		// use com.android.adbkeyboard if existed
-		if ud.IsAdbKeyBoardInstalled() {
-			err = ud.SendKeysByAdbKeyBoard(text)
-		} else {
-			_, err = ud.adbClient.RunShellCommand("input", "text", text)
+		data := map[string]interface{}{
+			"text": text,
+		}
+
+		// new data options in post data for extra uiautomator configurations
+		actionOptions.updateData(data)
+
+		_, err = ud.httpPOST(data, "/session", ud.sessionId, "/keys")
+	}
+	return
+}
+
+func (ud *uiaDriver) SendUnicodeKeys(text string, options ...ActionOption) (err error) {
+	// If the Unicode IME is not installed, fall back to the old interface.
+	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
+	// In release version: without the Unicode IME installed, the test cannot execute.
+	if !ud.IsUnicodeIMEInstalled() {
+		return fmt.Errorf("appium unicode ime not installed")
+	}
+	currentIme, err := ud.adbDriver.GetIme()
+	if err != nil {
+		return
+	}
+	if currentIme != UnicodeImePackageName {
+		defer func() {
+			_ = ud.adbDriver.SetIme(currentIme)
+		}()
+		err = ud.adbDriver.SetIme(UnicodeImePackageName)
+		if err != nil {
+			log.Warn().Err(err).Msgf("set Unicode Ime failed")
+			return
 		}
 	}
+	encodedStr, err := utf7.Encoding.NewEncoder().String(text)
+	if err != nil {
+		log.Warn().Err(err).Msgf("encode text with modified utf7 failed")
+		return
+	}
+	err = ud.SendActionKey(encodedStr, options...)
+	return
+}
+
+func (ud *uiaDriver) SendActionKey(text string, options ...ActionOption) (err error) {
+	actionOptions := NewActionOptions(options...)
+	var actions []interface{}
+	for i, c := range text {
+		actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)},
+			map[string]interface{}{"type": "keyUp", "value": string(c)})
+		if i != len(text)-1 {
+			actions = append(actions, map[string]interface{}{"type": "pause", "duration": 40})
+		}
+	}
+
+	data := map[string]interface{}{
+		"actions": []interface{}{
+			map[string]interface{}{
+				"type":    "key",
+				"id":      "key",
+				"actions": actions,
+			},
+		},
+	}
+
+	// new data options in post data for extra uiautomator configurations
+	actionOptions.updateData(data)
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "/actions/keys")
 	return
 }
 
@@ -492,3 +599,39 @@ func (ud *uiaDriver) Source(srcOpt ...SourceOption) (source string, err error) {
 	source = reply.Value
 	return
 }
+
+func (ud *uiaDriver) sourceTree(srcOpt ...SourceOption) (sourceTree *Hierarchy, err error) {
+	source, err := ud.Source()
+	if err != nil {
+		return
+	}
+	sourceTree = new(Hierarchy)
+	err = xml.Unmarshal([]byte(source), sourceTree)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (ud *uiaDriver) TapByText(text string, options ...ActionOption) error {
+	sourceTree, err := ud.sourceTree()
+	if err != nil {
+		return err
+	}
+	return ud.tapByTextUsingHierarchy(sourceTree, text, options...)
+}
+
+func (ud *uiaDriver) TapByTexts(actions ...TapTextAction) error {
+	sourceTree, err := ud.sourceTree()
+	if err != nil {
+		return err
+	}
+
+	for _, action := range actions {
+		err := ud.tapByTextUsingHierarchy(sourceTree, action.Text, action.Options...)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

hrp/pkg/uixt/interface.go

@@ -511,6 +511,10 @@ type WebDriver interface {
 	// since the location service needs some time to update the location data.
 	Location() (Location, error)
 	BatteryInfo() (BatteryInfo, error)
+
+	// WindowSize Return the width and height in portrait mode.
+	// when getting the window size in wda/ui2/adb, if the device is in landscape mode,
+	// the width and height will be reversed.
 	WindowSize() (Size, error)
 	Screen() (Screen, error)
 	Scale() (float64, error)
@@ -537,6 +541,8 @@ type WebDriver interface {
 	// StopCamera Stops the camera for recording
 	StopCamera() error
 
+	Orientation() (orientation Orientation, err error)
+
 	// Tap Sends a tap event at the coordinate.
 	Tap(x, y int, options ...ActionOption) error
 	TapFloat(x, y float64, options ...ActionOption) error
@@ -583,6 +589,11 @@ type WebDriver interface {
 
 	// Source Return application elements tree
 	Source(srcOpt ...SourceOption) (string, error)
+
+	TapByText(text string, options ...ActionOption) error
+
+	TapByTexts(actions ...TapTextAction) error
+
 	// AccessibleSource Return application elements accessibility tree
 	AccessibleSource() (string, error)
 

hrp/pkg/uixt/ios_driver.go

@@ -207,6 +207,14 @@ func (wd *wdaDriver) WindowSize() (size Size, err error) {
 	}
 	size.Height = size.Height * int(scale)
 	size.Width = size.Width * int(scale)
+	orientation, err := wd.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("window size get orientation failed, use default orientation")
+		orientation = OrientationPortrait
+	}
+	if orientation != OrientationPortrait {
+		size.Width, size.Height = size.Height, size.Width
+	}
 	return
 }
 
@@ -547,8 +555,8 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...Action
 
 	// update data options in post data for extra WDA configurations
 	actionOptions.updateData(data)
-
-	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
+	// wda 43 version
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/drag")
 	return
 }
 
@@ -751,6 +759,14 @@ func (wd *wdaDriver) Source(srcOpt ...SourceOption) (source string, err error) {
 	return
 }
 
+func (wd *wdaDriver) TapByText(text string, options ...ActionOption) error {
+	return errDriverNotImplemented
+}
+
+func (wd *wdaDriver) TapByTexts(actions ...TapTextAction) error {
+	return errDriverNotImplemented
+}
+
 func (wd *wdaDriver) AccessibleSource() (source string, err error) {
 	// [[FBRoute GET:@"/wda/accessibleSource"] respondWithTarget:self action:@selector(handleGetAccessibleSourceCommand:)]
 	// [[FBRoute GET:@"/wda/accessibleSource"].withoutSession

hrp/pkg/uixt/ios_test.go

@@ -10,17 +10,23 @@ import (
 )
 
 var (
-	bundleId = "com.apple.Preferences"
-	driver   WebDriver
+	bundleId     = "com.apple.Preferences"
+	driver       WebDriver
+	iOSDriverExt *DriverExt
 )
 
 func setup(t *testing.T) {
-	device, err := NewIOSDevice()
+	device, err := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800), WithWDALogOn(true))
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	driver, err = device.NewUSBDriver(nil)
+	capabilities := NewCapabilities()
+	capabilities.WithDefaultAlertAction(AlertActionAccept)
+	driver, err = device.NewUSBDriver(capabilities)
+	if err != nil {
+		t.Fatal(err)
+	}
+	iOSDriverExt, err = newDriverExt(device, driver, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -267,6 +273,16 @@ func Test_remoteWD_Drag(t *testing.T) {
 	}
 }
 
+func Test_Relative_Drag(t *testing.T) {
+	setup(t)
+
+	// err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5))
+	err := iOSDriverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func Test_remoteWD_SetPasteboard(t *testing.T) {
 	setup(t)
 
@@ -305,12 +321,14 @@ func Test_remoteWD_GetPasteboard(t *testing.T) {
 
 func Test_remoteWD_SendKeys(t *testing.T) {
 	setup(t)
-
-	err := driver.SendKeys("App Store")
+	driver.StartCaptureLog("hrp_wda_log")
+	err := driver.SendKeys("", WithIdentifier("test"))
+	result, _ := driver.StopCaptureLog()
 	// err := driver.SendKeys("App Store", WithFrequency(3))
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Log(result)
 }
 
 func Test_remoteWD_PressButton(t *testing.T) {
@@ -374,7 +392,7 @@ func Test_remoteWD_Source(t *testing.T) {
 	// 	t.Fatal(err)
 	// }
 
-	source, err = driver.Source(NewSourceOption().WithScope("AppiumAUT"))
+	source, err = driver.Source()
 	if err != nil {
 		t.Fatal(err)
 	}

hrp/pkg/uixt/swipe.go

@@ -19,17 +19,30 @@ func assertRelative(p float64) bool {
 func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...ActionOption) error {
 	width := dExt.windowSize.Width
 	height := dExt.windowSize.Height
+	orientation, err := dExt.Driver.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) get orientation failed, use default orientation",
+			fromX, fromY, toX, toY)
+		orientation = OrientationPortrait
+	}
 
 	if !assertRelative(fromX) || !assertRelative(fromY) ||
 		!assertRelative(toX) || !assertRelative(toY) {
 		return fmt.Errorf("fromX(%f), fromY(%f), toX(%f), toY(%f) must be less than 1",
 			fromX, fromY, toX, toY)
 	}
-
-	fromX = float64(width) * fromX
-	fromY = float64(height) * fromY
-	toX = float64(width) * toX
-	toY = float64(height) * toY
+	// 左转和右转都是"LANDSCAPE"
+	if orientation == OrientationPortrait {
+		fromX = float64(width) * fromX
+		fromY = float64(height) * fromY
+		toX = float64(width) * toX
+		toY = float64(height) * toY
+	} else {
+		fromX = float64(height) * fromX
+		fromY = float64(width) * fromY
+		toX = float64(height) * toX
+		toY = float64(width) * toY
+	}
 
 	return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, options...)
 }

hrp/pkg/uixt/tap.go

@@ -2,6 +2,7 @@ package uixt
 
 import (
 	"fmt"
+	"github.com/rs/zerolog/log"
 )
 
 func (dExt *DriverExt) TapAbsXY(x, y float64, options ...ActionOption) error {
@@ -15,9 +16,19 @@ func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error {
 		return fmt.Errorf("x, y percentage should be <= 1, got x=%v, y=%v", x, y)
 	}
 
-	x = x * float64(dExt.windowSize.Width)
-	y = y * float64(dExt.windowSize.Height)
-
+	orientation, err := dExt.Driver.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("tap (%v, %v) get orientation failed, use default orientation",
+			x, y)
+		orientation = OrientationPortrait
+	}
+	if orientation == OrientationPortrait {
+		x = x * float64(dExt.windowSize.Width)
+		y = y * float64(dExt.windowSize.Height)
+	} else {
+		x = x * float64(dExt.windowSize.Height)
+		y = y * float64(dExt.windowSize.Width)
+	}
 	return dExt.TapAbsXY(x, y, options...)
 }
 
@@ -86,7 +97,19 @@ func (dExt *DriverExt) DoubleTapXY(x, y float64) error {
 	if x > 1 || y > 1 {
 		return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
 	}
-
+	orientation, err := dExt.Driver.Orientation()
+	if err != nil {
+		log.Warn().Err(err).Msgf("tap (%v, %v) get orientation failed, use default orientation",
+			x, y)
+		orientation = OrientationPortrait
+	}
+	if orientation == OrientationPortrait {
+		x = x * float64(dExt.windowSize.Width)
+		y = y * float64(dExt.windowSize.Height)
+	} else {
+		x = x * float64(dExt.windowSize.Height)
+		y = y * float64(dExt.windowSize.Width)
+	}
 	x = x * float64(dExt.windowSize.Width)
 	y = y * float64(dExt.windowSize.Height)
 	return dExt.Driver.DoubleTapFloat(x, y)