feat: android input by appium ime

hrp/internal/version/VERSION

@@ -1 +1 @@
-v4.3.7
+v4.4.0

hrp/pkg/uixt/android_adb_driver.go

@@ -2,8 +2,8 @@ package uixt
 
 import (
 	"bytes"
-	"encoding/base64"
 	"fmt"
+	"github.com/httprunner/httprunner/v4/hrp/pkg/utf7"
 	"io/fs"
 	"io/ioutil"
 	"path/filepath"
@@ -22,6 +22,7 @@ import (
 )
 
 const AdbKeyBoardPackageName = "com.android.adbkeyboard/.AdbIME"
+const UnicodeImePackageName = "io.appium.settings/.UnicodeIME"
 
 type adbDriver struct {
 	Driver
@@ -353,14 +354,53 @@ func (ad *adbDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffe
 }
 
 func (ad *adbDriver) SendKeys(text string, options ...ActionOption) (err error) {
+	err = ad.SendUnicodeKeys(text, options...)
+	if err == nil {
+		return
+	}
+	err = ad.InputText(text, options...)
+	return
+}
+
+func (ad *adbDriver) InputText(text string, options ...ActionOption) (err error) {
 	// adb shell input text <text>
-	_, err = ad.adbClient.RunShellCommand("input", "text", encodeUnicodeText(text))
+	_, err = ad.adbClient.RunShellCommand("input", "text", text)
 	if err != nil {
 		return errors.Wrap(err, "send keys failed")
 	}
 	return nil
 }
 
+func (ad *adbDriver) SendUnicodeKeys(text string, options ...ActionOption) (err error) {
+	// If the Unicode IME is not installed, fall back to the old interface.
+	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
+	// In release version: without the Unicode IME installed, the test cannot execute.
+	if !ad.IsUnicodeIMEInstalled() {
+		return fmt.Errorf("appium unicode ime not installed")
+	}
+	currentIme, err := ad.GetIme()
+	if err != nil {
+		return
+	}
+	if currentIme != UnicodeImePackageName {
+		defer func() {
+			_ = ad.SetIme(currentIme)
+		}()
+		err = ad.SetIme(UnicodeImePackageName)
+		if err != nil {
+			log.Warn().Err(err).Msgf("set Unicode Ime failed")
+			return
+		}
+	}
+	encodedStr, err := utf7.Encoding.NewEncoder().String(text)
+	if err != nil {
+		log.Warn().Err(err).Msgf("encode text with modified utf7 failed")
+		return
+	}
+	err = ad.InputText("\""+strings.ReplaceAll(encodedStr, "\"", "\\\"")+"\"", options...)
+	return
+}
+
 func (ad *adbDriver) IsAdbKeyBoardInstalled() bool {
 	output, err := ad.adbClient.RunShellCommand("ime", "list", "-a")
 	if err != nil {
@@ -369,6 +409,14 @@ func (ad *adbDriver) IsAdbKeyBoardInstalled() bool {
 	return strings.Contains(output, AdbKeyBoardPackageName)
 }
 
+func (ad *adbDriver) IsUnicodeIMEInstalled() bool {
+	output, err := ad.adbClient.RunShellCommand("ime", "list", "-s")
+	if err != nil {
+		return false
+	}
+	return strings.Contains(output, UnicodeImePackageName)
+}
+
 func (ad *adbDriver) SendKeysByAdbKeyBoard(text string) (err error) {
 	defer func() {
 		// Reset to default, don't care which keyboard was chosen before switch:
@@ -561,6 +609,30 @@ func (ad *adbDriver) GetForegroundApp() (app AppInfo, err error) {
 	return AppInfo{}, errors.Wrap(code.MobileUIAssertForegroundAppError, "get foreground app failed")
 }
 
+func (ad *adbDriver) SetIme(ime string) error {
+	_, err := ad.adbClient.RunShellCommand("ime", "set", ime)
+	if err != nil {
+		return err
+	}
+	// even if the shell command has returned,
+	// as there might be a situation where the input method has not been completely switched yet
+	// Listen to the following message.
+	// InputMethodManagerService: onServiceConnected, name:ComponentInfo{io.appium.settings/io.appium.settings.UnicodeIME}, token:android.os.Binder@44f825
+	// But there is no such log on Vivo.
+	time.Sleep(3 * time.Second)
+	return nil
+}
+
+func (ad *adbDriver) GetIme() (ime string, err error) {
+	currentIme, err := ad.adbClient.RunShellCommand("settings", "get", "secure", "default_input_method")
+	if err != nil {
+		log.Warn().Err(err).Msgf("get default ime failed")
+		return
+	}
+	currentIme = strings.TrimSpace(currentIme)
+	return currentIme, nil
+}
+
 func (ad *adbDriver) AssertForegroundApp(packageName string, activityType ...string) error {
 	log.Debug().Str("package_name", packageName).
 		Strs("activity_type", activityType).
@@ -621,34 +693,6 @@ func (ad *adbDriver) AssertForegroundApp(packageName string, activityType ...str
 		"assert foreground activity failed")
 }
 
-func encodeUnicode(c int32) string {
-	var buffer bytes.Buffer
-	// Convert each rune (character) into two bytes
-	buffer.WriteByte(byte(c >> 8))
-	buffer.WriteByte(byte(c & 0xFF))
-	// Convert buffer bytes to base64 encoding
-	encoded := base64.StdEncoding.EncodeToString(buffer.Bytes())
-	// Replace "/" with "," and remove trailing "="
-	encoded = strings.ReplaceAll(encoded, "/", ",")
-	return strings.TrimRight(encoded, "=")
-}
-
-func encodeUnicodeText(text string) string {
-	text = strings.ReplaceAll(text, "&", "&-")
-	var sb strings.Builder
-	sb.WriteRune('"')
-	for _, c := range text {
-		if c <= 127 {
-			sb.WriteRune(c)
-		} else {
-			// Encode non-ASCII character and append it
-			sb.WriteString("&" + encodeUnicode(c) + "-")
-		}
-	}
-	sb.WriteRune('"')
-	return sb.String()
-}
-
 var androidActivities = map[string]map[string][]string{
 	// DY
 	"com.ss.android.ugc.aweme": {

hrp/pkg/uixt/android_device.go

@@ -3,17 +3,18 @@ package uixt
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
+	"github.com/httprunner/funplugin/myexec"
+	"github.com/httprunner/httprunner/v4/hrp/internal/json"
 	"net"
 	"os/exec"
 	"strings"
 
-	"github.com/httprunner/funplugin/myexec"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
 	"github.com/httprunner/httprunner/v4/hrp/internal/code"
-	"github.com/httprunner/httprunner/v4/hrp/internal/json"
 	"github.com/httprunner/httprunner/v4/hrp/pkg/gadb"
 )
 
@@ -147,6 +148,18 @@ func GetAndroidDevices(serial ...string) (devices []*gadb.Device, err error) {
 	return deviceList, nil
 }
 
+func encodeUnicode(c int32) string {
+	var buffer bytes.Buffer
+	// Convert each rune (character) into two bytes
+	buffer.WriteByte(byte(c >> 8))
+	buffer.WriteByte(byte(c & 0xFF))
+	// Convert buffer bytes to base64 encoding
+	encoded := base64.StdEncoding.EncodeToString(buffer.Bytes())
+	// Replace "/" with "," and remove trailing "="
+	encoded = strings.ReplaceAll(encoded, "/", ",")
+	return strings.TrimRight(encoded, "=")
+}
+
 type AndroidDevice struct {
 	d            *gadb.Device
 	logcat       *AdbLogcat
@@ -562,7 +575,7 @@ func (s UiSelectorHelper) Index(index int) UiSelectorHelper {
 //
 // For example, to simulate a user click on
 // the third image that is enabled in a UI screen, you
-// could specify a a search criteria where the instance is
+// could specify a search criteria where the instance is
 // 2, the `className(String)` matches the image
 // widget class, and `enabled(boolean)` is true.
 // The code would look like this:

hrp/pkg/uixt/android_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"testing"
 	"time"
 )
@@ -18,7 +19,7 @@ var (
 func setupAndroid(t *testing.T) {
 	device, err := NewAndroidDevice()
 	checkErr(t, err)
-	//device.UIA2 = true
+	device.UIA2 = false
 	driverExt, err = device.NewDriver()
 	checkErr(t, err)
 }
@@ -244,15 +245,13 @@ func TestDriver_Drag(t *testing.T) {
 }
 
 func TestDriver_SendKeys(t *testing.T) {
-	driver, err := NewUIADriver(nil, uiaServerURL)
+	setupAndroid(t)
+
+	err := driverExt.Driver.SendKeys("Android\"输入速度测试", WithIdentifier("test"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = driver.SendKeys("abc")
-	if err != nil {
-		t.Fatal(err)
-	}
 	time.Sleep(time.Second * 2)
 
 	//err = driver.SendKeys("def")

hrp/pkg/uixt/android_uia2_driver.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"github.com/httprunner/httprunner/v4/hrp/pkg/utf7"
 	"net"
 	"net/http"
 	"net/url"
@@ -452,21 +453,74 @@ func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error)
 	// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
 	// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
 	actionOptions := NewActionOptions(options...)
-	data := map[string]interface{}{
-		"text": text,
-	}
-	// new data options in post data for extra uiautomator configurations
-	actionOptions.updateData(data)
-
-	_, err = ud.httpPOST(data, "/session", ud.sessionId, "keys")
+	err = ud.SendUnicodeKeys(text, options...)
 	if err != nil {
-		// use com.android.adbkeyboard if existed
-		if ud.IsAdbKeyBoardInstalled() {
-			err = ud.SendKeysByAdbKeyBoard(text)
-		} else {
-			_, err = ud.adbClient.RunShellCommand("input", "text", text)
+		data := map[string]interface{}{
+			"text": text,
+		}
+
+		// new data options in post data for extra uiautomator configurations
+		actionOptions.updateData(data)
+
+		_, err = ud.httpPOST(data, "/session", ud.sessionId, "/keys")
+	}
+	return
+}
+
+func (ud *uiaDriver) SendUnicodeKeys(text string, options ...ActionOption) (err error) {
+	// If the Unicode IME is not installed, fall back to the old interface.
+	// There might be differences in the tracking schemes across different phones, and it is pending further verification.
+	// In release version: without the Unicode IME installed, the test cannot execute.
+	if !ud.IsUnicodeIMEInstalled() {
+		return fmt.Errorf("appium unicode ime not installed")
+	}
+	currentIme, err := ud.adbDriver.GetIme()
+	if err != nil {
+		return
+	}
+	if currentIme != UnicodeImePackageName {
+		defer func() {
+			_ = ud.adbDriver.SetIme(currentIme)
+		}()
+		err = ud.adbDriver.SetIme(UnicodeImePackageName)
+		if err != nil {
+			log.Warn().Err(err).Msgf("set Unicode Ime failed")
+			return
 		}
 	}
+	encodedStr, err := utf7.Encoding.NewEncoder().String(text)
+	if err != nil {
+		log.Warn().Err(err).Msgf("encode text with modified utf7 failed")
+		return
+	}
+	err = ud.SendActionKey(encodedStr, options...)
+	return
+}
+
+func (ud *uiaDriver) SendActionKey(text string, options ...ActionOption) (err error) {
+	actionOptions := NewActionOptions(options...)
+	var actions []interface{}
+	for i, c := range text {
+		actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)},
+			map[string]interface{}{"type": "keyUp", "value": string(c)})
+		if i != len(text)-1 {
+			actions = append(actions, map[string]interface{}{"type": "pause", "duration": 40})
+		}
+	}
+
+	data := map[string]interface{}{
+		"actions": []interface{}{
+			map[string]interface{}{
+				"type":    "key",
+				"id":      "key",
+				"actions": actions,
+			},
+		},
+	}
+
+	// new data options in post data for extra uiautomator configurations
+	actionOptions.updateData(data)
+	_, err = ud.httpPOST(data, "/session", ud.sessionId, "/actions/keys")
 	return
 }
 

hrp/pkg/uixt/ios_driver.go

@@ -555,8 +555,8 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...Action
 
 	// update data options in post data for extra WDA configurations
 	actionOptions.updateData(data)
-
-	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
+	// wda 43 version
+	_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/drag")
 	return
 }
 

hrp/pkg/uixt/ios_test.go

@@ -10,17 +10,23 @@ import (
 )
 
 var (
-	bundleId = "com.apple.Preferences"
-	driver   WebDriver
+	bundleId     = "com.apple.Preferences"
+	driver       WebDriver
+	iOSDriverExt *DriverExt
 )
 
 func setup(t *testing.T) {
-	device, err := NewIOSDevice()
+	device, err := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	driver, err = device.NewUSBDriver(nil)
+	capabilities := NewCapabilities()
+	capabilities.WithDefaultAlertAction(AlertActionAccept)
+	driver, err = device.NewUSBDriver(capabilities)
+	if err != nil {
+		t.Fatal(err)
+	}
+	iOSDriverExt, err = newDriverExt(device, driver, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -267,6 +273,16 @@ func Test_remoteWD_Drag(t *testing.T) {
 	}
 }
 
+func Test_Relative_Drag(t *testing.T) {
+	setup(t)
+
+	// err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5))
+	err := iOSDriverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func Test_remoteWD_SetPasteboard(t *testing.T) {
 	setup(t)
 

hrp/pkg/utf7/decoder.go

@@ -0,0 +1,149 @@
+package utf7
+
+import (
+	"errors"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// ErrInvalidUTF7 means that a transformer encountered invalid UTF-7.
+var ErrInvalidUTF7 = errors.New("utf7: invalid UTF-7")
+
+type decoder struct {
+	ascii bool
+}
+
+func (d *decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for i := 0; i < len(src); i++ {
+		ch := src[i]
+
+		if ch < min || ch > max { // Illegal code point in ASCII mode
+			err = ErrInvalidUTF7
+			return
+		}
+
+		if ch != '&' {
+			if nDst+1 > len(dst) {
+				err = transform.ErrShortDst
+				return
+			}
+
+			nSrc++
+
+			dst[nDst] = ch
+			nDst++
+
+			d.ascii = true
+			continue
+		}
+
+		// Find the end of the Base64 or "&-" segment
+		start := i + 1
+		for i++; i < len(src) && src[i] != '-'; i++ {
+			if src[i] == '\r' || src[i] == '\n' { // base64 package ignores CR and LF
+				err = ErrInvalidUTF7
+				return
+			}
+		}
+
+		if i == len(src) { // Implicit shift ("&...")
+			if atEOF {
+				err = ErrInvalidUTF7
+			} else {
+				err = transform.ErrShortSrc
+			}
+			return
+		}
+
+		var b []byte
+		if i == start { // Escape sequence "&-"
+			b = []byte{'&'}
+			d.ascii = true
+		} else { // Control or non-ASCII code points in base64
+			if !d.ascii { // Null shift ("&...-&...-")
+				err = ErrInvalidUTF7
+				return
+			}
+
+			b = decode(src[start:i])
+			d.ascii = false
+		}
+
+		if len(b) == 0 { // Bad encoding
+			err = ErrInvalidUTF7
+			return
+		}
+
+		if nDst+len(b) > len(dst) {
+			d.ascii = true
+			err = transform.ErrShortDst
+			return
+		}
+
+		nSrc = i + 1
+
+		for _, ch := range b {
+			dst[nDst] = ch
+			nDst++
+		}
+	}
+
+	if atEOF {
+		d.ascii = true
+	}
+
+	return
+}
+
+func (d *decoder) Reset() {
+	d.ascii = true
+}
+
+// Extracts UTF-16-BE bytes from base64 data and converts them to UTF-8.
+// A nil slice is returned if the encoding is invalid.
+func decode(b64 []byte) []byte {
+	var b []byte
+
+	// Allocate a single block of memory large enough to store the Base64 data
+	// (if padding is required), UTF-16-BE bytes, and decoded UTF-8 bytes.
+	// Since a 2-byte UTF-16 sequence may expand into a 3-byte UTF-8 sequence,
+	// double the space allocation for UTF-8.
+	if n := len(b64); b64[n-1] == '=' {
+		return nil
+	} else if n&3 == 0 {
+		b = make([]byte, b64Enc.DecodedLen(n)*3)
+	} else {
+		n += 4 - n&3
+		b = make([]byte, n+b64Enc.DecodedLen(n)*3)
+		copy(b[copy(b, b64):n], []byte("=="))
+		b64, b = b[:n], b[n:]
+	}
+
+	// Decode Base64 into the first 1/3rd of b
+	n, err := b64Enc.Decode(b, b64)
+	if err != nil || n&1 == 1 {
+		return nil
+	}
+
+	// Decode UTF-16-BE into the remaining 2/3rds of b
+	b, s := b[:n], b[n:]
+	j := 0
+	for i := 0; i < n; i += 2 {
+		r := rune(b[i])<<8 | rune(b[i+1])
+		if utf16.IsSurrogate(r) {
+			if i += 2; i == n {
+				return nil
+			}
+			r2 := rune(b[i])<<8 | rune(b[i+1])
+			if r = utf16.DecodeRune(r, r2); r == repl {
+				return nil
+			}
+		} else if min <= r && r <= max {
+			return nil
+		}
+		j += utf8.EncodeRune(s[j:], r)
+	}
+	return s[:j]
+}

hrp/pkg/utf7/encoder.go

@@ -0,0 +1,91 @@
+package utf7
+
+import (
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+type encoder struct{}
+
+func (e *encoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for i := 0; i < len(src); {
+		ch := src[i]
+
+		var b []byte
+		if min <= ch && ch <= max {
+			b = []byte{ch}
+			if ch == '&' {
+				b = append(b, '-')
+			}
+
+			i++
+		} else {
+			start := i
+
+			// Find the next printable ASCII code point
+			i++
+			for i < len(src) && (src[i] < min || src[i] > max) {
+				i++
+			}
+
+			if !atEOF && i == len(src) {
+				err = transform.ErrShortSrc
+				return
+			}
+
+			b = encode(src[start:i])
+		}
+
+		if nDst+len(b) > len(dst) {
+			err = transform.ErrShortDst
+			return
+		}
+
+		nSrc = i
+
+		for _, ch := range b {
+			dst[nDst] = ch
+			nDst++
+		}
+	}
+
+	return
+}
+
+func (e *encoder) Reset() {}
+
+// Converts string s from UTF-8 to UTF-16-BE, encodes the result as base64,
+// removes the padding, and adds UTF-7 shifts.
+func encode(s []byte) []byte {
+	// len(s) is sufficient for UTF-8 to UTF-16 conversion if there are no
+	// control code points (see table below).
+	b := make([]byte, 0, len(s)+4)
+	for len(s) > 0 {
+		r, size := utf8.DecodeRune(s)
+		if r > utf8.MaxRune {
+			r, size = utf8.RuneError, 1 // Bug fix (issue 3785)
+		}
+		s = s[size:]
+		if r1, r2 := utf16.EncodeRune(r); r1 != repl {
+			b = append(b, byte(r1>>8), byte(r1))
+			r = r2
+		}
+		b = append(b, byte(r>>8), byte(r))
+	}
+
+	// Encode as base64
+	n := b64Enc.EncodedLen(len(b)) + 2
+	b64 := make([]byte, n)
+	b64Enc.Encode(b64[1:], b)
+
+	// Strip padding
+	n -= 2 - (len(b)+2)%3
+	b64 = b64[:n]
+
+	// Add UTF-7 shifts
+	b64[0] = '&'
+	b64[n-1] = '-'
+	return b64
+}

hrp/pkg/utf7/utf7.go

@@ -0,0 +1,34 @@
+// Package utf7 implements modified UTF-7 encoding defined in RFC 3501 section 5.1.3
+package utf7
+
+import (
+	"encoding/base64"
+
+	"golang.org/x/text/encoding"
+)
+
+const (
+	min = 0x20 // Minimum self-representing UTF-7 value
+	max = 0x7E // Maximum self-representing UTF-7 value
+
+	repl = '\uFFFD' // Unicode replacement code point
+)
+
+var b64Enc = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,")
+
+type enc struct{}
+
+func (e enc) NewDecoder() *encoding.Decoder {
+	return &encoding.Decoder{
+		Transformer: &decoder{true},
+	}
+}
+
+func (e enc) NewEncoder() *encoding.Encoder {
+	return &encoding.Encoder{
+		Transformer: &encoder{},
+	}
+}
+
+// Encoding is the modified UTF-7 encoding.
+var Encoding encoding.Encoding = enc{}