GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-17 23:27:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move hrp/ to root folder

Browse Source
This commit is contained in:
lilong.129
2025-02-06 10:52:08 +08:00
parent 9376692b71
commit 1f063dd6f7
221 changed files with 206 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
pkg/uixt/README.md Normal file
View File

@@ -0,0 +1,34 @@
# uixt
From v4.3.0，HttpRunner will support mobile UI automation testing:
- iOS: based on [appium/WebDriverAgent], with forked client library [electricbubble/gwda] in golang
- Android: based on [appium-uiautomator2-server], with forked client library [electricbubble/guia2] in golang
Some UI recognition algorithms are also introduced for both iOS and Android:
- OCR: based on OCR API service from [volcengine], other API service may be extended
## Dependencies
### OCR
OCR API is a paid service, you need to pre-purchase and configure the environment variables.
- VEDEM_IMAGE_URL
- VEDEM_IMAGE_AK
- VEDEM_IMAGE_SK
## Thanks
This uixt module is initially forked from the following repos and made a lot of changes.
- [electricbubble/gwda]
- [electricbubble/guia2]
[appium/WebDriverAgent]: https://github.com/appium/WebDriverAgent
[electricbubble/gwda]: https://github.com/electricbubble/gwda
[electricbubble/guia2]: https://github.com/electricbubble/guia2
[volcengine]: https://www.volcengine.com/product/text-recognition
[appium-uiautomator2-server]: https://github.com/appium/appium-uiautomator2-server

874
pkg/uixt/action.go Normal file
View File

@@ -0,0 +1,874 @@
package uixt
import (
"encoding/json"
"fmt"
"math/rand"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
type ActionMethod string
const (
ACTION_LOG ActionMethod = "log"
ACTION_AppInstall ActionMethod = "install"
ACTION_AppUninstall ActionMethod = "uninstall"
ACTION_AppClear ActionMethod = "app_clear"
ACTION_AppStart ActionMethod = "app_start"
ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成
ACTION_AppTerminate ActionMethod = "app_terminate"
ACTION_AppStop ActionMethod = "app_stop"
ACTION_ScreenShot ActionMethod = "screenshot"
ACTION_Sleep ActionMethod = "sleep"
ACTION_SleepMS ActionMethod = "sleep_ms"
ACTION_SleepRandom ActionMethod = "sleep_random"
ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera
ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera
ACTION_SetClipboard ActionMethod = "set_clipboard"
ACTION_GetClipboard ActionMethod = "get_clipboard"
ACTION_SetIme ActionMethod = "set_ime"
ACTION_GetSource ActionMethod = "get_source"
ACTION_GetForegroundApp ActionMethod = "get_foreground_app"
ACTION_CallFunction ActionMethod = "call_function"
// UI handling
ACTION_Home ActionMethod = "home"
ACTION_TapXY ActionMethod = "tap_xy"
ACTION_TapAbsXY ActionMethod = "tap_abs_xy"
ACTION_TapByOCR ActionMethod = "tap_ocr"
ACTION_TapByCV ActionMethod = "tap_cv"
ACTION_Tap ActionMethod = "tap"
ACTION_DoubleTapXY ActionMethod = "double_tap_xy"
ACTION_DoubleTap ActionMethod = "double_tap"
ACTION_Swipe ActionMethod = "swipe"
ACTION_Input ActionMethod = "input"
ACTION_Back ActionMethod = "back"
ACTION_KeyCode ActionMethod = "keycode"
// custom actions
ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap
ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap
ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap
ACTION_ClosePopups ActionMethod = "close_popups"
ACTION_EndToEndDelay ActionMethod = "live_e2e"
ACTION_InstallApp ActionMethod = "install_app"
ACTION_UninstallApp ActionMethod = "uninstall_app"
ACTION_DownloadApp ActionMethod = "download_app"
)
const (
// UI validation
// selectors
SelectorName string = "ui_name"
SelectorLabel string = "ui_label"
SelectorOCR string = "ui_ocr"
SelectorImage string = "ui_image"
SelectorForegroundApp string = "ui_foreground_app"
// assertions
AssertionEqual string = "equal"
AssertionNotEqual string = "not_equal"
AssertionExists string = "exists"
AssertionNotExists string = "not_exists"
)
type MobileAction struct {
Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"`
Params interface{} `json:"params,omitempty" yaml:"params,omitempty"`
Fn func() `json:"-" yaml:"-"` // only used for function action, not serialized
Options *ActionOptions `json:"options,omitempty" yaml:"options,omitempty"`
ActionOptions
}
func (ma MobileAction) GetOptions() []ActionOption {
var actionOptionList []ActionOption
// Notice: merge options from ma.Options and ma.ActionOptions
if ma.Options != nil {
actionOptionList = append(actionOptionList, ma.Options.Options()...)
}
actionOptionList = append(actionOptionList, ma.ActionOptions.Options()...)
return actionOptionList
}
// (x1, y1) is the top left corner, (x2, y2) is the bottom right corner
// [x1, y1, x2, y2] in percentage of the screen
type Scope []float64
// [x1, y1, x2, y2] in absolute pixels
type AbsScope []int
func (s AbsScope) Option() ActionOption {
return WithAbsScope(s[0], s[1], s[2], s[3])
}
type ActionOptions struct {
// log
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
// control related
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Interval float64 `json:"interval,omitempty" yaml:"interval,omitempty"` // interval between retries in seconds
Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action
PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty"` // used to set duration of ios swipe action
Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty"`
// scope related
Scope Scope `json:"scope,omitempty" yaml:"scope,omitempty"`
AbsScope AbsScope `json:"abs_scope,omitempty" yaml:"abs_scope,omitempty"`
Regex bool `json:"regex,omitempty" yaml:"regex,omitempty"` // use regex to match text
Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point
OffsetRandomRange []int `json:"offset_random_range,omitempty" yaml:"offset_random_range,omitempty"` // set random range [min, max] for tap/swipe points
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element
MatchOne bool `json:"match_one,omitempty" yaml:"match_one,omitempty"` // match one of the targets if existed
// set custiom options such as textview, id, description
Custom map[string]interface{} `json:"custom,omitempty" yaml:"custom,omitempty"`
// screenshot related
ScreenShotWithOCR bool `json:"screenshot_with_ocr,omitempty" yaml:"screenshot_with_ocr,omitempty"`
ScreenShotWithUpload bool `json:"screenshot_with_upload,omitempty" yaml:"screenshot_with_upload,omitempty"`
ScreenShotWithLiveType bool `json:"screenshot_with_live_type,omitempty" yaml:"screenshot_with_live_type,omitempty"`
ScreenShotWithLivePopularity bool `json:"screenshot_with_live_popularity,omitempty" yaml:"screenshot_with_live_popularity,omitempty"`
ScreenShotWithUITypes []string `json:"screenshot_with_ui_types,omitempty" yaml:"screenshot_with_ui_types,omitempty"`
ScreenShotWithClosePopups bool `json:"screenshot_with_close_popups,omitempty" yaml:"screenshot_with_close_popups,omitempty"`
ScreenShotWithOCRCluster string `json:"screenshot_with_ocr_cluster,omitempty" yaml:"screenshot_with_ocr_cluster,omitempty"`
ScreenShotFileName string `json:"screenshot_file_name,omitempty" yaml:"screenshot_file_name,omitempty"`
}
func (o *ActionOptions) Options() []ActionOption {
options := make([]ActionOption, 0)
if o == nil {
return options
}
if o.Identifier != "" {
options = append(options, WithIdentifier(o.Identifier))
}
if o.MaxRetryTimes != 0 {
options = append(options, WithMaxRetryTimes(o.MaxRetryTimes))
}
if o.IgnoreNotFoundError {
options = append(options, WithIgnoreNotFoundError(true))
}
if o.Interval != 0 {
options = append(options, WithInterval(o.Interval))
}
if o.Duration != 0 {
options = append(options, WithDuration(o.Duration))
}
if o.PressDuration != 0 {
options = append(options, WithPressDuration(o.PressDuration))
}
if o.Steps != 0 {
options = append(options, WithSteps(o.Steps))
}
switch v := o.Direction.(type) {
case string:
options = append(options, WithDirection(v))
case []float64:
options = append(options, WithCustomDirection(
v[0], v[1],
v[2], v[3],
))
case []interface{}:
// loaded from json case
// custom direction: [fromX, fromY, toX, toY]
sx, _ := builtin.Interface2Float64(v[0])
sy, _ := builtin.Interface2Float64(v[1])
ex, _ := builtin.Interface2Float64(v[2])
ey, _ := builtin.Interface2Float64(v[3])
options = append(options, WithCustomDirection(
sx, sy,
ex, ey,
))
}
if o.Timeout != 0 {
options = append(options, WithTimeout(o.Timeout))
}
if o.Frequency != 0 {
options = append(options, WithFrequency(o.Frequency))
}
if len(o.AbsScope) == 4 {
options = append(options, WithAbsScope(
o.AbsScope[0], o.AbsScope[1], o.AbsScope[2], o.AbsScope[3]))
} else if len(o.Scope) == 4 {
options = append(options, WithScope(
o.Scope[0], o.Scope[1], o.Scope[2], o.Scope[3]))
}
if len(o.Offset) == 2 {
// for tap [x,y] offset
options = append(options, WithTapOffset(o.Offset[0], o.Offset[1]))
} else if len(o.Offset) == 4 {
// for swipe [fromX, fromY, toX, toY] offset
options = append(options, WithSwipeOffset(
o.Offset[0], o.Offset[1], o.Offset[2], o.Offset[3]))
}
if len(o.OffsetRandomRange) == 2 {
options = append(options, WithOffsetRandomRange(
o.OffsetRandomRange[0], o.OffsetRandomRange[1]))
}
if o.Regex {
options = append(options, WithRegex(true))
}
if o.Index != 0 {
options = append(options, WithIndex(o.Index))
}
if o.MatchOne {
options = append(options, WithMatchOne(true))
}
// custom options
if o.Custom != nil {
for k, v := range o.Custom {
options = append(options, WithCustomOption(k, v))
}
}
// screenshot options
if o.ScreenShotWithOCR {
options = append(options, WithScreenShotOCR(true))
}
if o.ScreenShotWithUpload {
options = append(options, WithScreenShotUpload(true))
}
if o.ScreenShotWithLiveType {
options = append(options, WithScreenShotLiveType(true))
}
if o.ScreenShotWithLivePopularity {
options = append(options, WithScreenShotLivePopularity(true))
}
if len(o.ScreenShotWithUITypes) > 0 {
options = append(options, WithScreenShotUITypes(o.ScreenShotWithUITypes...))
}
if o.ScreenShotWithClosePopups {
options = append(options, WithScreenShotClosePopups(true))
}
if o.ScreenShotWithOCRCluster != "" {
options = append(options, WithScreenOCRCluster(o.ScreenShotWithOCRCluster))
}
if o.ScreenShotFileName != "" {
options = append(options, WithScreenShotFileName(o.ScreenShotFileName))
}
return options
}
func (o *ActionOptions) screenshotActions() []string {
actions := []string{}
if o.ScreenShotWithUpload {
actions = append(actions, "upload")
}
if o.ScreenShotWithOCR {
actions = append(actions, "ocr")
}
if o.ScreenShotWithLiveType {
actions = append(actions, "liveType")
}
if o.ScreenShotWithLivePopularity {
actions = append(actions, "livePopularity")
}
// UI detection
if len(o.ScreenShotWithUITypes) > 0 {
actions = append(actions, "ui")
}
if o.ScreenShotWithClosePopups {
actions = append(actions, "close")
}
return actions
}
func (o *ActionOptions) getRandomOffset() float64 {
if len(o.OffsetRandomRange) != 2 {
// invalid offset random range, should be [min, max]
return 0
}
minOffset := o.OffsetRandomRange[0]
maxOffset := o.OffsetRandomRange[1]
return float64(builtin.GetRandomNumber(minOffset, maxOffset)) + rand.Float64()
}
func (o *ActionOptions) updateData(data map[string]interface{}) {
if o.Identifier != "" {
data["log"] = map[string]interface{}{
"enable": true,
"data": o.Identifier,
}
}
if o.Steps > 0 {
data["steps"] = o.Steps
}
if _, ok := data["steps"]; !ok {
data["steps"] = 12 // default steps
}
if o.Duration > 0 {
data["duration"] = o.Duration
}
if _, ok := data["duration"]; !ok {
data["duration"] = 0 // default duration
}
if o.Frequency > 0 {
data["frequency"] = o.Frequency
}
if _, ok := data["frequency"]; !ok {
data["frequency"] = 10 // default frequency
}
if _, ok := data["replace"]; !ok {
data["replace"] = true // default true
}
// custom options
if o.Custom != nil {
for k, v := range o.Custom {
data[k] = v
}
}
}
func NewActionOptions(options ...ActionOption) *ActionOptions {
actionOptions := &ActionOptions{}
for _, option := range options {
option(actionOptions)
}
return actionOptions
}
type TapTextAction struct {
Text string
Options []ActionOption
}
type ActionOption func(o *ActionOptions)
func WithCustomOption(key string, value interface{}) ActionOption {
return func(o *ActionOptions) {
if o.Custom == nil {
o.Custom = make(map[string]interface{})
}
o.Custom[key] = value
}
}
func WithIdentifier(identifier string) ActionOption {
return func(o *ActionOptions) {
o.Identifier = identifier
}
}
func WithIndex(index int) ActionOption {
return func(o *ActionOptions) {
o.Index = index
}
}
// set alias for compatibility
var WithWaitTime = WithInterval
func WithInterval(sec float64) ActionOption {
return func(o *ActionOptions) {
o.Interval = sec
}
}
func WithDuration(duration float64) ActionOption {
return func(o *ActionOptions) {
o.Duration = duration
}
}
func WithPressDuration(pressDuration float64) ActionOption {
return func(o *ActionOptions) {
o.PressDuration = pressDuration
}
}
func WithSteps(steps int) ActionOption {
return func(o *ActionOptions) {
o.Steps = steps
}
}
// WithDirection inputs direction (up, down, left, right)
func WithDirection(direction string) ActionOption {
return func(o *ActionOptions) {
o.Direction = direction
}
}
// WithCustomDirection inputs sx, sy, ex, ey
func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
return func(o *ActionOptions) {
o.Direction = []float64{sx, sy, ex, ey}
}
}
// WithScope inputs area of [(x1,y1), (x2,y2)]
// x1, y1, x2, y2 are all in [0, 1], which means the relative position of the screen
func WithScope(x1, y1, x2, y2 float64) ActionOption {
return func(o *ActionOptions) {
o.Scope = Scope{x1, y1, x2, y2}
}
}
// WithAbsScope inputs area of [(x1,y1), (x2,y2)]
// x1, y1, x2, y2 are all absolute position of the screen
func WithAbsScope(x1, y1, x2, y2 int) ActionOption {
return func(o *ActionOptions) {
o.AbsScope = AbsScope{x1, y1, x2, y2}
}
}
// Deprecated: use WithTapOffset instead
func WithOffset(offsetX, offsetY int) ActionOption {
return func(o *ActionOptions) {
o.Offset = []int{offsetX, offsetY}
}
}
// tap [x, y] with offset [offsetX, offsetY]
var WithTapOffset = WithOffset
// swipe [fromX, fromY, toX, toY] with offset [offsetFromX, offsetFromY, offsetToX, offsetToY]
func WithSwipeOffset(offsetFromX, offsetFromY, offsetToX, offsetToY int) ActionOption {
return func(o *ActionOptions) {
o.Offset = []int{offsetFromX, offsetFromY, offsetToX, offsetToY}
}
}
func WithOffsetRandomRange(min, max int) ActionOption {
return func(o *ActionOptions) {
o.OffsetRandomRange = []int{min, max}
}
}
func WithRegex(regex bool) ActionOption {
return func(o *ActionOptions) {
o.Regex = regex
}
}
func WithMatchOne(matchOne bool) ActionOption {
return func(o *ActionOptions) {
o.MatchOne = matchOne
}
}
func WithFrequency(frequency int) ActionOption {
return func(o *ActionOptions) {
o.Frequency = frequency
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *ActionOptions) {
o.MaxRetryTimes = maxRetryTimes
}
}
func WithTimeout(timeout int) ActionOption {
return func(o *ActionOptions) {
o.Timeout = timeout
}
}
func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
return func(o *ActionOptions) {
o.IgnoreNotFoundError = ignoreError
}
}
func WithScreenShotOCR(ocrOn bool) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithOCR = ocrOn
}
}
func WithScreenShotUpload(uploadOn bool) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithUpload = uploadOn
}
}
func WithScreenShotLiveType(liveTypeOn bool) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithLiveType = liveTypeOn
}
}
func WithScreenShotLivePopularity(livePopularityOn bool) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithLivePopularity = livePopularityOn
}
}
func WithScreenShotUITypes(uiTypes ...string) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithUITypes = uiTypes
}
}
func WithScreenShotClosePopups(closeOn bool) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithClosePopups = closeOn
}
}
func WithScreenOCRCluster(ocrCluster string) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotWithOCRCluster = ocrCluster
}
}
func WithScreenShotFileName(fileName string) ActionOption {
return func(o *ActionOptions) {
o.ScreenShotFileName = fileName
}
}
func (dExt *DriverExt) ParseActionOptions(options ...ActionOption) []ActionOption {
actionOptions := NewActionOptions(options...)
// convert relative scope to absolute scope
if len(actionOptions.AbsScope) != 4 && len(actionOptions.Scope) == 4 {
scope := actionOptions.Scope
actionOptions.AbsScope = dExt.GenAbsScope(
scope[0], scope[1], scope[2], scope[3])
}
return actionOptions.Options()
}
func (dExt *DriverExt) GenAbsScope(x1, y1, x2, y2 float64) AbsScope {
// convert relative scope to absolute scope
windowSize, _ := dExt.Driver.WindowSize()
absX1 := int(x1 * float64(windowSize.Width))
absY1 := int(y1 * float64(windowSize.Height))
absX2 := int(x2 * float64(windowSize.Width))
absY2 := int(y2 * float64(windowSize.Height))
return AbsScope{absX1, absY1, absX2, absY2}
}
func (dExt *DriverExt) DoAction(action MobileAction) (err error) {
actionStartTime := time.Now()
defer func() {
var logger *zerolog.Event
if err != nil {
logger = log.Error().Bool("success", false).Err(err)
} else {
logger = log.Debug().Bool("success", true)
}
logger = logger.
Str("method", string(action.Method)).
Interface("params", action.Params).
Int64("elapsed(ms)", time.Since(actionStartTime).Milliseconds())
logger.Msg("exec uixt action")
}()
switch action.Method {
case ACTION_AppInstall:
if appUrl, ok := action.Params.(string); ok {
if err = dExt.InstallByUrl(appUrl, WithRetryTimes(action.MaxRetryTimes)); err != nil {
return errors.Wrap(err, "failed to install app")
}
}
case ACTION_AppUninstall:
if packageName, ok := action.Params.(string); ok {
if err = dExt.Uninstall(packageName, action.GetOptions()...); err != nil {
return errors.Wrap(err, "failed to uninstall app")
}
}
case ACTION_AppClear:
if packageName, ok := action.Params.(string); ok {
if err = dExt.Driver.Clear(packageName); err != nil {
return errors.Wrap(err, "failed to clear app")
}
}
case ACTION_AppLaunch:
if bundleId, ok := action.Params.(string); ok {
return dExt.Driver.AppLaunch(bundleId)
}
return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
ACTION_AppLaunch, action.Params)
case ACTION_SwipeToTapApp:
if appName, ok := action.Params.(string); ok {
return dExt.swipeToTapApp(appName, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
if text, ok := action.Params.(string); ok {
return dExt.swipeToTapTexts([]string{text}, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
case ACTION_SwipeToTapTexts:
if texts, ok := action.Params.([]string); ok {
return dExt.swipeToTapTexts(texts, action.GetOptions()...)
}
if texts, err := builtin.ConvertToStringSlice(action.Params); err == nil {
return dExt.swipeToTapTexts(texts, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_SwipeToTapTexts, action.Params)
case ACTION_AppTerminate:
if bundleId, ok := action.Params.(string); ok {
success, err := dExt.Driver.AppTerminate(bundleId)
if err != nil {
return errors.Wrap(err, "failed to terminate app")
}
if !success {
log.Warn().Str("bundleId", bundleId).Msg("app was not running")
}
return nil
}
return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params)
case ACTION_SetClipboard:
if text, ok := action.Params.(string); ok {
err := dExt.Driver.SetPasteboard(PasteboardTypePlaintext, text)
if err != nil {
return errors.Wrap(err, "failed to set clipboard")
}
return nil
}
return fmt.Errorf("set_clioboard params should be text(string), got %v", action.Params)
case ACTION_Home:
return dExt.Driver.Homescreen()
case ACTION_SetIme:
if ime, ok := action.Params.(string); ok {
err = dExt.Driver.SetIme(ime)
if err != nil {
return errors.Wrap(err, "failed to set ime")
}
return nil
}
case ACTION_GetSource:
if packageName, ok := action.Params.(string); ok {
source := NewSourceOption().WithProcessName(packageName)
_, err = dExt.Driver.Source(source)
if err != nil {
return errors.Wrap(err, "failed to set ime")
}
return nil
}
case ACTION_TapXY:
if params, err := builtin.ConvertToFloat64Slice(action.Params); err == nil {
// relative x,y of window size: [0.5, 0.5]
if len(params) != 2 {
return fmt.Errorf("invalid tap location params: %v", params)
}
x, y := params[0], params[1]
return dExt.TapXY(x, y, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_TapAbsXY:
if params, err := builtin.ConvertToFloat64Slice(action.Params); err == nil {
// absolute coordinates x,y of window size: [100, 300]
if len(params) != 2 {
return fmt.Errorf("invalid tap location params: %v", params)
}
x, y := params[0], params[1]
return dExt.TapAbsXY(x, y, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
return dExt.TapByOCR(ocrText, action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
actionOptions := NewActionOptions(action.GetOptions()...)
if len(actionOptions.ScreenShotWithUITypes) > 0 {
return dExt.TapByUIDetection(action.GetOptions()...)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
if params, err := builtin.ConvertToFloat64Slice(action.Params); err == nil {
// relative x,y of window size: [0.5, 0.5]
if len(params) != 2 {
return fmt.Errorf("invalid tap location params: %v", params)
}
x, y := params[0], params[1]
return dExt.DoubleTapXY(x, y)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params)
case ACTION_DoubleTap:
if param, ok := action.Params.(string); ok {
return dExt.DoubleTap(param)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
case ACTION_Swipe:
params := action.Params
swipeAction := dExt.prepareSwipeAction(params, action.GetOptions()...)
return swipeAction(dExt)
case ACTION_Input:
// input text on current active element
// append \n to send text with enter
// send \b\b\b to delete 3 chars
param := fmt.Sprintf("%v", action.Params)
return dExt.Driver.Input(param, action.GetOptions()...)
case ACTION_Back:
return dExt.Driver.PressBack()
case ACTION_Sleep:
if param, ok := action.Params.(json.Number); ok {
seconds, _ := param.Float64()
time.Sleep(time.Duration(seconds*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(float64); ok {
time.Sleep(time.Duration(param*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(int64); ok {
time.Sleep(time.Duration(param) * time.Second)
return nil
} else if sd, ok := action.Params.(SleepConfig); ok {
sleepStrict(sd.StartTime, int64(sd.Seconds*1000))
return nil
}
return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
case ACTION_SleepMS:
if param, ok := action.Params.(json.Number); ok {
milliseconds, _ := param.Int64()
time.Sleep(time.Duration(milliseconds) * time.Millisecond)
return nil
} else if param, ok := action.Params.(int64); ok {
time.Sleep(time.Duration(param) * time.Millisecond)
return nil
} else if sd, ok := action.Params.(SleepConfig); ok {
sleepStrict(sd.StartTime, sd.Milliseconds)
return nil
}
return fmt.Errorf("invalid sleep ms params: %v(%T)", action.Params, action.Params)
case ACTION_SleepRandom:
if params, err := builtin.ConvertToFloat64Slice(action.Params); err == nil {
sleepStrict(time.Now(), getSimulationDuration(params))
return nil
}
return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
case ACTION_ScreenShot:
// take screenshot
log.Info().Msg("take screenshot for current screen")
_, err := dExt.GetScreenResult(action.GetOptions()...)
return err
case ACTION_StartCamera:
return dExt.Driver.StartCamera()
case ACTION_StopCamera:
return dExt.Driver.StopCamera()
case ACTION_ClosePopups:
return dExt.ClosePopupsHandler()
case ACTION_EndToEndDelay:
CollectEndToEndDelay(dExt, action.GetOptions()...)
return nil
case ACTION_CallFunction:
fn := action.Fn
fn()
return nil
default:
log.Warn().Str("action", string(action.Method)).Msg("action not implemented")
return errors.Wrapf(code.InvalidCaseError,
"UI action %v not implemented", action.Method)
}
return nil
}
type SleepConfig struct {
StartTime time.Time `json:"start_time"`
Seconds float64 `json:"seconds,omitempty"`
Milliseconds int64 `json:"milliseconds,omitempty"`
}
// getSimulationDuration returns simulation duration by given params (in seconds)
func getSimulationDuration(params []float64) (milliseconds int64) {
if len(params) == 1 {
// given constant duration time
return int64(params[0] * 1000)
}
if len(params) == 2 {
// given [min, max], missing weight
// append default weight 1
params = append(params, 1.0)
}
var sections []struct {
min, max, weight float64
}
totalProb := 0.0
for i := 0; i+3 <= len(params); i += 3 {
min := params[i]
max := params[i+1]
weight := params[i+2]
totalProb += weight
sections = append(sections,
struct{ min, max, weight float64 }{min, max, weight},
)
}
if totalProb == 0 {
log.Warn().Msg("total weight is 0, skip simulation")
return 0
}
r := rand.Float64()
accProb := 0.0
for _, s := range sections {
accProb += s.weight / totalProb
if r < accProb {
milliseconds := int64((s.min + rand.Float64()*(s.max-s.min)) * 1000)
log.Info().Int64("random(ms)", milliseconds).
Interface("strategy_params", params).Msg("get simulation duration")
return milliseconds
}
}
log.Warn().Interface("strategy_params", params).
Msg("get simulation duration failed, skip simulation")
return 0
}
// sleepStrict sleeps strict duration with given params
// startTime is used to correct sleep duration caused by process time
func sleepStrict(startTime time.Time, strictMilliseconds int64) {
var elapsed int64
if !startTime.IsZero() {
elapsed = time.Since(startTime).Milliseconds()
}
dur := strictMilliseconds - elapsed
// if elapsed time is greater than given duration, skip sleep to reduce deviation caused by process time
if dur <= 0 {
log.Warn().
Int64("elapsed(ms)", elapsed).
Int64("strictSleep(ms)", strictMilliseconds).
Msg("elapsed >= simulation duration, skip sleep")
return
}
log.Info().Int64("sleepDuration(ms)", dur).
Int64("elapsed(ms)", elapsed).
Int64("strictSleep(ms)", strictMilliseconds).
Msg("sleep remaining duration time")
time.Sleep(time.Duration(dur) * time.Millisecond)
}

46
pkg/uixt/action_test.go Normal file
View File

@@ -0,0 +1,46 @@
package uixt
import (
"testing"
"time"
)
func checkErr(t *testing.T, err error, msg ...string) {
if err != nil {
if len(msg) == 0 {
t.Fatal(err)
} else {
t.Fatal(msg, err)
}
}
}
func TestGetSimulationDuration(t *testing.T) {
params := []float64{1.23}
duration := getSimulationDuration(params)
if duration != 1230 {
t.Fatal("getSimulationDuration failed")
}
params = []float64{1, 2}
duration = getSimulationDuration(params)
if duration < 1000 || duration > 2000 {
t.Fatal("getSimulationDuration failed")
}
params = []float64{1, 5, 0.7, 5, 10, 0.3}
duration = getSimulationDuration(params)
if duration < 1000 || duration > 10000 {
t.Fatal("getSimulationDuration failed")
}
}
func TestSleepStrict(t *testing.T) {
startTime := time.Now()
sleepStrict(startTime, 1230)
dur := time.Since(startTime).Milliseconds()
t.Log(dur)
if dur < 1230 || dur > 1300 {
t.Fatalf("sleepRandom failed, dur: %d", dur)
}
}

302
pkg/uixt/ai.go Normal file
View File

@@ -0,0 +1,302 @@
package uixt
import (
"bytes"
"fmt"
"image"
"math"
"regexp"
"github.com/pkg/errors"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
type IImageService interface {
// GetImage returns image result including ocr texts, uploaded image url, etc
GetImage(imageBuf *bytes.Buffer, options ...ActionOption) (imageResult *ImageResult, err error)
}
type ImageResult struct {
URL string `json:"url,omitempty"` // image uploaded url
OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts
// NoLive（非直播间）
// Shop（电商）
// LifeService（生活服务）
// Show（秀场）
// Game（游戏）
// People（多人）
// PK（PK）
// Media（媒体）
// Chat（语音）
// Event（赛事）
LiveType string `json:"liveType,omitempty"` // 直播间类型
LivePopularity int64 `json:"livePopularity,omitempty"` // 直播间热度
UIResult UIResultMap `json:"uiResult,omitempty"` // 图标检测
ClosePopupsResult *ClosePopupsResult `json:"closeResult,omitempty"` // 弹窗按钮检测
}
type OCRResult struct {
Text string `json:"text"`
Points []PointF `json:"points"`
}
type OCRResults []OCRResult
func (o OCRResults) ToOCRTexts() (ocrTexts OCRTexts) {
for _, ocrResult := range o {
rect := image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(ocrResult.Points[0].X),
Y: int(ocrResult.Points[0].Y),
},
Max: image.Point{
X: int(ocrResult.Points[2].X),
Y: int(ocrResult.Points[2].Y),
},
}
rectStr := fmt.Sprintf("%d,%d,%d,%d",
rect.Min.X, rect.Min.Y, rect.Max.X, rect.Max.Y)
ocrText := OCRText{
Text: ocrResult.Text,
Rect: rect,
RectStr: rectStr,
}
ocrTexts = append(ocrTexts, ocrText)
}
return
}
type OCRText struct {
Text string `json:"text"`
RectStr string `json:"rect"`
Rect image.Rectangle `json:"-"`
}
func (t OCRText) Size() Size {
return Size{
Width: t.Rect.Dx(),
Height: t.Rect.Dy(),
}
}
func (t OCRText) Center() PointF {
return getRectangleCenterPoint(t.Rect)
}
func getRectangleCenterPoint(rect image.Rectangle) (point PointF) {
x, y := float64(rect.Min.X), float64(rect.Min.Y)
width, height := float64(rect.Dx()), float64(rect.Dy())
point = PointF{
X: x + width*0.5,
Y: y + height*0.5,
}
return point
}
type OCRTexts []OCRText
func (t OCRTexts) texts() (texts []string) {
for _, text := range t {
texts = append(texts, text.Text)
}
return texts
}
func (t OCRTexts) FilterScope(scope AbsScope) (results OCRTexts) {
for _, ocrText := range t {
rect := ocrText.Rect
// check if text in scope
if len(scope) == 4 {
if rect.Min.X < scope[0] ||
rect.Min.Y < scope[1] ||
rect.Max.X > scope[2] ||
rect.Max.Y > scope[3] {
// not in scope
continue
}
}
results = append(results, ocrText)
}
return
}
// FindText returns matched text with options
// Notice: filter scope should be specified with WithAbsScope
func (t OCRTexts) FindText(text string, options ...ActionOption) (result OCRText, err error) {
actionOptions := NewActionOptions(options...)
var results []OCRText
for _, ocrText := range t.FilterScope(actionOptions.AbsScope) {
if actionOptions.Regex {
// regex on, check if match regex
if !regexp.MustCompile(text).MatchString(ocrText.Text) {
continue
}
} else {
// regex off, check if match exactly
if ocrText.Text != text {
continue
}
}
results = append(results, ocrText)
// return the first one matched exactly when index not specified
if ocrText.Text == text && actionOptions.Index == 0 {
return ocrText, nil
}
}
if len(results) == 0 {
return OCRText{}, errors.Wrap(code.CVResultNotFoundError,
fmt.Sprintf("text %s not found in %v", text, t.texts()))
}
// get index
idx := actionOptions.Index
if idx < 0 {
idx = len(results) + idx
}
// index out of range
if idx >= len(results) || idx < 0 {
return OCRText{}, errors.Wrap(code.CVResultNotFoundError,
fmt.Sprintf("text %s found %d, index %d out of range", text, len(results), idx))
}
return results[idx], nil
}
func (t OCRTexts) FindTexts(texts []string, options ...ActionOption) (results OCRTexts, err error) {
actionOptions := NewActionOptions(options...)
for _, text := range texts {
ocrText, err := t.FindText(text, options...)
if err != nil {
continue
}
results = append(results, ocrText)
// found one, skip searching and return
if actionOptions.MatchOne {
return results, nil
}
}
if len(results) == len(texts) {
return results, nil
}
return nil, errors.Wrap(code.CVResultNotFoundError,
fmt.Sprintf("texts %s not found in %v", texts, t.texts()))
}
type UIResultMap map[string]UIResults
// FilterUIResults filters ui icons, the former the uiTypes, the higher the priority
func (u UIResultMap) FilterUIResults(uiTypes []string) (uiResults UIResults, err error) {
var ok bool
for _, uiType := range uiTypes {
uiResults, ok = u[uiType]
if ok && len(uiResults) != 0 {
return
}
}
err = errors.Wrap(code.CVResultNotFoundError, fmt.Sprintf("UI types %v not detected", uiTypes))
return
}
type UIResult struct {
Box
}
type Box struct {
Point PointF `json:"point"`
Width float64 `json:"width"`
Height float64 `json:"height"`
}
func (box Box) IsEmpty() bool {
return builtin.IsZeroFloat64(box.Width) && builtin.IsZeroFloat64(box.Height)
}
func (box Box) IsIdentical(box2 Box) bool {
// set the coordinate precision to 1 pixel
return box.Point.IsIdentical(box2.Point) &&
builtin.IsZeroFloat64(math.Abs(box.Width-box2.Width)) &&
builtin.IsZeroFloat64(math.Abs(box.Height-box2.Height))
}
func (box Box) Center() PointF {
return PointF{
X: box.Point.X + box.Width*0.5,
Y: box.Point.Y + box.Height*0.5,
}
}
type UIResults []UIResult
func (u UIResults) FilterScope(scope AbsScope) (results UIResults) {
for _, uiResult := range u {
rect := image.Rectangle{
Min: image.Point{
X: int(uiResult.Point.X),
Y: int(uiResult.Point.Y),
},
Max: image.Point{
X: int(uiResult.Point.X + uiResult.Width),
Y: int(uiResult.Point.Y + uiResult.Height),
},
}
// check if ui result in scope
if len(scope) == 4 {
if rect.Min.X < scope[0] ||
rect.Min.Y < scope[1] ||
rect.Max.X > scope[2] ||
rect.Max.Y > scope[3] {
// not in scope
continue
}
}
results = append(results, uiResult)
}
return
}
func (u UIResults) GetUIResult(options ...ActionOption) (UIResult, error) {
actionOptions := NewActionOptions(options...)
uiResults := u.FilterScope(actionOptions.AbsScope)
if len(uiResults) == 0 {
return UIResult{}, errors.Wrap(code.CVResultNotFoundError,
"ui types not found in scope")
}
// get index
idx := actionOptions.Index
if idx < 0 {
idx = len(uiResults) + idx
}
// index out of range
if idx >= len(uiResults) || idx < 0 {
return UIResult{}, errors.Wrap(code.CVResultNotFoundError,
fmt.Sprintf("ui types index %d out of range", idx))
}
return uiResults[idx], nil
}
// ClosePopupsResult represents the result of recognized popup to close
type ClosePopupsResult struct {
Type string `json:"type"`
PopupArea Box `json:"popupArea"`
CloseArea Box `json:"closeArea"`
Text string `json:"text"`
}
func (c ClosePopupsResult) IsEmpty() bool {
return c.PopupArea.IsEmpty() && c.CloseArea.IsEmpty()
}

248
pkg/uixt/ai_vedem.go Normal file
View File

@@ -0,0 +1,248 @@
package uixt
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/internal/json"
)
var client = &http.Client{
Timeout: time.Second * 10,
}
type APIResponseImage struct {
Code int `json:"code"`
Message string `json:"message"`
Result ImageResult `json:"result"`
}
func newVEDEMImageService() (*veDEMImageService, error) {
if err := checkEnv(); err != nil {
return nil, err
}
return &veDEMImageService{}, nil
}
// veDEMImageService implements IImageService interface
// actions:
//
// ocr - get ocr texts
// upload - get image uploaded url
// liveType - get live type
// popup - get popup windows
// close - get close popup
// ui - get ui position by type(s)
type veDEMImageService struct{}
func (s *veDEMImageService) GetImage(imageBuf *bytes.Buffer, options ...ActionOption) (imageResult *ImageResult, err error) {
actionOptions := NewActionOptions(options...)
screenshotActions := actionOptions.screenshotActions()
if len(screenshotActions) == 0 {
// skip
return nil, nil
}
start := time.Now()
defer func() {
elapsed := time.Since(start).Milliseconds()
var logger *zerolog.Event
if err != nil {
logger = log.Error().Err(err)
} else {
logger = log.Debug()
if imageResult.URL != "" {
logger = logger.Str("url", imageResult.URL)
}
if imageResult.UIResult != nil {
logger = logger.Interface("uiResult", imageResult.UIResult)
}
if imageResult.ClosePopupsResult != nil {
if imageResult.ClosePopupsResult.IsEmpty() {
// set nil to reduce unnecessary summary info
imageResult.ClosePopupsResult = nil
} else {
logger = logger.Interface("closePopupsResult", imageResult.ClosePopupsResult)
}
}
}
logger = logger.Int64("elapsed(ms)", elapsed)
logger.Msg("get image data by veDEM")
}()
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
for _, action := range screenshotActions {
bodyWriter.WriteField("actions", action)
}
for _, uiType := range actionOptions.ScreenShotWithUITypes {
bodyWriter.WriteField("uiTypes", uiType)
}
// 使用高精度集群
bodyWriter.WriteField("ocrCluster", "highPrecision")
if actionOptions.ScreenShotWithOCRCluster != "" {
bodyWriter.WriteField("ocrCluster", actionOptions.ScreenShotWithOCRCluster)
}
if actionOptions.Timeout > 0 {
bodyWriter.WriteField("timeout", fmt.Sprintf("%v", actionOptions.Timeout))
} else {
bodyWriter.WriteField("timeout", fmt.Sprintf("%v", 10))
}
formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png")
if err != nil {
err = errors.Wrap(code.CVRequestError,
fmt.Sprintf("create form file error: %v", err))
return
}
size, err := formWriter.Write(imageBuf.Bytes())
if err != nil {
err = errors.Wrap(code.CVRequestError,
fmt.Sprintf("write form error: %v", err))
return
}
err = bodyWriter.Close()
if err != nil {
err = errors.Wrap(code.CVRequestError,
fmt.Sprintf("close body writer error: %v", err))
return
}
var req *http.Request
var resp *http.Response
// retry 3 times
for i := 1; i <= 3; i++ {
copiedBodyBuf := &bytes.Buffer{}
if _, err := copiedBodyBuf.Write(bodyBuf.Bytes()); err != nil {
log.Error().Err(err).Msg("copy screenshot buffer failed")
continue
}
req, err = http.NewRequest("POST", os.Getenv("VEDEM_IMAGE_URL"), copiedBodyBuf)
if err != nil {
err = errors.Wrap(code.CVRequestError,
fmt.Sprintf("construct request error: %v", err))
return
}
// ppe env
// req.Header.Add("x-tt-env", "ppe_vedem_algorithm")
// req.Header.Add("x-use-ppe", "1")
signToken := "UNSIGNED-PAYLOAD"
token := builtin.Sign("auth-v2", os.Getenv("VEDEM_IMAGE_AK"), os.Getenv("VEDEM_IMAGE_SK"), []byte(signToken))
req.Header.Add("Agw-Auth", token)
req.Header.Add("Agw-Auth-Content", signToken)
req.Header.Add("Content-Type", bodyWriter.FormDataContentType())
start := time.Now()
resp, err = client.Do(req)
elapsed := time.Since(start)
if err != nil {
log.Error().Err(err).
Int("imageBufSize", size).
Msgf("request veDEM OCR service error, retry %d", i)
continue
}
logID := getLogID(resp.Header)
statusCode := resp.StatusCode
if statusCode != http.StatusOK {
log.Error().
Str("X-TT-LOGID", logID).
Int("imageBufSize", size).
Int("statusCode", statusCode).
Msgf("request veDEM OCR service failed, retry %d", i)
time.Sleep(1 * time.Second)
continue
}
log.Debug().
Str("X-TT-LOGID", logID).
Int("image_bytes", size).
Int64("elapsed(ms)", elapsed.Milliseconds()).
Msg("request OCR service success")
break
}
if resp == nil {
err = code.CVServiceConnectionError
return
}
defer resp.Body.Close()
results, err := io.ReadAll(resp.Body)
if err != nil {
err = errors.Wrap(code.CVResponseError,
fmt.Sprintf("read response body error: %v", err))
return
}
if resp.StatusCode != http.StatusOK {
err = errors.Wrap(code.CVResponseError,
fmt.Sprintf("unexpected response status code: %d, results: %v",
resp.StatusCode, string(results)))
return
}
var imageResponse APIResponseImage
err = json.Unmarshal(results, &imageResponse)
if err != nil {
err = errors.Wrap(code.CVResponseError,
fmt.Sprintf("json unmarshal veDEM image response body error, response=%s", string(results)))
return
}
if imageResponse.Code != 0 {
err = errors.Wrap(code.CVResponseError,
fmt.Sprintf("unexpected response data code: %d, message: %s",
imageResponse.Code, imageResponse.Message))
return
}
imageResult = &imageResponse.Result
return imageResult, nil
}
func checkEnv() error {
vedemImageURL := os.Getenv("VEDEM_IMAGE_URL")
if vedemImageURL == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_URL missed")
}
log.Info().Str("VEDEM_IMAGE_URL", vedemImageURL).Msg("get env")
if os.Getenv("VEDEM_IMAGE_AK") == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_AK missed")
}
if os.Getenv("VEDEM_IMAGE_SK") == "" {
return errors.Wrap(code.CVEnvMissedError, "VEDEM_IMAGE_SK missed")
}
return nil
}
func getLogID(header http.Header) string {
if len(header) == 0 {
return ""
}
logID, ok := header["X-Tt-Logid"]
if !ok || len(logID) == 0 {
return ""
}
return logID[0]
}

78
pkg/uixt/ai_vedem_test.go Normal file
View File

@@ -0,0 +1,78 @@
//go:build localtest
package uixt
import (
"bytes"
"fmt"
"os"
"testing"
)
func checkOCR(buff *bytes.Buffer) error {
service, err := newVEDEMImageService()
if err != nil {
return err
}
imageResult, err := service.GetImage(buff)
if err != nil {
return err
}
fmt.Println(fmt.Sprintf("imageResult: %v", imageResult))
return nil
}
func TestOCRWithScreenshot(t *testing.T) {
setupAndroidAdbDriver(t)
raw, err := driverExt.Driver.Screenshot()
if err != nil {
t.Fatal(err)
}
if err := checkOCR(raw); err != nil {
t.Fatal(err)
}
}
func TestOCRWithLocalFile(t *testing.T) {
imagePath := "/Users/debugtalk/Downloads/s1.png"
file, err := os.ReadFile(imagePath)
if err != nil {
t.Fatal(err)
}
buf := new(bytes.Buffer)
buf.Read(file)
if err := checkOCR(buf); err != nil {
t.Fatal(err)
}
}
func TestTapUIWithScreenshot(t *testing.T) {
serialNumber := os.Getenv("SERIAL_NUMBER")
device, _ := NewAndroidDevice(WithSerialNumber(serialNumber))
driver, err := device.NewDriver()
if err != nil {
t.Fatal(err)
}
err = driver.TapByUIDetection(
WithScreenShotUITypes("dyhouse", "shoppingbag"))
if err != nil {
t.Fatal(err)
}
}
func TestDriverExtOCR(t *testing.T) {
driverExt, err := iosDevice.NewDriver()
checkErr(t, err)
point, err := driverExt.FindScreenText("抖音")
checkErr(t, err)
t.Logf("point.X: %v, point.Y: %v", point.X, point.Y)
driverExt.Driver.Tap(point.X, point.Y-20)
}

1097
pkg/uixt/android_adb_driver.go Normal file
View File

File diff suppressed because it is too large Load Diff

1108
pkg/uixt/android_device.go Normal file
View File

File diff suppressed because it is too large Load Diff

881
pkg/uixt/android_key.go Normal file
View File

@@ -0,0 +1,881 @@
package uixt
// See https://developer.android.com/reference/android/view/KeyEvent
type KeyMeta int
const (
KMEmpty KeyMeta = 0 // As a `null`
KMCapLocked KeyMeta = 0x100 // SHIFT key locked in CAPS mode.
KMAltLocked KeyMeta = 0x200 // ALT key locked.
KMSymLocked KeyMeta = 0x400 // SYM key locked.
KMSelecting KeyMeta = 0x800 // Text is in selection mode.
KMAltOn KeyMeta = 0x02 // This mask is used to check whether one of the ALT meta keys is pressed.
KMAltLeftOn KeyMeta = 0x10 // This mask is used to check whether the left ALT meta key is pressed.
KMAltRightOn KeyMeta = 0x20 // This mask is used to check whether the right the ALT meta key is pressed.
KMShiftOn KeyMeta = 0x1 // This mask is used to check whether one of the SHIFT meta keys is pressed.
KMShiftLeftOn KeyMeta = 0x40 // This mask is used to check whether the left SHIFT meta key is pressed.
KMShiftRightOn KeyMeta = 0x80 // This mask is used to check whether the right SHIFT meta key is pressed.
KMSymOn KeyMeta = 0x4 // This mask is used to check whether the SYM meta key is pressed.
KMFunctionOn KeyMeta = 0x8 // This mask is used to check whether the FUNCTION meta key is pressed.
KMCtrlOn KeyMeta = 0x1000 // This mask is used to check whether one of the CTRL meta keys is pressed.
KMCtrlLeftOn KeyMeta = 0x2000 // This mask is used to check whether the left CTRL meta key is pressed.
KMCtrlRightOn KeyMeta = 0x4000 // This mask is used to check whether the right CTRL meta key is pressed.
KMMetaOn KeyMeta = 0x10000 // This mask is used to check whether one of the META meta keys is pressed.
KMMetaLeftOn KeyMeta = 0x20000 // This mask is used to check whether the left META meta key is pressed.
KMMetaRightOn KeyMeta = 0x40000 // This mask is used to check whether the right META meta key is pressed.
KMCapsLockOn KeyMeta = 0x100000 // This mask is used to check whether the CAPS LOCK meta key is on.
KMNumLockOn KeyMeta = 0x200000 // This mask is used to check whether the NUM LOCK meta key is on.
KMScrollLockOn KeyMeta = 0x400000 // This mask is used to check whether the SCROLL LOCK meta key is on.
KMShiftMask = KMShiftOn | KMShiftLeftOn | KMShiftRightOn
KMAltMask = KMAltOn | KMAltLeftOn | KMAltRightOn
KMCtrlMask = KMCtrlOn | KMCtrlLeftOn | KMCtrlRightOn
KMMetaMask = KMMetaOn | KMMetaLeftOn | KMMetaRightOn
)
type KeyFlag int
const (
// KFWokeHere This mask is set if the device woke because of this key event.
// Deprecated
KFWokeHere KeyFlag = 0x1
// KFSoftKeyboard This mask is set if the key event was generated by a software keyboard.
KFSoftKeyboard KeyFlag = 0x2
// KFKeepTouchMode This mask is set if we don't want the key event to cause us to leave touch mode.
KFKeepTouchMode KeyFlag = 0x4
// KFFromSystem This mask is set if an event was known to come from a trusted part
// of the system. That is, the event is known to come from the user,
// and could not have been spoofed by a third party component.
KFFromSystem KeyFlag = 0x8
// KFEditorAction This mask is used for compatibility, to identify enter keys that are
// coming from an IME whose enter key has been auto-labelled "next" or
// "done". This allows TextView to dispatch these as normal enter keys
// for old applications, but still do the appropriate action when receiving them.
KFEditorAction KeyFlag = 0x10
// KFCanceled When associated with up key events, this indicates that the key press
// has been canceled. Typically this is used with virtual touch screen
// keys, where the user can slide from the virtual key area on to the
// display: in that case, the application will receive a canceled up
// event and should not perform the action normally associated with the
// key. Note that for this to work, the application can not perform an
// action for a key until it receives an up or the long press timeout has expired.
KFCanceled KeyFlag = 0x20
// KFVirtualHardKey This key event was generated by a virtual (on-screen) hard key area.
// Typically this is an area of the touchscreen, outside of the regular
// display, dedicated to "hardware" buttons.
KFVirtualHardKey KeyFlag = 0x40
// KFLongPress This flag is set for the first key repeat that occurs after the long press timeout.
KFLongPress KeyFlag = 0x80
// KFCanceledLongPress Set when a key event has `KFCanceled` set because a long
// press action was executed while it was down.
KFCanceledLongPress KeyFlag = 0x100
// KFTracking Set for `ACTION_UP` when this event's key code is still being
// tracked from its initial down. That is, somebody requested that tracking
// started on the key down and a long press has not caused
// the tracking to be canceled.
KFTracking KeyFlag = 0x200
// KFFallback Set when a key event has been synthesized to implement default behavior
// for an event that the application did not handle.
// Fallback key events are generated by unhandled trackball motions
// (to emulate a directional keypad) and by certain unhandled key presses
// that are declared in the key map (such as special function numeric keypad
// keys when numlock is off).
KFFallback KeyFlag = 0x400
// KFPredispatch Signifies that the key is being predispatched.
// KFPredispatch KeyFlag = 0x20000000
// KFStartTracking Private control to determine when an app is tracking a key sequence.
// KFStartTracking KeyFlag = 0x40000000
// KFTainted Private flag that indicates when the system has detected that this key event
// may be inconsistent with respect to the sequence of previously delivered key events,
// such as when a key up event is sent but the key was not down.
// KFTainted KeyFlag = 0x80000000
)
type KeyCode int
const (
_ KeyCode = 0 // Unknown key code.
// KCSoftLeft Soft Left key
// Usually situated below the display on phones and used as a multi-function
// feature key for selecting a software defined function shown on the bottom left
// of the display.
KCSoftLeft KeyCode = 1
// KCSoftRight Soft Right key.
// Usually situated below the display on phones and used as a multi-function
// feature key for selecting a software defined function shown on the bottom right
// of the display.
KCSoftRight KeyCode = 2
// KCHome Home key.
// This key is handled by the framework and is never delivered to applications.
KCHome KeyCode = 3
KCBack KeyCode = 4 // Back key
KCCall KeyCode = 5 // Call key
KCEndCall KeyCode = 6 // End Call key
KC0 KeyCode = 7 // '0' key
KC1 KeyCode = 8 // '1' key
KC2 KeyCode = 9 // '2' key
KC3 KeyCode = 10 // '3' key
KC4 KeyCode = 11 // '4' key
KC5 KeyCode = 12 // '5' key
KC6 KeyCode = 13 // '6' key
KC7 KeyCode = 14 // '7' key
KC8 KeyCode = 15 // '8' key
KC9 KeyCode = 16 // '9' key
KCStar KeyCode = 17 // '*' key
KCPound KeyCode = 18 // '#' key
// KCDPadUp KeycodeDPadUp Directional Pad Up key.
// May also be synthesized from trackball motions.
KCDPadUp KeyCode = 19
// KCDPadDown Directional Pad Down key.
// May also be synthesized from trackball motions.
KCDPadDown KeyCode = 20
// KCDPadLeft Directional Pad Left key.
// May also be synthesized from trackball motions.
KCDPadLeft KeyCode = 21
// KCDPadRight Directional Pad Right key.
// May also be synthesized from trackball motions.
KCDPadRight KeyCode = 22
// KCDPadCenter Directional Pad Center key.
// May also be synthesized from trackball motions.
KCDPadCenter KeyCode = 23
// KCVolumeUp Volume Up key.
// Adjusts the speaker volume up.
KCVolumeUp KeyCode = 24
// KCVolumeDown Volume Down key.
// Adjusts the speaker volume down.
KCVolumeDown KeyCode = 25
// KCPower Power key.
KCPower KeyCode = 26
// KCCamera Camera key.
// Used to launch a camera application or take pictures.
KCCamera KeyCode = 27
KCClear KeyCode = 28 // Clear key
KCa KeyCode = 29 // 'a' key
KCb KeyCode = 30 // 'b' key
KCc KeyCode = 31 // 'c' key
KCd KeyCode = 32 // 'd' key
KCe KeyCode = 33 // 'e' key
KCf KeyCode = 34 // 'f' key
KCg KeyCode = 35 // 'g' key
KCh KeyCode = 36 // 'h' key
KCi KeyCode = 37 // 'i' key
KCj KeyCode = 38 // 'j' key
KCk KeyCode = 39 // 'k' key
KCl KeyCode = 40 // 'l' key
KCm KeyCode = 41 // 'm' key
KCn KeyCode = 42 // 'n' key
KCo KeyCode = 43 // 'o' key
KCp KeyCode = 44 // 'p' key
KCq KeyCode = 45 // 'q' key
KCr KeyCode = 46 // 'r' key
KCs KeyCode = 47 // 's' key
KCt KeyCode = 48 // 't' key
KCu KeyCode = 49 // 'u' key
KCv KeyCode = 50 // 'v' key
KCw KeyCode = 51 // 'w' key
KCx KeyCode = 52 // 'x' key
KCy KeyCode = 53 // 'y' key
KCz KeyCode = 54 // 'z' key
KCComma KeyCode = 55 // ',' key
KCPeriod KeyCode = 56 // '.' key
KCAltLeft KeyCode = 57 // Left Alt modifier key
KCAltRight KeyCode = 58 // Right Alt modifier key
KCShiftLeft KeyCode = 59 // Left Shift modifier key
KCShiftRight KeyCode = 60 // Right Shift modifier key
KCTab KeyCode = 61 // Tab key
KCSpace KeyCode = 62 // Space key
// KCSym Symbol modifier key.
// Used to enter alternate symbols.
KCSym KeyCode = 63
// KCExplorer Explorer special function key.
// Used to launch a browser application.
KCExplorer KeyCode = 64
// KCEnvelope Envelope special function key.
// Used to launch a mail application.
KCEnvelope KeyCode = 65
// KCEnter Enter key.
KCEnter KeyCode = 66
// KCDel Backspace key.
// Deletes characters before the insertion point, unlike `KCForwardDel`.
KCDel KeyCode = 67
KCGrave KeyCode = 68 // '`' (backtick) key
KCMinus KeyCode = 69 // '-'
KCEquals KeyCode = 70 // '=' key
KCLeftBracket KeyCode = 71 // '[' key
KCRightBracket KeyCode = 72 // ']' key
KCBackslash KeyCode = 73 // '\' key
KCSemicolon KeyCode = 74 // '' key
KCApostrophe KeyCode = 75 // ''' (apostrophe) key
KCSlash KeyCode = 76 // '/' key
KCAt KeyCode = 77 // '@' key
// KCNum Number modifier key.
// Used to enter numeric symbols.
// This key is not Num Lock; it is more like `KCAltLeft` and is
// interpreted as an ALT key by {@link android.text.method.MetaKeyKeyListener}.
KCNum KeyCode = 78
// KCHeadsetHook Headset Hook key.
// Used to hang up calls and stop media.
KCHeadsetHook KeyCode = 79
// KCFocus Camera Focus key.
// Used to focus the camera.
// *Camera* focus
KCFocus KeyCode = 80
KCPlus KeyCode = 81 // '+' key.
KCMenu KeyCode = 82 // Menu key.
KCNotification KeyCode = 83 // Notification key.
KCSearch KeyCode = 84 // Search key.
KCMediaPlayPause KeyCode = 85 // Play/Pause media key.
KCMediaStop KeyCode = 86 // Stop media key.
KCMediaNext KeyCode = 87 // Play Next media key.
KCMediaPrevious KeyCode = 88 // Play Previous media key.
KCMediaRewind KeyCode = 89 // Rewind media key.
KCMediaFastForward KeyCode = 90 // Fast Forward media key.
// KCMute Mute key.
// Mutes the microphone, unlike `KCVolumeMute`
KCMute KeyCode = 91
// KCPageUp Page Up key.
KCPageUp KeyCode = 92
// KCPageDown Page Down key.
KCPageDown KeyCode = 93
// KCPictSymbols Picture Symbols modifier key.
// Used to switch symbol sets (Emoji, Kao-moji).
// switch symbol-sets (Emoji,Kao-moji)
KCPictSymbols KeyCode = 94
// KCSwitchCharset Switch Charset modifier key.
// Used to switch character sets (Kanji, Katakana).
// switch char-sets (Kanji,Katakana)
KCSwitchCharset KeyCode = 95
// KCButtonA A Button key.
// On a game controller, the A button should be either the button labeled A
// or the first button on the bottom row of controller buttons.
KCButtonA KeyCode = 96
// KCButtonB B Button key.
// On a game controller, the B button should be either the button labeled B
// or the second button on the bottom row of controller buttons.
KCButtonB KeyCode = 97
// KCButtonC C Button key.
// On a game controller, the C button should be either the button labeled C
// or the third button on the bottom row of controller buttons.
KCButtonC KeyCode = 98
// KCButtonX X Button key.
// On a game controller, the X button should be either the button labeled X
// or the first button on the upper row of controller buttons.
KCButtonX KeyCode = 99
// KCButtonY Y Button key.
// On a game controller, the Y button should be either the button labeled Y
// or the second button on the upper row of controller buttons.
KCButtonY KeyCode = 100
// KCButtonZ Z Button key.
// On a game controller, the Z button should be either the button labeled Z
// or the third button on the upper row of controller buttons.
KCButtonZ KeyCode = 101
// KCButtonL1 L1 Button key.
// On a game controller, the L1 button should be either the button labeled L1 (or L)
// or the top left trigger button.
KCButtonL1 KeyCode = 102
// KCButtonR1 R1 Button key.
// On a game controller, the R1 button should be either the button labeled R1 (or R)
// or the top right trigger button.
KCButtonR1 KeyCode = 103
// KCButtonL2 L2 Button key.
// On a game controller, the L2 button should be either the button labeled L2
// or the bottom left trigger button.
KCButtonL2 KeyCode = 104
// KCButtonR2 R2 Button key.
// On a game controller, the R2 button should be either the button labeled R2
// or the bottom right trigger button.
KCButtonR2 KeyCode = 105
// KCButtonTHUMBL Left Thumb Button key.
// On a game controller, the left thumb button indicates that the left (or only)
// joystick is pressed.
KCButtonTHUMBL KeyCode = 106
// KCButtonTHUMBR Right Thumb Button key.
// On a game controller, the right thumb button indicates that the right
// joystick is pressed.
KCButtonTHUMBR KeyCode = 107
// KCButtonStart Start Button key.
// On a game controller, the button labeled Start.
KCButtonStart KeyCode = 108
// KCButtonSelect Select Button key.
// On a game controller, the button labeled Select.
KCButtonSelect KeyCode = 109
// KCButtonMode Mode Button key.
// On a game controller, the button labeled Mode.
KCButtonMode KeyCode = 110
// KCEscape Escape key.
KCEscape KeyCode = 111
// KCForwardDel Forward Delete key.
// Deletes characters ahead of the insertion point, unlike `KCDel`.
KCForwardDel KeyCode = 112
KCCtrlLeft KeyCode = 113 // Left Control modifier key
KCCtrlRight KeyCode = 114 // Right Control modifier key
KCCapsLock KeyCode = 115 // Caps Lock key
KCScrollLock KeyCode = 116 // Scroll Lock key
KCMetaLeft KeyCode = 117 // Left Meta modifier key
KCMetaRight KeyCode = 118 // Right Meta modifier key
KCFunction KeyCode = 119 // Function modifier key
KCSysRq KeyCode = 120 // System Request / Print Screen key
KCBreak KeyCode = 121 // Break / Pause key
// KCMoveHome Home Movement key.
// Used for scrolling or moving the cursor around to the start of a line
// or to the top of a list.
KCMoveHome KeyCode = 122
// KCMoveEnd End Movement key.
// Used for scrolling or moving the cursor around to the end of a line
// or to the bottom of a list.
KCMoveEnd KeyCode = 123
// KCInsert Insert key.
// Toggles insert / overwrite edit mode.
KCInsert KeyCode = 124
// KCForward Forward key.
// Navigates forward in the history stack. Complement of `KCBack`.
KCForward KeyCode = 125
// KCMediaPlay Play media key.
KCMediaPlay KeyCode = 126
// KCMediaPause Pause media key.
KCMediaPause KeyCode = 127
// KCMediaClose Close media key.
// May be used to close a CD tray, for example.
KCMediaClose KeyCode = 128
// KCMediaEject Eject media key.
// May be used to eject a CD tray, for example.
KCMediaEject KeyCode = 129
// KCMediaRecord Record media key.
KCMediaRecord KeyCode = 130
KCF1 KeyCode = 131 // F1 key.
KCF2 KeyCode = 132 // F2 key.
KCF3 KeyCode = 133 // F3 key.
KCF4 KeyCode = 134 // F4 key.
KCF5 KeyCode = 135 // F5 key.
KCF6 KeyCode = 136 // F6 key.
KCF7 KeyCode = 137 // F7 key.
KCF8 KeyCode = 138 // F8 key.
KCF9 KeyCode = 139 // F9 key.
KCF10 KeyCode = 140 // F10 key.
KCF11 KeyCode = 141 // F11 key.
KCF12 KeyCode = 142 // F12 key.
// KCNumLock Num Lock key.
// This is the Num Lock key; it is different from `KCNum`.
// This key alters the behavior of other keys on the numeric keypad.
KCNumLock KeyCode = 143
KCNumpad0 KeyCode = 144 // Numeric keypad '0' key
KCNumpad1 KeyCode = 145 // Numeric keypad '1' key
KCNumpad2 KeyCode = 146 // Numeric keypad '2' key
KCNumpad3 KeyCode = 147 // Numeric keypad '3' key
KCNumpad4 KeyCode = 148 // Numeric keypad '4' key
KCNumpad5 KeyCode = 149 // Numeric keypad '5' key
KCNumpad6 KeyCode = 150 // Numeric keypad '6' key
KCNumpad7 KeyCode = 151 // Numeric keypad '7' key
KCNumpad8 KeyCode = 152 // Numeric keypad '8' key
KCNumpad9 KeyCode = 153 // Numeric keypad '9' key
KCNumpadDivide KeyCode = 154 // Numeric keypad '/' key (for division)
KCNumpadMultiply KeyCode = 155 // Numeric keypad '*' key (for multiplication)
KCNumpadSubtract KeyCode = 156 // Numeric keypad '-' key (for subtraction)
KCNumpadAdd KeyCode = 157 // Numeric keypad '+' key (for addition)
KCNumpadDot KeyCode = 158 // Numeric keypad '.' key (for decimals or digit grouping)
KCNumpadComma KeyCode = 159 // Numeric keypad ',' key (for decimals or digit grouping)
KCNumpadEnter KeyCode = 160 // Numeric keypad Enter key
KCNumpadEquals KeyCode = 161 // Numeric keypad 'KeyCode =' key
KCNumpadLeftParen KeyCode = 162 // Numeric keypad '(' key
KCNumpadRightParen KeyCode = 163 // Numeric keypad ')' key
// KCVolumeMute Volume Mute key.
// Mutes the speaker, unlike `KCMute`.
// This key should normally be implemented as a toggle such that the first press
// mutes the speaker and the second press restores the original volume.
KCVolumeMute KeyCode = 164
// KCInfo Info key.
// Common on TV remotes to show additional information related to what is
// currently being viewed.
KCInfo KeyCode = 165
// KCChannelUp Channel up key.
// On TV remotes, increments the television channel.
KCChannelUp KeyCode = 166
// KCChannelDown Channel down key.
// On TV remotes, decrements the television channel.
KCChannelDown KeyCode = 167
// KCZoomIn Zoom in key.
KCZoomIn KeyCode = 168
// KCZoomOut Zoom out key.
KCZoomOut KeyCode = 169
// KCTv TV key.
// On TV remotes, switches to viewing live TV.
KCTv KeyCode = 170
// KCWindow Window key.
// On TV remotes, toggles picture-in-picture mode or other windowing functions.
// On Android Wear devices, triggers a display offset.
KCWindow KeyCode = 171
// KCGuide Guide key.
// On TV remotes, shows a programming guide.
KCGuide KeyCode = 172
// KCDvr DVR key.
// On some TV remotes, switches to a DVR mode for recorded shows.
KCDvr KeyCode = 173
// KCBookmark Bookmark key.
// On some TV remotes, bookmarks content or web pages.
KCBookmark KeyCode = 174
// KCCaptions Toggle captions key.
// Switches the mode for closed-captioning text, for example during television shows.
KCCaptions KeyCode = 175
// KCSettings Settings key.
// Starts the system settings activity.
KCSettings KeyCode = 176
// KCTvPower TV power key.
// On TV remotes, toggles the power on a television screen.
KCTvPower KeyCode = 177
// KCTvInput TV input key.
// On TV remotes, switches the input on a television screen.
KCTvInput KeyCode = 178
// KCStbPower Set-top-box power key.
// On TV remotes, toggles the power on an external Set-top-box.
KCStbPower KeyCode = 179
// KCStbInput Set-top-box input key.
// On TV remotes, switches the input mode on an external Set-top-box.
KCStbInput KeyCode = 180
// KCAvrPower A/V Receiver power key.
// On TV remotes, toggles the power on an external A/V Receiver.
KCAvrPower KeyCode = 181
// KCAvrInput A/V Receiver input key.
// On TV remotes, switches the input mode on an external A/V Receiver.
KCAvrInput KeyCode = 182
// KCProgRed Red "programmable" key.
// On TV remotes, acts as a contextual/programmable key.
KCProgRed KeyCode = 183
// KCProgGreen Green "programmable" key.
// On TV remotes, actsas a contextual/programmable key.
KCProgGreen KeyCode = 184
// KCProgYellow Yellow "programmable" key.
// On TV remotes, acts as a contextual/programmable key.
KCProgYellow KeyCode = 185
// KCProgBlue Blue "programmable" key.
// On TV remotes, acts as a contextual/programmable key.
KCProgBlue KeyCode = 186
// KCAppSwitch App switch key.
// Should bring up the application switcher dialog.
KCAppSwitch KeyCode = 187
KCButton1 KeyCode = 188 // Generic Game Pad Button #1
KCButton2 KeyCode = 189 // Generic Game Pad Button #2
KCButton3 KeyCode = 190 // Generic Game Pad Button #3
KCButton4 KeyCode = 191 // Generic Game Pad Button #4
KCButton5 KeyCode = 192 // Generic Game Pad Button #5
KCButton6 KeyCode = 193 // Generic Game Pad Button #6
KCButton7 KeyCode = 194 // Generic Game Pad Button #7
KCButton8 KeyCode = 195 // Generic Game Pad Button #8
KCButton9 KeyCode = 196 // Generic Game Pad Button #9
KCButton10 KeyCode = 197 // Generic Game Pad Button #10
KCButton11 KeyCode = 198 // Generic Game Pad Button #11
KCButton12 KeyCode = 199 // Generic Game Pad Button #12
KCButton13 KeyCode = 200 // Generic Game Pad Button #13
KCButton14 KeyCode = 201 // Generic Game Pad Button #14
KCButton15 KeyCode = 202 // Generic Game Pad Button #15
KCButton16 KeyCode = 203 // Generic Game Pad Button #16
// KCLanguageSwitch Language Switch key.
// Toggles the current input language such as switching between English and Japanese on
// a QWERTY keyboard. On some devices, the same function may be performed by
// pressing Shift+Spacebar.
KCLanguageSwitch KeyCode = 204
// Manner Mode key.
// Toggles silent or vibrate mode on and off to make the device behave more politely
// in certain settings such as on a crowded train. On some devices, the key may only
// operate when long-pressed.
KCMannerMode KeyCode = 205
// 3D Mode key.
// Toggles the display between 2D and 3D mode.
KC3dMode KeyCode = 206
// Contacts special function key.
// Used to launch an address book application.
KCContacts KeyCode = 207
// Calendar special function key.
// Used to launch a calendar application.
KCCalendar KeyCode = 208
// Music special function key.
// Used to launch a music player application.
KCMusic KeyCode = 209
// Calculator special function key.
// Used to launch a calculator application.
KCCalculator KeyCode = 210
// Japanese full-width / half-width key.
KCZenkakuHankaku KeyCode = 211
// Japanese alphanumeric key.
KCEisu KeyCode = 212
// Japanese non-conversion key.
KCMuhenkan KeyCode = 213
// Japanese conversion key.
KCHenkan KeyCode = 214
// Japanese katakana / hiragana key.
KCKatakanaHiragana KeyCode = 215
// Japanese Yen key.
KCYen KeyCode = 216
// Japanese Ro key.
KCRo KeyCode = 217
// Japanese kana key.
KCKana KeyCode = 218
// Assist key.
// Launches the global assist activity. Not delivered to applications.
KCAssist KeyCode = 219
// Brightness Down key.
// Adjusts the screen brightness down.
KCBrightnessDown KeyCode = 220
// Brightness Up key.
// Adjusts the screen brightness up.
KCBrightnessUp KeyCode = 221
// Audio Track key.
// Switches the audio tracks.
KCMediaAudioTrack KeyCode = 222
// Sleep key.
// Puts the device to sleep. Behaves somewhat like {@link #KEYCODE_POWER} but it
// has no effect if the device is already asleep.
KCSleep KeyCode = 223
// Wakeup key.
// Wakes up the device. Behaves somewhat like {@link #KEYCODE_POWER} but it
// has no effect if the device is already awake.
KCWakeup KeyCode = 224
// Pairing key.
// Initiates peripheral pairing mode. Useful for pairing remote control
// devices or game controllers, especially if no other input mode is
// available.
KCPairing KeyCode = 225
// Media Top Menu key.
// Goes to the top of media menu.
KCMediaTopMenu KeyCode = 226
// '11' key.
KC11 KeyCode = 227
// '12' key.
KC12 KeyCode = 228
// Last Channel key.
// Goes to the last viewed channel.
KCLastChannel KeyCode = 229
// TV data service key.
// Displays data services like weather, sports.
KCTvDataService KeyCode = 230
// Voice Assist key.
// Launches the global voice assist activity. Not delivered to applications.
KCVoiceAssist KeyCode = 231
// Radio key.
// Toggles TV service / Radio service.
KCTvRadioService KeyCode = 232
// Teletext key.
// Displays Teletext service.
KCTvTeletext KeyCode = 233
// Number entry key.
// Initiates to enter multi-digit channel nubmber when each digit key is assigned
// for selecting separate channel. Corresponds to Number Entry Mode (0x1D) of CEC
// User Control Code.
KCTvNumberEntry KeyCode = 234
// Analog Terrestrial key.
// Switches to analog terrestrial broadcast service.
KCTvTerrestrialAnalog KeyCode = 235
// Digital Terrestrial key.
// Switches to digital terrestrial broadcast service.
KCTvTerrestrialDigital KeyCode = 236
// Satellite key.
// Switches to digital satellite broadcast service.
KCTvSatellite KeyCode = 237
// BS key.
// Switches to BS digital satellite broadcasting service available in Japan.
KCTvSatelliteBs KeyCode = 238
// CS key.
// Switches to CS digital satellite broadcasting service available in Japan.
KCTvSatelliteCs KeyCode = 239
// BS/CS key.
// Toggles between BS and CS digital satellite services.
KCTvSatelliteService KeyCode = 240
// Toggle Network key.
// Toggles selecting broacast services.
KCTvNetwork KeyCode = 241
// Antenna/Cable key.
// Toggles broadcast input source between antenna and cable.
KCTvAntennaCable KeyCode = 242
// HDMI #1 key.
// Switches to HDMI input #1.
KCTvInputHdmi1 KeyCode = 243
// HDMI #2 key.
// Switches to HDMI input #2.
KCTvInputHdmi2 KeyCode = 244
// HDMI #3 key.
// Switches to HDMI input #3.
KCTvInputHdmi3 KeyCode = 245
// HDMI #4 key.
// Switches to HDMI input #4.
KCTvInputHdmi4 KeyCode = 246
// Composite #1 key.
// Switches to composite video input #1.
KCTvInputComposite1 KeyCode = 247
// Composite #2 key.
// Switches to composite video input #2.
KCTvInputComposite2 KeyCode = 248
// Component #1 key.
// Switches to component video input #1.
KCTvInputComponent1 KeyCode = 249
// Component #2 key.
// Switches to component video input #2.
KCTvInputComponent2 KeyCode = 250
// VGA #1 key.
// Switches to VGA (analog RGB) input #1.
KCTvInputVga1 KeyCode = 251
// Audio description key.
// Toggles audio description off / on.
KCTvAudioDescription KeyCode = 252
// Audio description mixing volume up key.
// Louden audio description volume as compared with normal audio volume.
KCTvAudioDescriptionMixUp KeyCode = 253
// Audio description mixing volume down key.
// Lessen audio description volume as compared with normal audio volume.
KCTvAudioDescriptionMixDown KeyCode = 254
// Zoom mode key.
// Changes Zoom mode (Normal, Full, Zoom, Wide-zoom, etc.)
KCTvZoomMode KeyCode = 255
// Contents menu key.
// Goes to the title list. Corresponds to Contents Menu (0x0B) of CEC User Control
// Code
KCTvContentsMenu KeyCode = 256
// Media context menu key.
// Goes to the context menu of media contents. Corresponds to Media Context-sensitive
// Menu (0x11) of CEC User Control Code.
KCTvMediaContextMenu KeyCode = 257
// Timer programming key.
// Goes to the timer recording menu. Corresponds to Timer Programming (0x54) of
// CEC User Control Code.
KCTvTimerProgramming KeyCode = 258
// Help key.
KCHelp KeyCode = 259
// Navigate to previous key.
// Goes backward by one item in an ordered collection of items.
KCNavigatePrevious KeyCode = 260
// Navigate to next key.
// Advances to the next item in an ordered collection of items.
KCNavigateNext KeyCode = 261
// Navigate in key.
// Activates the item that currently has focus or expands to the next level of a navigation
// hierarchy.
KCNavigateIn KeyCode = 262
// Navigate out key.
// Backs out one level of a navigation hierarchy or collapses the item that currently has
// focus.
KCNavigateOut KeyCode = 263
// Primary stem key for Wear
// Main power/reset button on watch.
KCStemPrimary KeyCode = 264
// Generic stem key 1 for Wear
KCStem1 KeyCode = 265
// Generic stem key 2 for Wear
KCStem2 KeyCode = 266
// Generic stem key 3 for Wear
KCStem3 KeyCode = 267
// Directional Pad Up-Left
KCDPadUpLeft KeyCode = 268
// Directional Pad Down-Left
KCDPadDownLeft KeyCode = 269
// Directional Pad Up-Right
KCDPadUpRight KeyCode = 270
// Directional Pad Down-Right
KCDPadDownRight KeyCode = 271
// Skip forward media key.
KCMediaSkipForward KeyCode = 272
// Skip backward media key.
KCMediaSkipBackward KeyCode = 273
// Step forward media key.
// Steps media forward, one frame at a time.
KCMediaStepForward KeyCode = 274
// Step backward media key.
// Steps media backward, one frame at a time.
KCMediaStepBackward KeyCode = 275
// put device to sleep unless a wakelock is held.
KCSoftSleep KeyCode = 276
// Cut key.
KCCut KeyCode = 277
// Copy key.
KCCopy KeyCode = 278
// Paste key.
KCPaste KeyCode = 279
// Consumed by the system for navigation up
KCSystemNavigationUp KeyCode = 280
// Consumed by the system for navigation down
KCSystemNavigationDown KeyCode = 281
// Consumed by the system for navigation left*/
KCSystemNavigationLeft KeyCode = 282
// Consumed by the system for navigation right
KCSystemNavigationRight KeyCode = 283
// Show all apps
KCAllApps KeyCode = 284
// Refresh key.
KCRefresh KeyCode = 285
)

62
pkg/uixt/android_layout.go Normal file
View File

@@ -0,0 +1,62 @@
package uixt
import (
"encoding/xml"
"fmt"
"regexp"
"strconv"
)
type Attributes struct {
Index int `xml:"index,attr"`
Package string `xml:"package,attr"`
Class string `xml:"class,attr"`
Text string `xml:"text,attr"`
ResourceId string `xml:"resource-id,attr"`
Checkable bool `xml:"checkable,attr"`
Checked bool `xml:"checked,attr"`
Clickable bool `xml:"clickable,attr"`
Enabled bool `xml:"enabled,attr"`
Focusable bool `xml:"focusable,attr"`
Focused bool `xml:"focused,attr"`
LongClickable bool `xml:"long-clickable,attr"`
Password bool `xml:"password,attr"`
Scrollable bool `xml:"scrollable,attr"`
Selected bool `xml:"selected,attr"`
Bounds *Bounds `xml:"bounds,attr"`
Displayed bool `xml:"displayed,attr"`
}
type Hierarchy struct {
XMLName xml.Name `xml:"hierarchy"`
Attributes
Layout []Layout `xml:",any"`
}
type Layout struct {
Attributes
Layout []Layout `xml:",any"`
}
type Bounds struct {
X1, Y1, X2, Y2 int
}
func (b *Bounds) Center() (float64, float64) {
return float64(b.X1+b.X2) / 2, float64(b.Y1+b.Y2) / 2
}
func (b *Bounds) UnmarshalXMLAttr(attr xml.Attr) error {
// 正则表达式用于解析格式为"[x1,y1][x2,y2]"
re := regexp.MustCompile(`\[(\d+),(\d+)]\[(\d+),(\d+)]`)
matches := re.FindStringSubmatch(attr.Value)
if matches == nil {
return fmt.Errorf("bounds format is incorrect")
}
// 转换字符串为整数
b.X1, _ = strconv.Atoi(matches[1])
b.Y1, _ = strconv.Atoi(matches[2])
b.X2, _ = strconv.Atoi(matches[3])
b.Y2, _ = strconv.Atoi(matches[4])
return nil
}

294
pkg/uixt/android_stub_driver.go Normal file
View File

@@ -0,0 +1,294 @@
package uixt
import (
"errors"
"fmt"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/internal/json"
)
type stubAndroidDriver struct {
socket net.Conn
seq int
timeout time.Duration
adbDriver
}
const StubSocketName = "com.bytest.device"
type AppLoginInfo struct {
Did string `json:"did,omitempty" yaml:"did,omitempty"`
Uid string `json:"uid,omitempty" yaml:"uid,omitempty"`
IsLogin bool `json:"is_login,omitempty" yaml:"is_login,omitempty"`
}
// newStubAndroidDriver
// 创建stub Driver address为forward后的端口格式127.0.0.1:${port}
func newStubAndroidDriver(address string, urlPrefix string, readTimeout ...time.Duration) (*stubAndroidDriver, error) {
timeout := 10 * time.Second
if len(readTimeout) > 0 {
timeout = readTimeout[0]
}
conn, err := net.Dial("tcp", address)
if err != nil {
log.Err(err).Msg(fmt.Sprintf("failed to connect %s", address))
return nil, err
}
driver := &stubAndroidDriver{
socket: conn,
timeout: timeout,
}
if driver.urlPrefix, err = url.Parse(urlPrefix); err != nil {
return nil, err
}
driver.NewSession(nil)
return driver, nil
}
func (sad *stubAndroidDriver) httpGET(pathElem ...string) (rawResp rawResponse, err error) {
var localPort int
{
tmpURL, _ := url.Parse(sad.urlPrefix.String())
hostname := tmpURL.Hostname()
if strings.HasPrefix(hostname, forwardToPrefix) {
localPort, _ = strconv.Atoi(strings.TrimPrefix(hostname, forwardToPrefix))
}
}
conn, err := net.Dial("tcp", fmt.Sprintf(":%d", localPort))
if err != nil {
return nil, fmt.Errorf("adb forward: %w", err)
}
sad.client = convertToHTTPClient(conn)
return sad.Request(http.MethodGet, sad.concatURL(nil, pathElem...), nil)
}
func (sad *stubAndroidDriver) httpPOST(data interface{}, pathElem ...string) (rawResp rawResponse, err error) {
var localPort int
{
tmpURL, _ := url.Parse(sad.urlPrefix.String())
hostname := tmpURL.Hostname()
if strings.HasPrefix(hostname, forwardToPrefix) {
localPort, _ = strconv.Atoi(strings.TrimPrefix(hostname, forwardToPrefix))
}
}
conn, err := net.Dial("tcp", fmt.Sprintf(":%d", localPort))
if err != nil {
return nil, fmt.Errorf("adb forward: %w", err)
}
sad.client = convertToHTTPClient(conn)
var bsJSON []byte = nil
if data != nil {
if bsJSON, err = json.Marshal(data); err != nil {
return nil, err
}
}
return sad.Request(http.MethodPost, sad.concatURL(nil, pathElem...), bsJSON)
}
func (sad *stubAndroidDriver) NewSession(capabilities Capabilities) (SessionInfo, error) {
sad.Driver.session.Reset()
return SessionInfo{}, errDriverNotImplemented
}
func (sad *stubAndroidDriver) sendCommand(packageName string, cmdType string, params map[string]interface{}, readTimeout ...time.Duration) (interface{}, error) {
sad.seq++
packet := map[string]interface{}{
"Seq": sad.seq,
"Cmd": cmdType,
"v": "",
}
for key, value := range params {
if key == "Cmd" || key == "Seq" {
return "", errors.New("params cannot be Cmd or Seq")
}
packet[key] = value
}
data, err := json.Marshal(packet)
if err != nil {
return nil, err
}
res, err := sad.adbClient.RunStubCommand(append(data, '\n'), packageName)
if err != nil {
return nil, err
}
var resultMap map[string]interface{}
if err := json.Unmarshal([]byte(res), &resultMap); err != nil {
return nil, err
}
if resultMap["Error"] != nil {
return nil, fmt.Errorf("failed to call stub command: %s", resultMap["Error"].(string))
}
return resultMap["Result"], nil
}
func (sad *stubAndroidDriver) DeleteSession() error {
return sad.close()
}
func (sad *stubAndroidDriver) close() error {
if sad.socket != nil {
return sad.socket.Close()
}
return nil
}
func (sad *stubAndroidDriver) Status() (DeviceStatus, error) {
app, err := sad.GetForegroundApp()
if err != nil {
return DeviceStatus{}, err
}
res, err := sad.sendCommand(app.PackageName, "Hello", nil)
if err != nil {
return DeviceStatus{}, err
}
log.Info().Msg(fmt.Sprintf("ping stub result :%v", res))
return DeviceStatus{}, nil
}
func (sad *stubAndroidDriver) Source(srcOpt ...SourceOption) (source string, err error) {
app, err := sad.GetForegroundApp()
if err != nil {
return "", err
}
params := map[string]interface{}{
"ClassName": "com.bytedance.byteinsight.MockOperator",
"Method": "getLayout",
"RetType": "",
"Args": []string{},
}
res, err := sad.sendCommand(app.PackageName, "CallStaticMethod", params)
if err != nil {
return "", err
}
return res.(string), nil
}
func (sad *stubAndroidDriver) LoginNoneUI(packageName, phoneNumber string, captcha, password string) (info AppLoginInfo, err error) {
params := map[string]interface{}{
"phone": phoneNumber,
}
if captcha != "" {
params["captcha"] = captcha
} else if password != "" {
params["password"] = password
} else {
return info, fmt.Errorf("password and capcha is empty")
}
resp, err := sad.httpPOST(params, "/host", "/login", "account")
if err != nil {
return info, err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return info, err
}
log.Info().Msgf("%v", res)
if res["isSuccess"] != true {
err = fmt.Errorf("falied to login %s", res["data"])
log.Err(err).Msgf("%v", res)
return info, err
}
time.Sleep(20 * time.Second)
info, err = sad.getLoginAppInfo(packageName)
if err != nil || !info.IsLogin {
return info, fmt.Errorf("falied to login %v", info)
}
return info, nil
}
func (sad *stubAndroidDriver) LogoutNoneUI(packageName string) error {
resp, err := sad.httpGET("/host", "/logout")
if err != nil {
return err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return err
}
log.Info().Msgf("%v", res)
if res["isSuccess"] != true {
err = fmt.Errorf("falied to logout %s", res["data"])
log.Err(err).Msgf("%v", res)
return err
}
fmt.Printf("%v", resp)
if err != nil {
return err
}
time.Sleep(3 * time.Second)
return nil
}
func (sad *stubAndroidDriver) LoginNoneUIDynamic(packageName, phoneNumber string, captcha string) error {
params := map[string]interface{}{
"ClassName": "qe.python.test.LoginUtil",
"Method": "loginSync",
"RetType": "",
"Args": []string{phoneNumber, captcha},
}
res, err := sad.sendCommand(packageName, "CallStaticMethod", params)
if err != nil {
return err
}
log.Info().Msg(res.(string))
return nil
}
func (sad *stubAndroidDriver) SetHDTStatus(status bool) error {
_, err := sad.adbClient.RunShellCommand("settings", "put", "global", "feedbacker_sso_bypass_token", "default_sso_bypass_token")
if err != nil {
log.Warn().Msg(fmt.Sprintf("failed to disable sso, error: %v", err))
}
params := map[string]interface{}{
"ClassName": "com.bytedance.ies.stark.framework.HybridDevTool",
"Method": "setEnabled",
"RetType": "",
"Args": []bool{status},
}
res, err := sad.sendCommand("com.ss.android.ugc.aweme", "CallStaticMethod", params)
if err != nil {
return fmt.Errorf("failed to set hds status %v, error: %v", status, err)
}
log.Info().Msg(fmt.Sprintf("set hdt status result: %s", res))
return nil
}
func (sad *stubAndroidDriver) getLoginAppInfo(packageName string) (info AppLoginInfo, err error) {
resp, err := sad.httpGET("/host", "/app", "/info")
if err != nil {
return info, err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return info, err
}
if res["isSuccess"] != true {
err = fmt.Errorf("falied to get app info %s", res["data"])
log.Err(err).Msgf("%v", res)
return info, err
}
err = json.Unmarshal([]byte(res["data"].(string)), &info)
if err != nil {
err = fmt.Errorf("falied to parse app info %s", res["data"])
return
}
return info, nil
}

141
pkg/uixt/android_stub_driver_test.go Normal file
View File

@@ -0,0 +1,141 @@
package uixt
import (
"fmt"
"os"
"testing"
)
var androidStubDriver *stubAndroidDriver
func setupStubDriver(t *testing.T) {
device, err := NewAndroidDevice()
checkErr(t, err)
device.STUB = true
androidStubDriver, err = device.NewStubDriver(Capabilities{})
checkErr(t, err)
}
func TestHello(t *testing.T) {
setupStubDriver(t)
status, err := androidStubDriver.Status()
if err != nil {
t.Fatal(err)
}
t.Log(status)
}
func TestSource(t *testing.T) {
setupStubDriver(t)
source, err := androidStubDriver.Source()
if err != nil {
t.Fatal(err)
}
t.Log(source)
}
func TestLogin(t *testing.T) {
setupStubDriver(t)
info, err := androidStubDriver.LoginNoneUI("com.ss.android.ugc.aweme", "12342316231", "8517", "")
if err != nil {
t.Fatal(err)
}
t.Log(info)
}
func TestLogout(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.LogoutNoneUI("com.ss.android.ugc.aweme")
if err != nil {
t.Fatal(err)
}
}
func TestSwipe(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.Swipe(878, 2375, 672, 2375)
if err != nil {
t.Fatal(err)
}
}
func TestTap(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.Tap(900, 400)
if err != nil {
t.Fatal(err)
}
}
func TestDoubleTap(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.DoubleTap(500, 500)
if err != nil {
t.Fatal(err)
}
}
func TestLongPress(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.Swipe(1036, 1076, 1036, 1076, WithDuration(3))
if err != nil {
t.Fatal(err)
}
}
func TestInput(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.Input("\"哈哈\"")
if err != nil {
t.Fatal(err)
}
}
func TestSave(t *testing.T) {
setupStubDriver(t)
raw, err := androidStubDriver.Screenshot()
if err != nil {
t.Fatal(err)
}
source, err := androidStubDriver.Source()
if err != nil {
t.Fatal(err)
}
step := 14
file, err := os.Create(fmt.Sprintf("/Users/bytedance/workcode/wings_algorithm/testcases/data/cases/0/%d.jpg", step))
if err != nil {
t.Fatal(err)
}
file.Write(raw.Bytes())
file, err = os.Create(fmt.Sprintf("/Users/bytedance/workcode/wings_algorithm/testcases/data/cases/0/%d.json", step))
if err != nil {
t.Fatal(err)
}
file.Write([]byte(source))
}
func TestAppLaunch(t *testing.T) {
setupStubDriver(t)
err := androidStubDriver.AppLaunch("com.ss.android.ugc.aweme")
if err != nil {
t.Fatal(err)
}
}
func TestAppTerminal(t *testing.T) {
setupStubDriver(t)
_, err := androidStubDriver.AppTerminate("com.ss.android.ugc.aweme")
if err != nil {
t.Fatal(err)
}
}
func TestAppInfo(t *testing.T) {
setupStubDriver(t)
info, err := androidStubDriver.getLoginAppInfo("com.ss.android.ugc.aweme")
if err != nil {
t.Fatal(err)
}
t.Log(info)
}

522
pkg/uixt/android_test.go Normal file
View File

@@ -0,0 +1,522 @@
//go:build localtest
package uixt
import (
"os"
"strings"
"testing"
"time"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
var (
uiaServerURL = "http://forward-to-6790:6790/wd/hub"
driverExt *DriverExt
)
func setupAndroidAdbDriver(t *testing.T) {
device, err := NewAndroidDevice()
checkErr(t, err)
device.UIA2 = false
device.LogOn = false
driverExt, err = device.NewDriver()
checkErr(t, err)
}
func setupAndroidUIA2Driver(t *testing.T) {
device, err := NewAndroidDevice()
checkErr(t, err)
device.UIA2 = true
device.LogOn = false
driverExt, err = device.NewDriver()
checkErr(t, err)
}
func TestAndroidDevice_GetPackageInfo(t *testing.T) {
device, err := NewAndroidDevice()
checkErr(t, err)
appInfo, err := device.GetPackageInfo("com.android.settings")
checkErr(t, err)
t.Log(appInfo)
}
func TestAndroidDevice_GetCurrentWindow(t *testing.T) {
device, err := NewAndroidDevice()
checkErr(t, err)
windowInfo, err := device.GetCurrentWindow()
checkErr(t, err)
t.Logf("packageName: %s\tactivityName: %s", windowInfo.PackageName, windowInfo.Activity)
}
func TestDriver_NewSession(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
firstMatchEntry := make(map[string]interface{})
firstMatchEntry["package"] = "com.android.settings"
firstMatchEntry["activity"] = "com.android.settings/.Settings"
caps := Capabilities{
"firstMatch": []interface{}{firstMatchEntry},
"alwaysMatch": struct{}{},
}
session, err := driver.NewSession(caps)
if err != nil {
t.Fatal(err)
}
if len(session.SessionId) == 0 {
t.Fatal("should not be empty")
}
}
func TestNewDriver(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
t.Log(driver.session.ID)
}
func TestDriver_Quit(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
if err = driver.DeleteSession(); err != nil {
t.Fatal(err)
}
}
func TestDriver_Status(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
_, err = driver.Status()
if err != nil {
t.Fatal(err)
}
}
func TestDriver_Screenshot(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
screenshot, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
t.Log(os.WriteFile("/Users/hero/Desktop/s1.png", screenshot.Bytes(), 0o600))
}
func TestDriver_Rotation(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
rotation, err := driver.Rotation()
if err != nil {
t.Fatal(err)
}
t.Logf("x = %d\ty = %d\tz = %d", rotation.X, rotation.Y, rotation.Z)
}
func TestDriver_DeviceSize(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
deviceSize, err := driver.WindowSize()
if err != nil {
t.Fatal(err)
}
t.Logf("width = %d\theight = %d", deviceSize.Width, deviceSize.Height)
}
func TestDriver_Source(t *testing.T) {
setupAndroidUIA2Driver(t)
source, err := driverExt.Driver.Source()
if err != nil {
t.Fatal(err)
}
t.Log(source)
}
func TestDriver_TapByText(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
err = driver.TapByText("安装")
if err != nil {
t.Fatal(err)
}
}
func TestDriver_BatteryInfo(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
batteryInfo, err := driver.BatteryInfo()
if err != nil {
t.Fatal(err)
}
t.Log(batteryInfo)
}
func TestDriver_GetAppiumSettings(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
appiumSettings, err := driver.GetAppiumSettings()
if err != nil {
t.Fatal(err)
}
for k := range appiumSettings {
t.Logf("key: %s\tvalue: %v", k, appiumSettings[k])
}
// t.Log(appiumSettings)
}
func TestDriver_DeviceInfo(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
devInfo, err := driver.DeviceInfo()
if err != nil {
t.Fatal(err)
}
t.Logf("api version: %s", devInfo.APIVersion)
t.Logf("platform version: %s", devInfo.PlatformVersion)
t.Logf("bluetooth state: %s", devInfo.Bluetooth.State)
}
func TestDriver_Tap(t *testing.T) {
setupAndroidUIA2Driver(t)
driverExt.Driver.StartCaptureLog("")
err := driverExt.TapXY(0.5, 0.5, WithIdentifier("test"), WithPressDuration(4))
if err != nil {
t.Fatal(err)
}
//time.Sleep(time.Second)
//
//err = driverExt.Driver.Tap(60.5, 125.5, WithIdentifier("test"))
//if err != nil {
// t.Fatal(err)
//}
//time.Sleep(time.Second)
//result, _ := driverExt.Driver.StopCaptureLog()
//t.Log(result)
}
func TestDriver_Swipe(t *testing.T) {
setupAndroidUIA2Driver(t)
err := driverExt.Driver.Swipe(400, 1000, 400, 500, WithPressDuration(0.5))
if err != nil {
t.Fatal(err)
}
}
func TestDriver_Swipe_Relative(t *testing.T) {
setupAndroidUIA2Driver(t)
err := driverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5)
if err != nil {
t.Fatal(err)
}
}
func TestDriver_Drag(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
err = driver.Drag(400, 260, 400, 500)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 200)
err = driver.Drag(400, 501.5, 400, 261.5)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Millisecond * 200)
}
func TestDriver_SendKeys(t *testing.T) {
setupAndroidUIA2Driver(t)
err := driverExt.Driver.SendKeys("辽宁省沈阳市新民市民族街36-4", WithIdentifier("test"))
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second * 2)
//err = driver.SendKeys("def")
//if err != nil {
// t.Fatal(err)
//}
//time.Sleep(time.Second * 2)
//err = driver.SendKeys("\\n")
// err = driver.SendKeys(`\n`, false)
//if err != nil {
// t.Fatal(err)
//}
}
func TestDriver_PressBack(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
err = driver.PressBack()
if err != nil {
t.Fatal(err)
}
}
func TestDriver_SetRotation(t *testing.T) {
driver, err := NewUIADriver(nil, uiaServerURL)
if err != nil {
t.Fatal(err)
}
// err = driver.SetRotation(Rotation{Z: 0})
err = driver.SetRotation(Rotation{Z: 270})
if err != nil {
t.Fatal(err)
}
}
func TestDriver_GetOrientation(t *testing.T) {
setupAndroidUIA2Driver(t)
_, _ = driverExt.Driver.AppTerminate("com.quark.browser")
_ = driverExt.Driver.AppLaunch("com.quark.browser")
time.Sleep(2 * time.Second)
_ = driverExt.Driver.Homescreen()
}
func TestUiSelectorHelper_NewUiSelectorHelper(t *testing.T) {
uiSelector := NewUiSelectorHelper().Text("a").String()
if uiSelector != `new UiSelector().text("a");` {
t.Fatal("[ERROR]", uiSelector)
}
uiSelector = NewUiSelectorHelper().Text("a").TextStartsWith("b").String()
if uiSelector != `new UiSelector().text("a").textStartsWith("b");` {
t.Fatal("[ERROR]", uiSelector)
}
uiSelector = NewUiSelectorHelper().ClassName("android.widget.LinearLayout").Index(6).String()
if uiSelector != `new UiSelector().className("android.widget.LinearLayout").index(6);` {
t.Fatal("[ERROR]", uiSelector)
}
uiSelector = NewUiSelectorHelper().Focused(false).Instance(6).String()
if uiSelector != `new UiSelector().focused(false).instance(6);` {
t.Fatal("[ERROR]", uiSelector)
}
uiSelector = NewUiSelectorHelper().ChildSelector(NewUiSelectorHelper().Enabled(true)).String()
if uiSelector != `new UiSelector().childSelector(new UiSelector().enabled(true));` {
t.Fatal("[ERROR]", uiSelector)
}
}
func Test_getFreePort(t *testing.T) {
freePort, err := builtin.GetFreePort()
if err != nil {
t.Fatal(err)
}
t.Log(freePort)
}
func TestDeviceList(t *testing.T) {
devices, err := GetAndroidDevices()
if err != nil {
t.Fatal(err)
}
for i := range devices {
t.Log(devices[i].Serial())
}
}
func TestDriver_AppLaunch(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewDriver()
if err != nil {
t.Fatal(err)
}
err = driver.Driver.AppLaunch("com.android.settings")
if err != nil {
t.Fatal(err)
}
raw, err := driver.Driver.Screenshot()
if err != nil {
t.Fatal(err)
}
t.Log(os.WriteFile("s1.png", raw.Bytes(), 0o600))
}
func TestDriver_IsAppInForeground(t *testing.T) {
setupAndroidUIA2Driver(t)
// setupAndroidAdbDriver(t)
err := driverExt.Driver.AppLaunch("com.android.settings")
checkErr(t, err)
app, err := driverExt.Driver.GetForegroundApp()
checkErr(t, err)
if app.PackageName != "com.android.settings" {
t.FailNow()
}
if app.Activity != ".Settings" {
t.FailNow()
}
err = driverExt.Driver.AssertForegroundApp("com.android.settings")
if err != nil {
t.Fatal(err)
}
time.Sleep(2 * time.Second)
_, err = driverExt.Driver.AppTerminate("com.android.settings")
if err != nil {
t.Fatal(err)
}
err = driverExt.Driver.AssertForegroundApp("com.android.settings")
if err == nil {
t.Fatal(err)
}
}
func TestDriver_KeepAlive(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewDriver()
if err != nil {
t.Fatal(err)
}
err = driver.Driver.AppLaunch("com.android.settings")
if err != nil {
t.Fatal(err)
}
_, err = driver.Driver.Screenshot()
if err != nil {
t.Fatal(err)
}
time.Sleep(60 * time.Second)
_, err = driver.Driver.Screenshot()
if err != nil {
t.Fatal(err)
}
}
func TestDriver_AppTerminate(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewDriver()
if err != nil {
t.Fatal(err)
}
_, err = driver.Driver.AppTerminate("tv.danmaku.bili")
if err != nil {
t.Fatal(err)
}
}
func TestConvertPoints(t *testing.T) {
data := "10-09 20:16:48.216 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317808206,\"ext\":\"输入\",\"from\":{\"x\":0.0,\"y\":0.0},\"operation\":\"Gtf-SendKeys\",\"run_time\":627,\"start\":1665317807579,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":0.0,\"y\":0.0}}\n10-09 20:18:22.899 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317902898,\"ext\":\"进入直播间\",\"from\":{\"x\":717.0,\"y\":2117.5},\"operation\":\"Gtf-Tap\",\"run_time\":121,\"start\":1665317902777,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":717.0,\"y\":2117.5}}\n10-09 20:18:32.063 I/iesqaMonitor(17845): {\"duration\":0,\"end\":1665317912062,\"ext\":\"第一次上划\",\"from\":{\"x\":1437.0,\"y\":2409.9},\"operation\":\"Gtf-Swipe\",\"run_time\":32,\"start\":1665317912030,\"start_first\":0,\"start_last\":0,\"to\":{\"x\":1437.0,\"y\":2409.9}}"
eps := ConvertPoints(strings.Split(data, "\n"))
if len(eps) != 3 {
t.Fatal()
}
}
func TestDriver_ShellInputUnicode(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewAdbDriver()
if err != nil {
t.Fatal(err)
}
err = driver.SendKeys("test中文输入&")
if err != nil {
t.Fatal(err)
}
raw, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
t.Log(os.WriteFile("s1.png", raw.Bytes(), 0o600))
}
func TestTapTexts(t *testing.T) {
setupAndroidUIA2Driver(t)
actions := []TapTextAction{
{Text: "^.*无视风险安装$", Options: []ActionOption{WithTapOffset(100, 0), WithRegex(true), WithIgnoreNotFoundError(true)}},
{Text: "已了解此应用未经检测.*", Options: []ActionOption{WithTapOffset(-450, 0), WithRegex(true), WithIgnoreNotFoundError(true)}},
{Text: "^(.*无视风险安装|确定|继续|完成|点击继续安装|继续安装旧版本|替换|安装|授权本次安装|继续安装|重新安装)$", Options: []ActionOption{WithRegex(true), WithIgnoreNotFoundError(true)}},
}
err := driverExt.Driver.TapByTexts(actions...)
if err != nil {
t.Fatal(err)
}
}
func TestRecordVideo(t *testing.T) {
setupAndroidAdbDriver(t)
path, err := driverExt.Driver.(*adbDriver).RecordScreen("", 5*time.Second)
if err != nil {
t.Fatal(err)
}
println(path)
}
func Test_Android_Backspace(t *testing.T) {
setupAndroidAdbDriver(t)
err := driverExt.Driver.Backspace(1)
if err != nil {
t.Fatal(err)
}
}

655
pkg/uixt/android_uia2_driver.go Normal file
View File

@@ -0,0 +1,655 @@
package uixt
import (
"bytes"
"encoding/base64"
"encoding/json"
"encoding/xml"
"fmt"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/pkg/utf7"
)
var errDriverNotImplemented = errors.New("driver method not implemented")
type uiaDriver struct {
adbDriver
}
func NewUIADriver(capabilities Capabilities, urlPrefix string) (driver *uiaDriver, err error) {
log.Info().Msg("init uiautomator2 driver")
if capabilities == nil {
capabilities = NewCapabilities()
capabilities.WithWaitForIdleTimeout(0)
}
driver = new(uiaDriver)
if driver.urlPrefix, err = url.Parse(urlPrefix); err != nil {
return nil, err
}
var localPort int
{
tmpURL, _ := url.Parse(driver.urlPrefix.String())
hostname := tmpURL.Hostname()
if strings.HasPrefix(hostname, forwardToPrefix) {
localPort, _ = strconv.Atoi(strings.TrimPrefix(hostname, forwardToPrefix))
}
}
conn, err := net.Dial("tcp", fmt.Sprintf(":%d", localPort))
if err != nil {
return nil, fmt.Errorf("adb forward: %w", err)
}
driver.client = convertToHTTPClient(conn)
_, err = driver.NewSession(capabilities)
if err != nil {
return nil, errors.Wrap(err, "create UIAutomator session failed")
}
return driver, nil
}
type BatteryStatus int
const (
_ = iota
BatteryStatusUnknown BatteryStatus = iota
BatteryStatusCharging
BatteryStatusDischarging
BatteryStatusNotCharging
BatteryStatusFull
)
func (bs BatteryStatus) String() string {
switch bs {
case BatteryStatusUnknown:
return "unknown"
case BatteryStatusCharging:
return "charging"
case BatteryStatusDischarging:
return "discharging"
case BatteryStatusNotCharging:
return "not charging"
case BatteryStatusFull:
return "full"
default:
return fmt.Sprintf("unknown status code (%d)", bs)
}
}
func (ud *uiaDriver) resetDriver() error {
newUIADriver, err := NewUIADriver(NewCapabilities(), ud.urlPrefix.String())
if err != nil {
return err
}
ud.client = newUIADriver.client
ud.session.ID = newUIADriver.session.ID
return nil
}
func (ud *uiaDriver) httpRequest(method string, rawURL string, rawBody []byte) (rawResp rawResponse, err error) {
for retryCount := 1; retryCount <= 5; retryCount++ {
rawResp, err = ud.Driver.Request(method, rawURL, rawBody)
if err == nil {
return
}
// wait for UIA2 server to resume automatically
time.Sleep(3 * time.Second)
oldSessionID := ud.session.ID
if err2 := ud.resetDriver(); err2 != nil {
log.Err(err2).Msgf("failed to reset uia2 driver, retry count: %v", retryCount)
continue
}
log.Debug().Str("new session", ud.session.ID).Str("old session", oldSessionID).Msgf("successful to reset uia2 driver, retry count: %v", retryCount)
if oldSessionID != "" {
rawURL = strings.Replace(rawURL, oldSessionID, ud.session.ID, 1)
}
}
return
}
func (ud *uiaDriver) httpGET(pathElem ...string) (rawResp rawResponse, err error) {
return ud.httpRequest(http.MethodGet, ud.concatURL(nil, pathElem...), nil)
}
func (ud *uiaDriver) httpPOST(data interface{}, pathElem ...string) (rawResp rawResponse, err error) {
var bsJSON []byte = nil
if data != nil {
if bsJSON, err = json.Marshal(data); err != nil {
return nil, err
}
}
return ud.httpRequest(http.MethodPost, ud.concatURL(nil, pathElem...), bsJSON)
}
func (ud *uiaDriver) httpDELETE(pathElem ...string) (rawResp rawResponse, err error) {
return ud.httpRequest(http.MethodDelete, ud.concatURL(nil, pathElem...), nil)
}
func (ud *uiaDriver) NewSession(capabilities Capabilities) (sessionInfo SessionInfo, err error) {
// register(postHandler, new NewSession("/wd/hub/session"))
var rawResp rawResponse
data := make(map[string]interface{})
if len(capabilities) == 0 {
data["capabilities"] = make(map[string]interface{})
} else {
data["capabilities"] = map[string]interface{}{"alwaysMatch": capabilities}
}
if rawResp, err = ud.Driver.POST(data, "/session"); err != nil {
return SessionInfo{SessionId: ""}, err
}
reply := new(struct{ Value struct{ SessionId string } })
if err = json.Unmarshal(rawResp, reply); err != nil {
return SessionInfo{SessionId: ""}, err
}
sessionID := reply.Value.SessionId
ud.Driver.session.Reset()
ud.Driver.session.ID = sessionID
// d.sessionIdCache[sessionID] = true
return SessionInfo{SessionId: sessionID}, nil
}
func (ud *uiaDriver) DeleteSession() (err error) {
if ud.session.ID == "" {
return nil
}
if _, err = ud.httpDELETE("/session", ud.session.ID); err == nil {
ud.session.ID = ""
}
return err
}
func (ud *uiaDriver) Status() (deviceStatus DeviceStatus, err error) {
// register(getHandler, new Status("/wd/hub/status"))
var rawResp rawResponse
// Notice: use Driver.GET instead of httpGET to avoid loop calling
if rawResp, err = ud.Driver.GET("/status"); err != nil {
return DeviceStatus{Ready: false}, err
}
reply := new(struct {
Value struct {
// Message string
Ready bool
}
})
if err = json.Unmarshal(rawResp, reply); err != nil {
return DeviceStatus{Ready: false}, err
}
return DeviceStatus{Ready: true}, nil
}
func (ud *uiaDriver) DeviceInfo() (deviceInfo DeviceInfo, err error) {
// register(getHandler, new GetDeviceInfo("/wd/hub/session/:sessionId/appium/device/info"))
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "appium/device/info"); err != nil {
return DeviceInfo{}, err
}
reply := new(struct{ Value struct{ DeviceInfo } })
if err = json.Unmarshal(rawResp, reply); err != nil {
return DeviceInfo{}, err
}
deviceInfo = reply.Value.DeviceInfo
return
}
func (ud *uiaDriver) BatteryInfo() (batteryInfo BatteryInfo, err error) {
// register(getHandler, new GetBatteryInfo("/wd/hub/session/:sessionId/appium/device/battery_info"))
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "appium/device/battery_info"); err != nil {
return BatteryInfo{}, err
}
reply := new(struct{ Value struct{ BatteryInfo } })
if err = json.Unmarshal(rawResp, reply); err != nil {
return BatteryInfo{}, err
}
if reply.Value.Level == -1 || reply.Value.Status == -1 {
return reply.Value.BatteryInfo, errors.New("cannot be retrieved from the system")
}
batteryInfo = reply.Value.BatteryInfo
return
}
func (ud *uiaDriver) WindowSize() (size Size, err error) {
// register(getHandler, new GetDeviceSize("/wd/hub/session/:sessionId/window/:windowHandle/size"))
if !ud.windowSize.IsNil() {
// use cached window size
return ud.windowSize, nil
}
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "window/:windowHandle/size"); err != nil {
return Size{}, errors.Wrap(err, "get window size failed by UIA2 request")
}
reply := new(struct{ Value struct{ Size } })
if err = json.Unmarshal(rawResp, reply); err != nil {
return Size{}, errors.Wrap(err, "get window size failed by UIA2 response")
}
size = reply.Value.Size
// check orientation
orientation, err := ud.Orientation()
if err != nil {
log.Warn().Err(err).Msgf("window size get orientation failed, use default orientation")
orientation = OrientationPortrait
}
if orientation != OrientationPortrait {
size.Width, size.Height = size.Height, size.Width
}
ud.windowSize = size // cache window size
return size, nil
}
// PressBack simulates a short press on the BACK button.
func (ud *uiaDriver) PressBack(options ...ActionOption) (err error) {
// register(postHandler, new PressBack("/wd/hub/session/:sessionId/back"))
_, err = ud.httpPOST(nil, "/session", ud.session.ID, "back")
return
}
func (ud *uiaDriver) Homescreen() (err error) {
return ud.PressKeyCodes(KCHome, KMEmpty)
}
func (ud *uiaDriver) PressKeyCode(keyCode KeyCode) (err error) {
return ud.PressKeyCodes(keyCode, KMEmpty)
}
func (ud *uiaDriver) PressKeyCodes(keyCode KeyCode, metaState KeyMeta, flags ...KeyFlag) (err error) {
// register(postHandler, new PressKeyCodeAsync("/wd/hub/session/:sessionId/appium/device/press_keycode"))
data := map[string]interface{}{
"keycode": keyCode,
}
if metaState != KMEmpty {
data["metastate"] = metaState
}
if len(flags) != 0 {
data["flags"] = flags[0]
}
_, err = ud.httpPOST(data, "/session", ud.session.ID, "appium/device/press_keycode")
return
}
func (ud *uiaDriver) Orientation() (orientation Orientation, err error) {
// [[FBRoute GET:@"/orientation"] respondWithTarget:self action:@selector(handleGetOrientation:)]
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "/orientation"); err != nil {
return "", err
}
reply := new(struct{ Value Orientation })
if err = json.Unmarshal(rawResp, reply); err != nil {
return "", err
}
orientation = reply.Value
return
}
func (ud *uiaDriver) DoubleTap(x, y float64, options ...ActionOption) error {
return ud.DoubleFloatTap(x, y)
}
func (ud *uiaDriver) DoubleFloatTap(x, y float64) error {
data := map[string]interface{}{
"actions": []interface{}{
map[string]interface{}{
"type": "pointer",
"parameters": map[string]string{"pointerType": "touch"},
"id": "touch",
"actions": []interface{}{
map[string]interface{}{"type": "pointerMove", "duration": 0, "x": x, "y": y, "origin": "viewport"},
map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
},
},
},
}
_, err := ud.httpPOST(data, "/session", ud.session.ID, "actions/tap")
return err
}
func (ud *uiaDriver) Tap(x, y float64, options ...ActionOption) (err error) {
// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
actionOptions := NewActionOptions(options...)
if len(actionOptions.Offset) == 2 {
x += float64(actionOptions.Offset[0])
y += float64(actionOptions.Offset[1])
}
x += actionOptions.getRandomOffset()
y += actionOptions.getRandomOffset()
duration := 100.0
if actionOptions.PressDuration > 0 {
duration = actionOptions.PressDuration * 1000
}
data := map[string]interface{}{
"actions": []interface{}{
map[string]interface{}{
"type": "pointer",
"parameters": map[string]string{"pointerType": "touch"},
"id": "touch",
"actions": []interface{}{
map[string]interface{}{"type": "pointerMove", "duration": 0, "x": x, "y": y, "origin": "viewport"},
map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
map[string]interface{}{"type": "pause", "duration": duration},
map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
},
},
},
}
// update data options in post data for extra uiautomator configurations
actionOptions.updateData(data)
_, err = ud.httpPOST(data, "/session", ud.session.ID, "actions/tap")
return err
}
func (ud *uiaDriver) TouchAndHold(x, y float64, options ...ActionOption) (err error) {
opts := NewActionOptions(options...)
duration := opts.Duration
if duration == 0 {
duration = 1.0
}
// register(postHandler, new TouchLongClick("/wd/hub/session/:sessionId/touch/longclick"))
data := map[string]interface{}{
"params": map[string]interface{}{
"x": x,
"y": y,
"duration": int(duration * 1000),
},
}
_, err = ud.httpPOST(data, "/session", ud.session.ID, "touch/longclick")
return
}
// Drag performs a swipe from one coordinate to another coordinate. You can control
// the smoothness and speed of the swipe by specifying the number of steps.
// Each step execution is throttled to 5 milliseconds per step, so for a 100
// steps, the swipe will take around 0.5 seconds to complete.
func (ud *uiaDriver) Drag(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) {
actionOptions := NewActionOptions(options...)
if len(actionOptions.Offset) == 4 {
fromX += float64(actionOptions.Offset[0])
fromY += float64(actionOptions.Offset[1])
toX += float64(actionOptions.Offset[2])
toY += float64(actionOptions.Offset[3])
}
fromX += actionOptions.getRandomOffset()
fromY += actionOptions.getRandomOffset()
toX += actionOptions.getRandomOffset()
toY += actionOptions.getRandomOffset()
data := map[string]interface{}{
"startX": fromX,
"startY": fromY,
"endX": toX,
"endY": toY,
}
// update data options in post data for extra uiautomator configurations
actionOptions.updateData(data)
// register(postHandler, new Drag("/wd/hub/session/:sessionId/touch/drag"))
_, err = ud.httpPOST(data, "/session", ud.session.ID, "touch/drag")
return
}
// Swipe performs a swipe from one coordinate to another using the number of steps
// to determine smoothness and speed. Each step execution is throttled to 5ms
// per step. So for a 100 steps, the swipe will take about 1/2 second to complete.
//
// `steps` is the number of move steps sent to the system
func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY float64, options ...ActionOption) error {
// register(postHandler, new Swipe("/wd/hub/session/:sessionId/touch/perform"))
actionOptions := NewActionOptions(options...)
if len(actionOptions.Offset) == 4 {
fromX += float64(actionOptions.Offset[0])
fromY += float64(actionOptions.Offset[1])
toX += float64(actionOptions.Offset[2])
toY += float64(actionOptions.Offset[3])
}
fromX += actionOptions.getRandomOffset()
fromY += actionOptions.getRandomOffset()
toX += actionOptions.getRandomOffset()
toY += actionOptions.getRandomOffset()
duration := 200.0
if actionOptions.PressDuration > 0 {
duration = actionOptions.PressDuration * 1000
}
data := map[string]interface{}{
"actions": []interface{}{
map[string]interface{}{
"type": "pointer",
"parameters": map[string]string{"pointerType": "touch"},
"id": "touch",
"actions": []interface{}{
map[string]interface{}{"type": "pointerMove", "duration": 0, "x": fromX, "y": fromY, "origin": "viewport"},
map[string]interface{}{"type": "pointerDown", "duration": 0, "button": 0},
map[string]interface{}{"type": "pointerMove", "duration": duration, "x": toX, "y": toY, "origin": "viewport"},
map[string]interface{}{"type": "pointerUp", "duration": 0, "button": 0},
},
},
},
}
// update data options in post data for extra uiautomator configurations
actionOptions.updateData(data)
_, err := ud.httpPOST(data, "/session", ud.session.ID, "actions/swipe")
return err
}
func (ud *uiaDriver) SetPasteboard(contentType PasteboardType, content string) (err error) {
lbl := content
const defaultLabelLen = 10
if len(lbl) > defaultLabelLen {
lbl = lbl[:defaultLabelLen]
}
data := map[string]interface{}{
"contentType": contentType,
"label": lbl,
"content": base64.StdEncoding.EncodeToString([]byte(content)),
}
// register(postHandler, new SetClipboard("/wd/hub/session/:sessionId/appium/device/set_clipboard"))
_, err = ud.httpPOST(data, "/session", ud.session.ID, "appium/device/set_clipboard")
return
}
func (ud *uiaDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error) {
if len(contentType) == 0 {
contentType = PasteboardTypePlaintext
}
// register(postHandler, new GetClipboard("/wd/hub/session/:sessionId/appium/device/get_clipboard"))
data := map[string]interface{}{
"contentType": contentType[0],
}
var rawResp rawResponse
if rawResp, err = ud.httpPOST(data, "/session", ud.session.ID, "appium/device/get_clipboard"); err != nil {
return
}
reply := new(struct{ Value string })
if err = json.Unmarshal(rawResp, reply); err != nil {
return
}
if data, err := base64.StdEncoding.DecodeString(reply.Value); err != nil {
raw.Write([]byte(reply.Value))
} else {
raw.Write(data)
}
return
}
// SendKeys Android input does not support setting frequency.
func (ud *uiaDriver) SendKeys(text string, options ...ActionOption) (err error) {
// register(postHandler, new SendKeysToElement("/wd/hub/session/:sessionId/keys"))
// https://github.com/appium/appium-uiautomator2-server/blob/master/app/src/main/java/io/appium/uiautomator2/handler/SendKeysToElement.java#L76-L85
actionOptions := NewActionOptions(options...)
err = ud.SendUnicodeKeys(text, options...)
if err != nil {
data := map[string]interface{}{
"text": text,
}
// new data options in post data for extra uiautomator configurations
actionOptions.updateData(data)
_, err = ud.httpPOST(data, "/session", ud.session.ID, "/keys")
}
return
}
func (ud *uiaDriver) SendUnicodeKeys(text string, options ...ActionOption) (err error) {
// If the Unicode IME is not installed, fall back to the old interface.
// There might be differences in the tracking schemes across different phones, and it is pending further verification.
// In release version: without the Unicode IME installed, the test cannot execute.
if !ud.IsUnicodeIMEInstalled() {
return fmt.Errorf("appium unicode ime not installed")
}
currentIme, err := ud.adbDriver.GetIme()
if err != nil {
return
}
if currentIme != UnicodeImePackageName {
defer func() {
_ = ud.adbDriver.SetIme(currentIme)
}()
err = ud.adbDriver.SetIme(UnicodeImePackageName)
if err != nil {
log.Warn().Err(err).Msgf("set Unicode Ime failed")
return
}
}
encodedStr, err := utf7.Encoding.NewEncoder().String(text)
if err != nil {
log.Warn().Err(err).Msgf("encode text with modified utf7 failed")
return
}
err = ud.SendActionKey(encodedStr, options...)
return
}
func (ud *uiaDriver) SendActionKey(text string, options ...ActionOption) (err error) {
actionOptions := NewActionOptions(options...)
var actions []interface{}
for i, c := range text {
actions = append(actions, map[string]interface{}{"type": "keyDown", "value": string(c)},
map[string]interface{}{"type": "keyUp", "value": string(c)})
if i != len(text)-1 {
actions = append(actions, map[string]interface{}{"type": "pause", "duration": 40})
}
}
data := map[string]interface{}{
"actions": []interface{}{
map[string]interface{}{
"type": "key",
"id": "key",
"actions": actions,
},
},
}
// new data options in post data for extra uiautomator configurations
actionOptions.updateData(data)
_, err = ud.httpPOST(data, "/session", ud.session.ID, "/actions/keys")
return
}
func (ud *uiaDriver) Input(text string, options ...ActionOption) (err error) {
return ud.SendKeys(text, options...)
}
func (ud *uiaDriver) Rotation() (rotation Rotation, err error) {
// register(getHandler, new GetRotation("/wd/hub/session/:sessionId/rotation"))
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "rotation"); err != nil {
return Rotation{}, err
}
reply := new(struct{ Value Rotation })
if err = json.Unmarshal(rawResp, reply); err != nil {
return Rotation{}, err
}
rotation = reply.Value
return
}
func (ud *uiaDriver) Screenshot() (raw *bytes.Buffer, err error) {
// https://bytedance.larkoffice.com/docx/C8qEdmSHnoRvMaxZauocMiYpnLh
// ui2截图受内存影响，改为adb截图
return ud.adbDriver.Screenshot()
}
func (ud *uiaDriver) Source(srcOpt ...SourceOption) (source string, err error) {
// register(getHandler, new Source("/wd/hub/session/:sessionId/source"))
var rawResp rawResponse
if rawResp, err = ud.httpGET("/session", ud.session.ID, "source"); err != nil {
return "", err
}
reply := new(struct{ Value string })
if err = json.Unmarshal(rawResp, reply); err != nil {
return "", err
}
source = reply.Value
return
}
func (ud *uiaDriver) sourceTree(srcOpt ...SourceOption) (sourceTree *Hierarchy, err error) {
source, err := ud.Source()
if err != nil {
return
}
sourceTree = new(Hierarchy)
err = xml.Unmarshal([]byte(source), sourceTree)
if err != nil {
return
}
return
}
func (ud *uiaDriver) TapByText(text string, options ...ActionOption) error {
sourceTree, err := ud.sourceTree()
if err != nil {
return err
}
return ud.tapByTextUsingHierarchy(sourceTree, text, options...)
}
func (ud *uiaDriver) TapByTexts(actions ...TapTextAction) error {
sourceTree, err := ud.sourceTree()
if err != nil {
return err
}
for _, action := range actions {
err := ud.tapByTextUsingHierarchy(sourceTree, action.Text, action.Options...)
if err != nil {
return err
}
}
return nil
}
func (ud *uiaDriver) GetDriverResults() []*DriverResult {
defer func() {
ud.Driver.driverResults = nil
}()
return ud.Driver.driverResults
}

204
pkg/uixt/client.go Normal file
View File

@@ -0,0 +1,204 @@
package uixt
import (
"bytes"
"context"
"encoding/json"
"io"
"net"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type DriverSession struct {
ID string
// cache uia2/wda request and response
requests []*DriverResult
// cache screenshot ocr results
screenResults []*ScreenResult // list of actions
// cache e2e delay
e2eDelay []timeLog
}
func (d *DriverSession) addScreenResult(screenResult *ScreenResult) {
d.screenResults = append(d.screenResults, screenResult)
}
func (d *DriverSession) addRequestResult(driverResult *DriverResult) {
d.requests = append(d.requests, driverResult)
}
func (d *DriverSession) Reset() {
d.screenResults = make([]*ScreenResult, 0)
d.requests = make([]*DriverResult, 0)
d.e2eDelay = nil
}
type Attachments map[string]interface{}
func (d *DriverSession) Get(withReset bool) Attachments {
data := Attachments{
"screen_results": d.screenResults,
}
if len(d.requests) != 0 {
data["requests"] = d.requests
}
if d.e2eDelay != nil {
data["e2e_results"] = d.e2eDelay
}
if withReset {
d.Reset()
}
return data
}
type Driver struct {
urlPrefix *url.URL
client *http.Client
// cache to avoid repeated query
scale float64
windowSize Size
driverResults []*DriverResult
// cache session data
session DriverSession
}
type DriverResult struct {
RequestMethod string `json:"request_method"`
RequestUrl string `json:"request_url"`
RequestBody string `json:"request_body,omitempty"`
RequestTime time.Time `json:"request_time"`
Success bool `json:"success"`
ResponseStatus int `json:"response_status"`
ResponseDuration int64 `json:"response_duration(ms)"` // ms
ResponseBody string `json:"response_body"`
Error string `json:"error,omitempty"`
}
func (wd *Driver) concatURL(u *url.URL, elem ...string) string {
var tmp *url.URL
if u == nil {
u = wd.urlPrefix
}
tmp, _ = url.Parse(u.String())
tmp.Path = path.Join(append([]string{u.Path}, elem...)...)
return tmp.String()
}
func (wd *Driver) GET(pathElem ...string) (rawResp rawResponse, err error) {
return wd.Request(http.MethodGet, wd.concatURL(nil, pathElem...), nil)
}
func (wd *Driver) POST(data interface{}, pathElem ...string) (rawResp rawResponse, err error) {
var bsJSON []byte = nil
if data != nil {
if bsJSON, err = json.Marshal(data); err != nil {
return nil, err
}
}
return wd.Request(http.MethodPost, wd.concatURL(nil, pathElem...), bsJSON)
}
func (wd *Driver) DELETE(pathElem ...string) (rawResp rawResponse, err error) {
return wd.Request(http.MethodDelete, wd.concatURL(nil, pathElem...), nil)
}
func (wd *Driver) Request(method string, rawURL string, rawBody []byte) (rawResp rawResponse, err error) {
driverResult := &DriverResult{
RequestMethod: method,
RequestUrl: rawURL,
RequestBody: string(rawBody),
}
defer func() {
wd.session.addRequestResult(driverResult)
var logger *zerolog.Event
if err != nil {
driverResult.Success = false
driverResult.Error = err.Error()
logger = log.Error().Bool("success", false).Err(err)
} else {
driverResult.Success = true
logger = log.Debug().Bool("success", true)
}
logger = logger.Str("request_method", method).Str("request_url", rawURL).
Str("request_body", string(rawBody))
if !driverResult.RequestTime.IsZero() {
logger = logger.Int64("request_time", driverResult.RequestTime.UnixMilli())
}
if driverResult.ResponseStatus != 0 {
logger = logger.
Int("response_status", driverResult.ResponseStatus).
Int64("response_duration(ms)", driverResult.ResponseDuration).
Str("response_body", driverResult.ResponseBody)
}
logger.Msg("request uixt driver")
}()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var req *http.Request
if req, err = http.NewRequestWithContext(ctx, method, rawURL, bytes.NewBuffer(rawBody)); err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json;charset=UTF-8")
req.Header.Set("Accept", "application/json")
driverResult.RequestTime = time.Now()
var resp *http.Response
if resp, err = wd.client.Do(req); err != nil {
return nil, err
}
defer func() {
// https://github.com/etcd-io/etcd/blob/v3.3.25/pkg/httputil/httputil.go#L16-L22
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
rawResp, err = io.ReadAll(resp.Body)
duration := time.Since(driverResult.RequestTime)
driverResult.ResponseDuration = duration.Milliseconds()
driverResult.ResponseStatus = resp.StatusCode
if strings.HasSuffix(rawURL, "screenshot") {
// avoid printing screenshot data
driverResult.ResponseBody = "OMITTED"
} else {
driverResult.ResponseBody = string(rawResp)
}
if err != nil {
return nil, err
}
if err = rawResp.checkErr(); err != nil {
if resp.StatusCode == http.StatusOK {
return rawResp, nil
}
return nil, err
}
return
}
func convertToHTTPClient(conn net.Conn) *http.Client {
return &http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return conn, nil
},
},
Timeout: 30 * time.Second,
}
}

56
pkg/uixt/demo/main_test.go Normal file
View File

@@ -0,0 +1,56 @@
//go:build localtest
package demo
import (
"testing"
"time"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/pkg/uixt"
)
func TestIOSDemo(t *testing.T) {
device, err := uixt.NewIOSDevice(
uixt.WithWDAPort(8700), uixt.WithWDAMjpegPort(8800),
uixt.WithResetHomeOnStartup(false), // not reset home on startup
)
if err != nil {
t.Fatal(err)
}
capabilities := uixt.NewCapabilities()
capabilities.WithDefaultAlertAction(uixt.AlertActionAccept) // or uixt.AlertActionDismiss
driverExt, err := device.NewDriver(uixt.WithDriverCapabilities(capabilities))
if err != nil {
t.Fatal(err)
}
// release session
defer func() {
driverExt.Driver.DeleteSession()
}()
// 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」
for {
// take screenshot and get screen texts by OCR
ocrTexts, err := driverExt.GetScreenTexts()
if err != nil {
log.Error().Err(err).Msg("OCR GetTexts failed")
t.Fatal(err)
}
points, err := ocrTexts.FindTexts([]string{"青少年模式", "我知道了"})
if err != nil {
time.Sleep(1 * time.Second)
continue
}
point := points[1].Center()
err = driverExt.TapAbsXY(point.X, point.Y)
if err != nil {
t.Fatal(err)
}
}
}

29
pkg/uixt/drag.go Normal file
View File

@@ -0,0 +1,29 @@
package uixt
import (
"fmt"
"github.com/httprunner/httprunner/v5/code"
"github.com/pkg/errors"
)
func (dExt *DriverExt) Drag(fromX, fromY, toX, toY float64, options ...ActionOption) (err error) {
windowSize, err := dExt.Driver.WindowSize()
if err != nil {
return errors.Wrap(code.DeviceGetInfoError, err.Error())
}
width := windowSize.Width
height := windowSize.Height
if !assertRelative(fromX) || !assertRelative(fromY) ||
!assertRelative(toX) || !assertRelative(toY) {
return fmt.Errorf("fromX(%f), fromY(%f), toX(%f), toY(%f) must be less than 1",
fromX, fromY, toX, toY)
}
fromX = float64(width) * fromX
fromY = float64(height) * fromY
toX = float64(width) * toX
toY = float64(height) * toY
return dExt.Driver.Drag(fromX, fromY, toX, toY, options...)
}

BIN
pkg/uixt/evalite Normal file
View File

Binary file not shown.

136
pkg/uixt/ext.go Normal file
View File

@@ -0,0 +1,136 @@
package uixt
import (
"context"
"fmt"
_ "image/gif"
_ "image/png"
"github.com/httprunner/funplugin"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/internal/config"
)
type DriverExt struct {
Ctx context.Context
Device IDevice
Driver IWebDriver
ImageService IImageService // used to extract image data
// funplugin
plugin funplugin.IPlugin
}
func newDriverExt(device IDevice, driver IWebDriver, options ...DriverOption) (dExt *DriverExt, err error) {
driverOptions := NewDriverOptions()
for _, option := range options {
option(driverOptions)
}
dExt = &DriverExt{
Device: device,
Driver: driver,
plugin: driverOptions.plugin,
}
if driverOptions.withImageService {
if dExt.ImageService, err = newVEDEMImageService(); err != nil {
return nil, err
}
}
if driverOptions.withResultFolder {
// create results directory
if err = builtin.EnsureFolderExists(config.ResultsPath); err != nil {
return nil, errors.Wrap(err, "create results directory failed")
}
if err = builtin.EnsureFolderExists(config.ScreenShotsPath); err != nil {
return nil, errors.Wrap(err, "create screenshots directory failed")
}
}
return dExt, nil
}
func (dExt *DriverExt) Init() error {
// unlock device screen
err := dExt.Driver.Unlock()
if err != nil {
log.Error().Err(err).Msg("unlock device screen failed")
return err
}
return nil
}
func (dExt *DriverExt) assertOCR(text, assert string) error {
var options []ActionOption
options = append(options, WithScreenShotFileName(fmt.Sprintf("assert_ocr_%s", text)))
switch assert {
case AssertionEqual:
_, err := dExt.FindScreenText(text, options...)
if err != nil {
return errors.Wrap(err, "assert ocr equal failed")
}
case AssertionNotEqual:
_, err := dExt.FindScreenText(text, options...)
if err == nil {
return errors.New("assert ocr not equal failed")
}
case AssertionExists:
options = append(options, WithRegex(true))
_, err := dExt.FindScreenText(text, options...)
if err != nil {
return errors.Wrap(err, "assert ocr exists failed")
}
case AssertionNotExists:
options = append(options, WithRegex(true))
_, err := dExt.FindScreenText(text, options...)
if err == nil {
return errors.New("assert ocr not exists failed")
}
default:
return fmt.Errorf("unexpected assert method %s", assert)
}
return nil
}
func (dExt *DriverExt) assertForegroundApp(appName, assert string) (err error) {
err = dExt.Driver.AssertForegroundApp(appName)
switch assert {
case AssertionEqual:
if err != nil {
return errors.Wrap(err, "assert foreground app equal failed")
}
case AssertionNotEqual:
if err == nil {
return errors.New("assert foreground app not equal failed")
}
default:
return fmt.Errorf("unexpected assert method %s", assert)
}
return nil
}
func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) (err error) {
switch check {
case SelectorOCR:
err = dExt.assertOCR(expected, assert)
case SelectorForegroundApp:
err = dExt.assertForegroundApp(expected, assert)
}
if err != nil {
if message == nil {
message = []string{""}
}
log.Error().Err(err).Str("assert", assert).Str("expect", expected).
Str("msg", message[0]).Msg("validate failed")
return err
}
log.Info().Str("assert", assert).Str("expect", expected).Msg("validate success")
return nil
}

163
pkg/uixt/harmony_device.go Normal file
View File

@@ -0,0 +1,163 @@
package uixt
import (
"fmt"
"code.byted.org/iesqa/ghdc"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
)
var (
HdcServerHost = "localhost"
HdcServerPort = ghdc.HdcServerPort // 5037
)
type HarmonyDevice struct {
d *ghdc.Device
ConnectKey string `json:"connect_key,omitempty" yaml:"connect_key,omitempty"`
LogOn bool `json:"log_on,omitempty" yaml:"log_on,omitempty"`
}
func (dev *HarmonyDevice) Options() (deviceOptions []HarmonyDeviceOption) {
if dev.ConnectKey != "" {
deviceOptions = append(deviceOptions, WithConnectKey(dev.ConnectKey))
}
if dev.LogOn {
deviceOptions = append(deviceOptions, WithLogOn(true))
}
return
}
type HarmonyDeviceOption func(*HarmonyDevice)
func WithConnectKey(connectKey string) HarmonyDeviceOption {
return func(device *HarmonyDevice) {
device.ConnectKey = connectKey
}
}
func WithLogOn(logOn bool) HarmonyDeviceOption {
return func(device *HarmonyDevice) {
device.LogOn = logOn
}
}
func NewHarmonyDevice(options ...HarmonyDeviceOption) (device *HarmonyDevice, err error) {
device = &HarmonyDevice{}
for _, option := range options {
option(device)
}
deviceList, err := GetHarmonyDevices(device.ConnectKey)
if err != nil {
return nil, errors.Wrap(code.DeviceConnectionError, err.Error())
}
if device.ConnectKey == "" && len(deviceList) > 1 {
return nil, errors.Wrap(code.DeviceConnectionError, "more than one device connected, please specify the serial")
}
dev := deviceList[0]
if device.ConnectKey == "" {
selectSerial := dev.Serial()
device.ConnectKey = selectSerial
log.Warn().
Str("connectKey", device.ConnectKey).
Msg("harmony ConnectKey is not specified, select the first one")
}
device.d = dev
log.Info().Str("connectKey", device.ConnectKey).Msg("init harmony device")
return device, nil
}
func GetHarmonyDevices(serial ...string) (devices []*ghdc.Device, err error) {
var hdcClient ghdc.Client
if hdcClient, err = ghdc.NewClientWith(HdcServerHost, HdcServerPort); err != nil {
return nil, err
}
var deviceList []ghdc.Device
if deviceList, err = hdcClient.DeviceList(); err != nil {
return nil, err
}
// filter by serial
for _, d := range deviceList {
for _, s := range serial {
if s != "" && s != d.Serial() {
continue
}
devices = append(devices, &d)
}
}
if len(devices) == 0 {
var err error
if serial == nil || (len(serial) == 1 && serial[0] == "") {
err = fmt.Errorf("no harmony device found")
} else {
err = fmt.Errorf("no harmony device found for serial %v", serial)
}
return nil, err
}
return devices, nil
}
func (dev *HarmonyDevice) Init() error {
return nil
}
func (dev *HarmonyDevice) UUID() string {
return dev.ConnectKey
}
func (dev *HarmonyDevice) LogEnabled() bool {
return dev.LogOn
}
func (dev *HarmonyDevice) NewDriver(options ...DriverOption) (driverExt *DriverExt, err error) {
driver, err := newHarmonyDriver(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to new harmony driver")
return nil, err
}
driverExt, err = newDriverExt(dev, driver, options...)
if err != nil {
return nil, err
}
return driverExt, nil
}
func (dev *HarmonyDevice) NewUSBDriver(options ...DriverOption) (driver IWebDriver, err error) {
harmonyDriver, err := newHarmonyDriver(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to new harmony driver")
return nil, err
}
return harmonyDriver, nil
}
func (dev *HarmonyDevice) Install(appPath string, options ...InstallOption) error {
return nil
}
func (dev *HarmonyDevice) Uninstall(packageName string) error {
return nil
}
func (dev *HarmonyDevice) GetPackageInfo(packageName string) (AppInfo, error) {
log.Warn().Msg("get package info not implemented for harmony device, skip")
return AppInfo{}, nil
}
func (dev *HarmonyDevice) GetCurrentWindow() (WindowInfo, error) {
return WindowInfo{}, nil
}

335
pkg/uixt/harmony_hdc_driver.go Normal file
View File

@@ -0,0 +1,335 @@
package uixt
import (
"bytes"
"fmt"
"os"
"regexp"
"time"
"code.byted.org/iesqa/ghdc"
"github.com/rs/zerolog/log"
)
type hdcDriver struct {
points []ExportPoint
Driver
device *ghdc.Device
uiDriver *ghdc.UIDriver
}
type PowerStatus string
const (
POWER_STATUS_SUSPEND PowerStatus = "POWER_STATUS_SUSPEND"
POWER_STATUS_OFF PowerStatus = "POWER_STATUS_OFF"
POWER_STATUS_ON PowerStatus = "POWER_STATUS_ON"
)
func newHarmonyDriver(device *ghdc.Device) (driver *hdcDriver, err error) {
driver = new(hdcDriver)
driver.device = device
uiDriver, err := ghdc.NewUIDriver(*device)
if err != nil {
log.Error().Err(err).Msg("failed to new harmony ui driver")
return nil, err
}
driver.uiDriver = uiDriver
driver.NewSession(nil)
return
}
func (hd *hdcDriver) NewSession(capabilities Capabilities) (SessionInfo, error) {
hd.Driver.session.Reset()
hd.Unlock()
return SessionInfo{}, errDriverNotImplemented
}
func (hd *hdcDriver) DeleteSession() error {
return errDriverNotImplemented
}
func (hd *hdcDriver) GetSession() *DriverSession {
return &hd.Driver.session
}
func (hd *hdcDriver) Status() (DeviceStatus, error) {
return DeviceStatus{}, errDriverNotImplemented
}
func (hd *hdcDriver) DeviceInfo() (DeviceInfo, error) {
return DeviceInfo{}, errDriverNotImplemented
}
func (hd *hdcDriver) Location() (Location, error) {
return Location{}, errDriverNotImplemented
}
func (hd *hdcDriver) BatteryInfo() (BatteryInfo, error) {
return BatteryInfo{}, errDriverNotImplemented
}
func (hd *hdcDriver) WindowSize() (size Size, err error) {
display, err := hd.uiDriver.GetDisplaySize()
if err != nil {
log.Error().Err(err).Msg("failed to get window size")
return Size{}, err
}
size.Width = display.Width
size.Height = display.Height
return size, err
}
func (hd *hdcDriver) Screen() (Screen, error) {
return Screen{}, errDriverNotImplemented
}
func (hd *hdcDriver) Scale() (float64, error) {
return 1, nil
}
func (hd *hdcDriver) GetTimestamp() (timestamp int64, err error) {
return 0, errDriverNotImplemented
}
func (hd *hdcDriver) Homescreen() error {
return hd.uiDriver.PressKey(ghdc.KEYCODE_HOME)
}
func (hd *hdcDriver) Unlock() (err error) {
// Todo 检查是否锁屏 hdc shell hidumper -s RenderService -a screen
screenInfo, err := hd.device.RunShellCommand("hidumper", "-s", "RenderService", "-a", "screen")
if err != nil {
return err
}
re := regexp.MustCompile(`powerstatus=([\w_]+)`)
match := re.FindStringSubmatch(screenInfo)
log.Info().Msg("screen info: " + screenInfo)
if len(match) <= 1 {
return fmt.Errorf("failed to unlock; failed to find powerstatus")
}
if match[1] == string(POWER_STATUS_SUSPEND) || match[1] == string(POWER_STATUS_OFF) {
err = hd.uiDriver.PressPowerKey()
if err != nil {
return err
}
}
return hd.Swipe(500, 1500, 500, 500)
}
func (hd *hdcDriver) AppLaunch(packageName string) error {
// Todo
return errDriverNotImplemented
}
func (hd *hdcDriver) AppTerminate(packageName string) (bool, error) {
_, err := hd.device.RunShellCommand("aa", "force-stop", packageName)
if err != nil {
log.Error().Err(err).Msg("failed to terminal app")
return false, err
}
return true, nil
}
func (hd *hdcDriver) GetForegroundApp() (app AppInfo, err error) {
// Todo
return AppInfo{}, errDriverNotImplemented
}
func (hd *hdcDriver) AssertForegroundApp(packageName string, activityType ...string) error {
// Todo
return nil
}
func (hd *hdcDriver) StartCamera() error {
return errDriverNotImplemented
}
func (hd *hdcDriver) StopCamera() error {
return errDriverNotImplemented
}
func (hd *hdcDriver) Orientation() (orientation Orientation, err error) {
return OrientationPortrait, nil
}
func (hd *hdcDriver) Tap(x, y float64, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
if len(actionOptions.Offset) == 2 {
x += float64(actionOptions.Offset[0])
y += float64(actionOptions.Offset[1])
}
x += actionOptions.getRandomOffset()
y += actionOptions.getRandomOffset()
if actionOptions.Identifier != "" {
startTime := int(time.Now().UnixMilli())
hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100})
}
return hd.uiDriver.InjectGesture(ghdc.NewGesture().Start(ghdc.Point{X: int(x), Y: int(y)}).Pause(100))
}
func (hd *hdcDriver) DoubleTap(x, y float64, options ...ActionOption) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) TouchAndHold(x, y float64, options ...ActionOption) (err error) {
return errDriverNotImplemented
}
func (hd *hdcDriver) Drag(fromX, fromY, toX, toY float64, options ...ActionOption) error {
return errDriverNotImplemented
}
// Swipe works like Drag, but `pressForDuration` value is 0
func (hd *hdcDriver) Swipe(fromX, fromY, toX, toY float64, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
if len(actionOptions.Offset) == 4 {
fromX += float64(actionOptions.Offset[0])
fromY += float64(actionOptions.Offset[1])
toX += float64(actionOptions.Offset[2])
toY += float64(actionOptions.Offset[3])
}
fromX += actionOptions.getRandomOffset()
fromY += actionOptions.getRandomOffset()
toX += actionOptions.getRandomOffset()
toY += actionOptions.getRandomOffset()
duration := 200
if actionOptions.PressDuration > 0 {
duration = int(actionOptions.PressDuration * 1000)
}
if actionOptions.Identifier != "" {
startTime := int(time.Now().UnixMilli())
hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100})
}
return hd.uiDriver.InjectGesture(ghdc.NewGesture().Start(ghdc.Point{X: int(fromX), Y: int(fromY)}).MoveTo(ghdc.Point{X: int(toX), Y: int(toY)}, duration))
}
func (hd *hdcDriver) SetPasteboard(contentType PasteboardType, content string) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error) {
return nil, errDriverNotImplemented
}
func (hd *hdcDriver) SetIme(ime string) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) SendKeys(text string, options ...ActionOption) error {
return hd.uiDriver.InputText(text)
}
func (hd *hdcDriver) Input(text string, options ...ActionOption) error {
return hd.uiDriver.InputText(text)
}
func (hd *hdcDriver) Clear(packageName string) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) PressButton(devBtn DeviceButton) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) PressBack(options ...ActionOption) error {
return hd.uiDriver.PressBack()
}
func (hd *hdcDriver) Backspace(count int, options ...ActionOption) (err error) {
return nil
}
func (hd *hdcDriver) PressKeyCode(keyCode KeyCode) (err error) {
return errDriverNotImplemented
}
func (hd *hdcDriver) PressHarmonyKeyCode(keyCode ghdc.KeyCode) (err error) {
return hd.uiDriver.PressKey(keyCode)
}
func (hd *hdcDriver) Screenshot() (*bytes.Buffer, error) {
tempDir := os.TempDir()
screenshotPath := fmt.Sprintf("%s/screenshot_%d.png", tempDir, time.Now().Unix())
err := hd.uiDriver.Screenshot(screenshotPath)
if err != nil {
log.Error().Err(err).Msg("failed to screenshot")
return nil, err
}
defer func() {
_ = os.Remove(screenshotPath)
}()
raw, err := os.ReadFile(screenshotPath)
if err != nil {
log.Error().Err(err).Msg("failed to screenshot")
return nil, err
}
return bytes.NewBuffer(raw), nil
}
func (hd *hdcDriver) Source(srcOpt ...SourceOption) (string, error) {
return "", nil
}
func (hd *hdcDriver) LoginNoneUI(packageName, phoneNumber string, captcha, password string) (info AppLoginInfo, err error) {
err = errDriverNotImplemented
return
}
func (hd *hdcDriver) LogoutNoneUI(packageName string) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) TapByText(text string, options ...ActionOption) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) TapByTexts(actions ...TapTextAction) error {
return errDriverNotImplemented
}
func (hd *hdcDriver) AccessibleSource() (string, error) {
return "", errDriverNotImplemented
}
func (hd *hdcDriver) HealthCheck() error {
return errDriverNotImplemented
}
func (hd *hdcDriver) GetAppiumSettings() (map[string]interface{}, error) {
return nil, errDriverNotImplemented
}
func (hd *hdcDriver) SetAppiumSettings(settings map[string]interface{}) (map[string]interface{}, error) {
return nil, errDriverNotImplemented
}
func (hd *hdcDriver) IsHealthy() (bool, error) {
return false, errDriverNotImplemented
}
func (hd *hdcDriver) StartCaptureLog(identifier ...string) (err error) {
return errDriverNotImplemented
}
func (hd *hdcDriver) StopCaptureLog() (result interface{}, err error) {
// defer clear(hd.points)
return hd.points, nil
}
func (hd *hdcDriver) GetDriverResults() []*DriverResult {
return nil
}
func (hd *hdcDriver) RecordScreen(folderPath string, duration time.Duration) (videoPath string, err error) {
return "", nil
}
func (hd *hdcDriver) TearDown() error {
return nil
}

108
pkg/uixt/harmony_test.go Normal file
View File

@@ -0,0 +1,108 @@
//go:build localtest
package uixt
import (
"fmt"
"testing"
)
var harmonyDriverExt *DriverExt
func setupHarmonyDevice(t *testing.T) {
device, err := NewHarmonyDevice()
if err != nil {
t.Fatal(err)
}
driver, err = device.NewUSBDriver()
if err != nil {
t.Fatal(err)
}
harmonyDriverExt, err = newDriverExt(device, driver)
if err != nil {
t.Fatal(err)
}
}
func TestWindowSize(t *testing.T) {
setupHarmonyDevice(t)
size, err := driver.WindowSize()
if err != nil {
t.Fatal(err)
}
t.Log(fmt.Sprintf("width: %d, height: %d", size.Width, size.Height))
}
func TestHarmonyTap(t *testing.T) {
setupHarmonyDevice(t)
err := harmonyDriverExt.TapAbsXY(200, 2000)
if err != nil {
t.Fatal(err)
}
}
func TestHarmonySwipe(t *testing.T) {
setupHarmonyDevice(t)
err := harmonyDriverExt.SwipeLeft()
if err != nil {
t.Fatal(err)
}
}
func TestHarmonyInput(t *testing.T) {
setupHarmonyDevice(t)
err := harmonyDriverExt.Input("test")
if err != nil {
t.Fatal(err)
}
}
func TestHomeScreen(t *testing.T) {
setupHarmonyDevice(t)
err := driver.Homescreen()
if err != nil {
t.Fatal(err)
}
}
func TestUnlock(t *testing.T) {
setupHarmonyDevice(t)
err := driver.Unlock()
if err != nil {
t.Fatal(err)
}
}
func TestPressBack(t *testing.T) {
setupHarmonyDevice(t)
err := driver.PressBack()
if err != nil {
t.Fatal(err)
}
}
func TestScreenshot(t *testing.T) {
setupHarmonyDevice(t)
screenshot, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
t.Log(screenshot)
}
func TestLaunch(t *testing.T) {
setupHarmonyDevice(t)
err := driver.AppLaunch("")
if err != nil {
t.Fatal(err)
}
}
func TestForegroundApp(t *testing.T) {
setupHarmonyDevice(t)
appInfo, err := driver.GetForegroundApp()
if err != nil {
t.Fatal(err)
}
t.Log(appInfo)
}

15
pkg/uixt/input.go Normal file
View File

@@ -0,0 +1,15 @@
package uixt
import (
"github.com/pkg/errors"
"github.com/httprunner/httprunner/v5/code"
)
func (dExt *DriverExt) Input(text string) (err error) {
err = dExt.Driver.Input(text)
if err != nil {
return errors.Wrap(code.MobileUIInputError, err.Error())
}
return nil
}

134
pkg/uixt/install.go Normal file
View File

@@ -0,0 +1,134 @@
package uixt
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
type InstallOptions struct {
Reinstall bool
GrantPermission bool
Downgrade bool
RetryTimes int
}
type InstallOption func(o *InstallOptions)
func NewInstallOptions(options ...InstallOption) *InstallOptions {
installOptions := &InstallOptions{}
for _, option := range options {
option(installOptions)
}
return installOptions
}
func WithReinstall(reinstall bool) InstallOption {
return func(o *InstallOptions) {
o.Reinstall = reinstall
}
}
func WithGrantPermission(grantPermission bool) InstallOption {
return func(o *InstallOptions) {
o.GrantPermission = grantPermission
}
}
func WithDowngrade(downgrade bool) InstallOption {
return func(o *InstallOptions) {
o.Downgrade = downgrade
}
}
func WithRetryTimes(retryTimes int) InstallOption {
return func(o *InstallOptions) {
o.RetryTimes = retryTimes
}
}
type InstallResult struct {
Result int `json:"result"`
ErrorCode int `json:"errorCode"`
ErrorMsg string `json:"errorMsg"`
}
func (dExt *DriverExt) InstallByUrl(url string, options ...InstallOption) error {
// 获取当前目录
cwd, err := os.Getwd()
if err != nil {
return err
}
// 将文件保存到当前目录
appPath := filepath.Join(cwd, fmt.Sprint(time.Now().UnixNano())) // 替换为你想保存的文件名
err = builtin.DownloadFile(appPath, url)
if err != nil {
log.Error().Err(err).Msg("download file failed")
return err
}
err = dExt.Install(appPath, options...)
if err != nil {
log.Error().Err(err).Msg("install app failed")
return err
}
return nil
}
func (dExt *DriverExt) Install(filePath string, options ...InstallOption) error {
if _, ok := dExt.Device.(*AndroidDevice); ok {
stopChan := make(chan struct{})
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
actions := []TapTextAction{
{
Text: "^.*无视风险安装$",
Options: []ActionOption{WithTapOffset(100, 0), WithRegex(true), WithIgnoreNotFoundError(true)},
},
{
Text: "^已了解此应用未经检测.*",
Options: []ActionOption{WithTapOffset(-450, 0), WithRegex(true), WithIgnoreNotFoundError(true)},
},
}
_ = dExt.Driver.TapByTexts(actions...)
_ = dExt.TapByOCR(
"^(.*无视风险安装|确定|继续|完成|点击继续安装|继续安装旧版本|替换|授权本次安装|稍后提醒|继续安装|重新安装|安装)$",
WithRegex(true), WithIgnoreNotFoundError(true),
)
case <-stopChan:
log.Info().Msg("Ticker stopped")
return
}
}
}()
defer func() {
close(stopChan)
}()
}
return dExt.Device.Install(filePath, options...)
}
func (dExt *DriverExt) Uninstall(packageName string, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
err := dExt.Device.Uninstall(packageName)
if err != nil {
log.Warn().Err(err).Msg("failed to uninstall")
}
if actionOptions.IgnoreNotFoundError {
return nil
}
return err
}

660
pkg/uixt/interface.go Normal file
View File

@@ -0,0 +1,660 @@
package uixt
import (
"bytes"
"math"
"strings"
"time"
"github.com/httprunner/funplugin"
)
var (
DefaultWaitTimeout = 60 * time.Second
DefaultWaitInterval = 400 * time.Millisecond
)
type AlertAction string
const (
AlertActionAccept AlertAction = "accept"
AlertActionDismiss AlertAction = "dismiss"
)
type Capabilities map[string]interface{}
func NewCapabilities() Capabilities {
return make(Capabilities)
}
// WithDefaultAlertAction
func (caps Capabilities) WithDefaultAlertAction(alertAction AlertAction) Capabilities {
caps["defaultAlertAction"] = alertAction
return caps
}
// WithMaxTypingFrequency
//
// Defaults to `60`.
func (caps Capabilities) WithMaxTypingFrequency(n int) Capabilities {
if n <= 0 {
n = 60
}
caps["maxTypingFrequency"] = n
return caps
}
// WithWaitForIdleTimeout
//
// Defaults to `10`
func (caps Capabilities) WithWaitForIdleTimeout(second float64) Capabilities {
caps["waitForIdleTimeout"] = second
return caps
}
// WithShouldUseTestManagerForVisibilityDetection If set to YES will ask TestManagerDaemon for element visibility
//
// Defaults to `false`
func (caps Capabilities) WithShouldUseTestManagerForVisibilityDetection(b bool) Capabilities {
caps["shouldUseTestManagerForVisibilityDetection"] = b
return caps
}
// WithShouldUseCompactResponses If set to YES will use compact (standards-compliant) & faster responses
//
// Defaults to `true`
func (caps Capabilities) WithShouldUseCompactResponses(b bool) Capabilities {
caps["shouldUseCompactResponses"] = b
return caps
}
// WithElementResponseAttributes If shouldUseCompactResponses == NO,
// is the comma-separated list of fields to return with each element.
//
// Defaults to `type,label`.
func (caps Capabilities) WithElementResponseAttributes(s string) Capabilities {
caps["elementResponseAttributes"] = s
return caps
}
// WithShouldUseSingletonTestManager
//
// Defaults to `true`
func (caps Capabilities) WithShouldUseSingletonTestManager(b bool) Capabilities {
caps["shouldUseSingletonTestManager"] = b
return caps
}
// WithDisableAutomaticScreenshots
//
// Defaults to `true`
func (caps Capabilities) WithDisableAutomaticScreenshots(b bool) Capabilities {
caps["disableAutomaticScreenshots"] = b
return caps
}
// WithShouldTerminateApp
//
// Defaults to `true`
func (caps Capabilities) WithShouldTerminateApp(b bool) Capabilities {
caps["shouldTerminateApp"] = b
return caps
}
// WithEventloopIdleDelaySec
// Delays the invocation of '-[XCUIApplicationProcess setEventLoopHasIdled:]' by the timer interval passed.
// which is skipped on setting it to zero.
func (caps Capabilities) WithEventloopIdleDelaySec(second float64) Capabilities {
caps["eventloopIdleDelaySec"] = second
return caps
}
type SessionInfo struct {
SessionId string `json:"sessionId"`
Capabilities struct {
Device string `json:"device"`
BrowserName string `json:"browserName"`
SdkVersion string `json:"sdkVersion"`
CFBundleIdentifier string `json:"CFBundleIdentifier"`
} `json:"capabilities"`
}
type DeviceStatus struct {
Message string `json:"message"`
State string `json:"state"`
OS struct {
TestmanagerdVersion int `json:"testmanagerdVersion"`
Name string `json:"name"`
SdkVersion string `json:"sdkVersion"`
Version string `json:"version"`
} `json:"os"`
IOS struct {
IP string `json:"ip"`
SimulatorVersion string `json:"simulatorVersion"`
} `json:"ios"`
Ready bool `json:"ready"`
Build struct {
Time string `json:"time"`
ProductBundleIdentifier string `json:"productBundleIdentifier"`
} `json:"build"`
}
type DeviceInfo struct {
TimeZone string `json:"timeZone"`
CurrentLocale string `json:"currentLocale"`
Model string `json:"model"`
UUID string `json:"uuid"`
UserInterfaceIdiom int `json:"userInterfaceIdiom"`
UserInterfaceStyle string `json:"userInterfaceStyle"`
Name string `json:"name"`
IsSimulator bool `json:"isSimulator"`
ThermalState int `json:"thermalState"`
// ANDROID_ID A 64-bit number (as a hex string) that is uniquely generated when the user
// first sets up the device and should remain constant for the lifetime of the user's device. The value
// may change if a factory reset is performed on the device.
AndroidID string `json:"androidId"`
// Build.MANUFACTURER value
Manufacturer string `json:"manufacturer"`
// Build.BRAND value
Brand string `json:"brand"`
// Current running OS's API VERSION
APIVersion string `json:"apiVersion"`
// The current version string, for example "1.0" or "3.4b5"
PlatformVersion string `json:"platformVersion"`
// the name of the current celluar network carrier
CarrierName string `json:"carrierName"`
// the real size of the default display
RealDisplaySize string `json:"realDisplaySize"`
// The logical density of the display in Density Independent Pixel units.
DisplayDensity int `json:"displayDensity"`
// available networks
Networks []networkInfo `json:"networks"`
// current system locale
Locale string `json:"locale"`
Bluetooth struct {
State string `json:"state"`
} `json:"bluetooth"`
}
type networkCapabilities struct {
TransportTypes string `json:"transportTypes"`
NetworkCapabilities string `json:"networkCapabilities"`
LinkUpstreamBandwidthKbps int `json:"linkUpstreamBandwidthKbps"`
LinkDownBandwidthKbps int `json:"linkDownBandwidthKbps"`
SignalStrength int `json:"signalStrength"`
SSID string `json:"SSID"`
}
type networkInfo struct {
Type int `json:"type"`
TypeName string `json:"typeName"`
Subtype int `json:"subtype"`
SubtypeName string `json:"subtypeName"`
IsConnected bool `json:"isConnected"`
DetailedState string `json:"detailedState"`
State string `json:"state"`
ExtraInfo string `json:"extraInfo"`
IsAvailable bool `json:"isAvailable"`
IsRoaming bool `json:"isRoaming"`
IsFailover bool `json:"isFailover"`
Capabilities networkCapabilities `json:"capabilities"`
}
type Location struct {
AuthorizationStatus int `json:"authorizationStatus"`
Longitude float64 `json:"longitude"`
Latitude float64 `json:"latitude"`
Altitude float64 `json:"altitude"`
}
type BatteryInfo struct {
// Battery level in range [0.0, 1.0], where 1.0 means 100% charge.
Level float64 `json:"level"`
// Battery state ( 1: on battery, discharging; 2: plugged in, less than 100%, 3: plugged in, at 100% )
State BatteryState `json:"state"`
Status BatteryStatus `json:"status"`
}
type BatteryState int
const (
_ = iota
BatteryStateUnplugged BatteryState = iota // on battery, discharging
BatteryStateCharging // plugged in, less than 100%
BatteryStateFull // plugged in, at 100%
)
func (v BatteryState) String() string {
switch v {
case BatteryStateUnplugged:
return "On battery, discharging"
case BatteryStateCharging:
return "Plugged in, less than 100%"
case BatteryStateFull:
return "Plugged in, at 100%"
default:
return "UNKNOWN"
}
}
type Size struct {
Width int `json:"width"`
Height int `json:"height"`
}
func (s Size) IsNil() bool {
return s.Width == 0 && s.Height == 0
}
type Screen struct {
StatusBarSize Size `json:"statusBarSize"`
Scale float64 `json:"scale"`
}
type AppInfo struct {
Name string `json:"name,omitempty"`
AppBaseInfo
}
type WindowInfo struct {
PackageName string `json:"packageName,omitempty"`
Activity string `json:"activity,omitempty"`
}
type AppBaseInfo struct {
Pid int `json:"pid,omitempty"`
BundleId string `json:"bundleId,omitempty"` // ios package name
ViewController string `json:"viewController,omitempty"` // ios view controller
PackageName string `json:"packageName,omitempty"` // android package name
Activity string `json:"activity,omitempty"` // android activity
VersionName string `json:"versionName,omitempty"`
VersionCode interface{} `json:"versionCode,omitempty"` // int or string
AppName string `json:"appName,omitempty"`
AppPath string `json:"appPath,omitempty"`
AppMD5 string `json:"appMD5,omitempty"`
// AppIcon string `json:"appIcon,omitempty"`
}
type AppState int
const (
AppStateNotRunning AppState = 1 << iota
AppStateRunningBack
AppStateRunningFront
)
func (v AppState) String() string {
switch v {
case AppStateNotRunning:
return "Not Running"
case AppStateRunningBack:
return "Running (Back)"
case AppStateRunningFront:
return "Running (Front)"
default:
return "UNKNOWN"
}
}
// PasteboardType The type of the item on the pasteboard.
type PasteboardType string
const (
PasteboardTypePlaintext PasteboardType = "plaintext"
PasteboardTypeImage PasteboardType = "image"
PasteboardTypeUrl PasteboardType = "url"
)
const (
TextBackspace string = "\u0008"
TextDelete string = "\u007F"
)
// type KeyboardKeyLabel string
//
// const (
// KeyboardKeyReturn = "return"
// )
// DeviceButton A physical button on an iOS device.
type DeviceButton string
const (
DeviceButtonHome DeviceButton = "home"
DeviceButtonVolumeUp DeviceButton = "volumeUp"
DeviceButtonVolumeDown DeviceButton = "volumeDown"
)
type NotificationType string
const (
NotificationTypePlain NotificationType = "plain"
NotificationTypeDarwin NotificationType = "darwin"
)
type Orientation string
const (
// OrientationPortrait Device oriented vertically, home button on the bottom
OrientationPortrait Orientation = "PORTRAIT"
// OrientationPortraitUpsideDown Device oriented vertically, home button on the top
OrientationPortraitUpsideDown Orientation = "UIA_DEVICE_ORIENTATION_PORTRAIT_UPSIDEDOWN"
// OrientationLandscapeLeft Device oriented horizontally, home button on the right
OrientationLandscapeLeft Orientation = "LANDSCAPE"
// OrientationLandscapeRight Device oriented horizontally, home button on the left
OrientationLandscapeRight Orientation = "UIA_DEVICE_ORIENTATION_LANDSCAPERIGHT"
)
type Rotation struct {
X int `json:"x"`
Y int `json:"y"`
Z int `json:"z"`
}
// SourceOption Configure the format or attribute of the Source
type SourceOption map[string]interface{}
func NewSourceOption() SourceOption {
return make(SourceOption)
}
// WithFormatAsJson Application elements tree in form of json string
func (opt SourceOption) WithFormatAsJson() SourceOption {
opt["format"] = "json"
return opt
}
func (opt SourceOption) WithProcessName(processName string) SourceOption {
opt["processName"] = processName
return opt
}
// WithFormatAsXml Application elements tree in form of xml string
func (opt SourceOption) WithFormatAsXml() SourceOption {
opt["format"] = "xml"
return opt
}
// WithFormatAsDescription Application elements tree in form of internal XCTest debugDescription string
func (opt SourceOption) WithFormatAsDescription() SourceOption {
opt["format"] = "description"
return opt
}
// WithScope Allows to provide XML scope.
//
// only `xml` is supported.
func (opt SourceOption) WithScope(scope string) SourceOption {
if vFormat, ok := opt["format"]; ok && vFormat != "xml" {
return opt
}
opt["scope"] = scope
return opt
}
// WithExcludedAttributes Excludes the given attribute names.
// only `xml` is supported.
func (opt SourceOption) WithExcludedAttributes(attributes []string) SourceOption {
if vFormat, ok := opt["format"]; ok && vFormat != "xml" {
return opt
}
opt["excluded_attributes"] = strings.Join(attributes, ",")
return opt
}
type Condition func(wd IWebDriver) (bool, error)
type Direction string
const (
DirectionUp Direction = "up"
DirectionDown Direction = "down"
DirectionLeft Direction = "left"
DirectionRight Direction = "right"
)
type PickerWheelOrder string
const (
PickerWheelOrderNext PickerWheelOrder = "next"
PickerWheelOrderPrevious PickerWheelOrder = "previous"
)
type Point struct {
X int `json:"x"` // upper left X coordinate of selected element
Y int `json:"y"` // upper left Y coordinate of selected element
}
type PointF struct {
X float64 `json:"x"`
Y float64 `json:"y"`
}
func (p PointF) IsIdentical(p2 PointF) bool {
// set the coordinate precision to 1 pixel
return math.Abs(p.X-p2.X) < 1 && math.Abs(p.Y-p2.Y) < 1
}
type Rect struct {
Point
Size
}
type DriverOptions struct {
capabilities Capabilities
plugin funplugin.IPlugin
withImageService bool
withResultFolder bool
withUIAction bool
}
func NewDriverOptions() *DriverOptions {
return &DriverOptions{
withImageService: true,
withResultFolder: true,
withUIAction: true,
}
}
type DriverOption func(*DriverOptions)
func WithDriverCapabilities(capabilities Capabilities) DriverOption {
return func(options *DriverOptions) {
options.capabilities = capabilities
}
}
func WithDriverImageService(withImageService bool) DriverOption {
return func(options *DriverOptions) {
options.withImageService = withImageService
}
}
func WithDriverResultFolder(withResultFolder bool) DriverOption {
return func(options *DriverOptions) {
options.withResultFolder = withResultFolder
}
}
func WithUIAction(withUIAction bool) DriverOption {
return func(options *DriverOptions) {
options.withUIAction = withUIAction
}
}
func WithDriverPlugin(plugin funplugin.IPlugin) DriverOption {
return func(options *DriverOptions) {
options.plugin = plugin
}
}
// current implemeted device: IOSDevice, AndroidDevice, HarmonyDevice
type IDevice interface {
Init() error // init android device
UUID() string // ios udid or android serial
LogEnabled() bool
// TODO: add ctx to NewDriver
NewDriver(...DriverOption) (driverExt *DriverExt, err error)
Install(appPath string, options ...InstallOption) error
Uninstall(packageName string) error
GetPackageInfo(packageName string) (AppInfo, error)
GetCurrentWindow() (windowInfo WindowInfo, err error)
// Teardown() error
}
type ForegroundApp struct {
PackageName string
Activity string
}
// IWebDriver defines methods supported by IWebDriver drivers.
type IWebDriver interface {
// NewSession starts a new session and returns the SessionInfo.
NewSession(capabilities Capabilities) (SessionInfo, error)
// DeleteSession Kills application associated with that session and removes session
// 1) alertsMonitor disable
// 2) testedApplicationBundleId terminate
DeleteSession() error
// GetSession returns session cache, including requests, screenshots, etc.
GetSession() *DriverSession
Status() (DeviceStatus, error)
DeviceInfo() (DeviceInfo, error)
// Location Returns device location data.
//
// It requires to configure location access permission by manual.
// The response of 'latitude', 'longitude' and 'altitude' are always zero (0) without authorization.
// 'authorizationStatus' indicates current authorization status. '3' is 'Always'.
// https://developer.apple.com/documentation/corelocation/clauthorizationstatus
//
// Settings -> Privacy -> Location Service -> WebDriverAgent-Runner -> Always
//
// The return value could be zero even if the permission is set to 'Always'
// since the location service needs some time to update the location data.
Location() (Location, error)
BatteryInfo() (BatteryInfo, error)
// WindowSize Return the width and height in portrait mode.
// when getting the window size in wda/ui2/adb, if the device is in landscape mode,
// the width and height will be reversed.
WindowSize() (Size, error)
Screen() (Screen, error)
Scale() (float64, error)
// GetTimestamp returns the timestamp of the mobile device
GetTimestamp() (timestamp int64, err error)
// Homescreen Forces the device under test to switch to the home screen
Homescreen() error
Unlock() (err error)
// AppLaunch Launch an application with given bundle identifier in scope of current session.
// !This method is only available since Xcode9 SDK
AppLaunch(packageName string) error
// AppTerminate Terminate an application with the given package name.
// Either `true` if the app has been successfully terminated or `false` if it was not running
AppTerminate(packageName string) (bool, error)
// GetForegroundApp returns current foreground app package name and activity name
GetForegroundApp() (app AppInfo, err error)
// AssertForegroundApp returns nil if the given package and activity are in foreground
AssertForegroundApp(packageName string, activityType ...string) error
// StartCamera Starts a new camera for recording
StartCamera() error
// StopCamera Stops the camera for recording
StopCamera() error
Orientation() (orientation Orientation, err error)
// Tap Sends a tap event at the coordinate.
Tap(x, y float64, options ...ActionOption) error
// DoubleTap Sends a double tap event at the coordinate.
DoubleTap(x, y float64, options ...ActionOption) error
// TouchAndHold Initiates a long-press gesture at the coordinate, holding for the specified duration.
// second: The default value is 1
TouchAndHold(x, y float64, options ...ActionOption) error
// Drag Initiates a press-and-hold gesture at the coordinate, then drags to another coordinate.
// WithPressDurationOption option can be used to set pressForDuration (default to 1 second).
Drag(fromX, fromY, toX, toY float64, options ...ActionOption) error
// Swipe works like Drag, but `pressForDuration` value is 0
Swipe(fromX, fromY, toX, toY float64, options ...ActionOption) error
// SetPasteboard Sets data to the general pasteboard
SetPasteboard(contentType PasteboardType, content string) error
// GetPasteboard Gets the data contained in the general pasteboard.
// It worked when `WDA` was foreground. https://github.com/appium/WebDriverAgent/issues/330
GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error)
SetIme(ime string) error
// SendKeys Types a string into active element. There must be element with keyboard focus,
// otherwise an error is raised.
// WithFrequency option can be used to set frequency of typing (letters per sec). The default value is 60
SendKeys(text string, options ...ActionOption) error
// Input works like SendKeys
Input(text string, options ...ActionOption) error
Clear(packageName string) error
// PressButton Presses the corresponding hardware button on the device
PressButton(devBtn DeviceButton) error
// PressBack Presses the back button
PressBack(options ...ActionOption) error
PressKeyCode(keyCode KeyCode) (err error)
Backspace(count int, options ...ActionOption) (err error)
Screenshot() (*bytes.Buffer, error)
// Source Return application elements tree
Source(srcOpt ...SourceOption) (string, error)
LoginNoneUI(packageName, phoneNumber string, captcha, password string) (info AppLoginInfo, err error)
LogoutNoneUI(packageName string) error
TapByText(text string, options ...ActionOption) error
TapByTexts(actions ...TapTextAction) error
// AccessibleSource Return application elements accessibility tree
AccessibleSource() (string, error)
// HealthCheck Health check might modify simulator state so it should only be called in-between testing sessions
// Checks health of XCTest by:
// 1) Querying application for some elements,
// 2) Triggering some device events.
HealthCheck() error
GetAppiumSettings() (map[string]interface{}, error)
SetAppiumSettings(settings map[string]interface{}) (map[string]interface{}, error)
IsHealthy() (bool, error)
// triggers the log capture and returns the log entries
StartCaptureLog(identifier ...string) (err error)
StopCaptureLog() (result interface{}, err error)
GetDriverResults() []*DriverResult
RecordScreen(folderPath string, duration time.Duration) (videoPath string, err error)
TearDown() error
}

831
pkg/uixt/ios_device.go Normal file
View File

@@ -0,0 +1,831 @@
package uixt
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strconv"
"time"
"github.com/Masterminds/semver"
"github.com/danielpaulus/go-ios/ios"
"github.com/danielpaulus/go-ios/ios/deviceinfo"
"github.com/danielpaulus/go-ios/ios/diagnostics"
"github.com/danielpaulus/go-ios/ios/forward"
"github.com/danielpaulus/go-ios/ios/imagemounter"
"github.com/danielpaulus/go-ios/ios/installationproxy"
"github.com/danielpaulus/go-ios/ios/instruments"
"github.com/danielpaulus/go-ios/ios/testmanagerd"
"github.com/danielpaulus/go-ios/ios/tunnel"
"github.com/danielpaulus/go-ios/ios/zipconduit"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
const (
defaultWDAPort = 8100
defaultMjpegPort = 9100
defaultBightInsightPort = 8000
defaultDouyinServerPort = 32921
)
const (
// Changes the value of maximum depth for traversing elements source tree.
// It may help to prevent out of memory or timeout errors while getting the elements source tree,
// but it might restrict the depth of source tree.
// A part of elements source tree might be lost if the value was too small. Defaults to 50
snapshotMaxDepth = 10
// Allows to customize accept/dismiss alert button selector.
// It helps you to handle an arbitrary element as accept button in accept alert command.
// The selector should be a valid class chain expression, where the search root is the alert element itself.
// The default button location algorithm is used if the provided selector is wrong or does not match any element.
// e.g. **/XCUIElementTypeButton[`label CONTAINS[c] ‘accept’`]
acceptAlertButtonSelector = "**/XCUIElementTypeButton[`label IN {'允许','好','仅在使用应用期间','稍后再说'}`]"
dismissAlertButtonSelector = "**/XCUIElementTypeButton[`label IN {'不允许','暂不'}`]"
)
var tunnelManager *tunnel.TunnelManager = nil
type IOSDeviceOption func(*IOSDevice)
func WithUDID(udid string) IOSDeviceOption {
return func(device *IOSDevice) {
device.UDID = udid
}
}
func WithWDAPort(port int) IOSDeviceOption {
return func(device *IOSDevice) {
device.Port = port
}
}
func WithWDAMjpegPort(port int) IOSDeviceOption {
return func(device *IOSDevice) {
device.MjpegPort = port
}
}
func WithWDALogOn(logOn bool) IOSDeviceOption {
return func(device *IOSDevice) {
device.LogOn = logOn
}
}
func WithIOSStub(stub bool) IOSDeviceOption {
return func(device *IOSDevice) {
device.STUB = stub
}
}
func WithResetHomeOnStartup(reset bool) IOSDeviceOption {
return func(device *IOSDevice) {
device.ResetHomeOnStartup = reset
}
}
func WithSnapshotMaxDepth(depth int) IOSDeviceOption {
return func(device *IOSDevice) {
device.SnapshotMaxDepth = depth
}
}
func WithAcceptAlertButtonSelector(selector string) IOSDeviceOption {
return func(device *IOSDevice) {
device.AcceptAlertButtonSelector = selector
}
}
func WithDismissAlertButtonSelector(selector string) IOSDeviceOption {
return func(device *IOSDevice) {
device.DismissAlertButtonSelector = selector
}
}
func GetIOSDevices(udid ...string) (deviceList []ios.DeviceEntry, err error) {
devices, err := ios.ListDevices()
if err != nil {
return nil, errors.Wrap(code.DeviceConnectionError,
fmt.Sprintf("list ios devices failed: %v", err))
}
for _, d := range devices.DeviceList {
if len(udid) > 0 {
for _, u := range udid {
if u != "" && u != d.Properties.SerialNumber {
continue
}
// filter non-usb ios devices
if d.Properties.ConnectionType != "USB" {
continue
}
deviceList = append(deviceList, d)
}
} else {
deviceList = devices.DeviceList
}
}
if len(deviceList) == 0 {
var err error
if udid == nil || (len(udid) == 1 && udid[0] == "") {
err = fmt.Errorf("no ios device found")
} else {
err = fmt.Errorf("no ios device found for udid %v", udid)
}
return nil, err
}
return deviceList, nil
}
func GetIOSDeviceOptions(dev *IOSDevice) (deviceOptions []IOSDeviceOption) {
if dev.UDID != "" {
deviceOptions = append(deviceOptions, WithUDID(dev.UDID))
}
if dev.Port != 0 {
deviceOptions = append(deviceOptions, WithWDAPort(dev.Port))
}
if dev.MjpegPort != 0 {
deviceOptions = append(deviceOptions, WithWDAMjpegPort(dev.MjpegPort))
}
if dev.LogOn {
deviceOptions = append(deviceOptions, WithWDALogOn(true))
}
if dev.ResetHomeOnStartup {
deviceOptions = append(deviceOptions, WithResetHomeOnStartup(true))
}
if dev.SnapshotMaxDepth != 0 {
deviceOptions = append(deviceOptions, WithSnapshotMaxDepth(dev.SnapshotMaxDepth))
}
if dev.AcceptAlertButtonSelector != "" {
deviceOptions = append(deviceOptions, WithAcceptAlertButtonSelector(dev.AcceptAlertButtonSelector))
}
if dev.DismissAlertButtonSelector != "" {
deviceOptions = append(deviceOptions, WithDismissAlertButtonSelector(dev.DismissAlertButtonSelector))
}
return
}
func StartTunnel(recordsPath string, tunnelInfoPort int, userspaceTUN bool) (err error) {
pm, err := tunnel.NewPairRecordManager(recordsPath)
if err != nil {
return err
}
tm := tunnel.NewTunnelManager(pm, userspaceTUN)
go func() {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
err := tm.UpdateTunnels(context.Background())
if err != nil {
log.Error().Err(err).Msg("failed to update tunnels")
}
}
}()
go func() {
err := tunnel.ServeTunnelInfo(tm, tunnelInfoPort)
if err != nil {
log.Error().Err(err).Msg("failed to start tunnel server")
}
}()
log.Info().Msg("Tunnel server started")
return nil
}
func RebootTunnel() (err error) {
if tunnelManager != nil {
_ = tunnelManager.Close()
}
return StartTunnel(os.TempDir(), ios.HttpApiPort(), true)
}
func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) {
device = &IOSDevice{
Port: defaultWDAPort,
MjpegPort: defaultMjpegPort,
SnapshotMaxDepth: snapshotMaxDepth,
AcceptAlertButtonSelector: acceptAlertButtonSelector,
DismissAlertButtonSelector: dismissAlertButtonSelector,
// switch to iOS springboard before init WDA session
// avoid getting stuck when some super app is active such as douyin or wexin
ResetHomeOnStartup: true,
listeners: make(map[int]*forward.ConnListener),
}
for _, option := range options {
option(device)
}
deviceList, err := GetIOSDevices(device.UDID)
if err != nil {
return nil, errors.Wrap(code.DeviceConnectionError, err.Error())
}
if device.UDID == "" && len(deviceList) > 1 {
return nil, errors.Wrap(code.DeviceConnectionError, "more than one device connected, please specify the udid")
}
dev := deviceList[0]
udid := dev.Properties.SerialNumber
if device.UDID == "" {
device.UDID = udid
log.Warn().
Str("udid", udid).
Msg("ios UDID is not specified, select the first one")
}
device.d = dev
log.Info().Str("udid", device.UDID).Msg("init ios device")
err = device.Init()
if err != nil {
_ = device.Teardown()
return nil, err
}
return device, nil
}
type IOSDevice struct {
d ios.DeviceEntry
listeners map[int]*forward.ConnListener
UDID string `json:"udid,omitempty" yaml:"udid,omitempty"`
Port int `json:"port,omitempty" yaml:"port,omitempty"` // WDA remote port
MjpegPort int `json:"mjpeg_port,omitempty" yaml:"mjpeg_port,omitempty"` // WDA remote MJPEG port
STUB bool `json:"stub,omitempty" yaml:"stub,omitempty"` // use stub
LogOn bool `json:"log_on,omitempty" yaml:"log_on,omitempty"`
// switch to iOS springboard before init WDA session
ResetHomeOnStartup bool `json:"reset_home_on_startup,omitempty" yaml:"reset_home_on_startup,omitempty"`
// config appium settings
SnapshotMaxDepth int `json:"snapshot_max_depth,omitempty" yaml:"snapshot_max_depth,omitempty"`
AcceptAlertButtonSelector string `json:"accept_alert_button_selector,omitempty" yaml:"accept_alert_button_selector,omitempty"`
DismissAlertButtonSelector string `json:"dismiss_alert_button_selector,omitempty" yaml:"dismiss_alert_button_selector,omitempty"`
}
func (dev *IOSDevice) Options() (deviceOptions []IOSDeviceOption) {
if dev.UDID != "" {
deviceOptions = append(deviceOptions, WithUDID(dev.UDID))
}
if dev.Port != 0 {
deviceOptions = append(deviceOptions, WithWDAPort(dev.Port))
}
if dev.MjpegPort != 0 {
deviceOptions = append(deviceOptions, WithWDAMjpegPort(dev.MjpegPort))
}
if dev.STUB {
deviceOptions = append(deviceOptions, WithIOSStub(true))
}
if dev.LogOn {
deviceOptions = append(deviceOptions, WithWDALogOn(true))
}
if dev.ResetHomeOnStartup {
deviceOptions = append(deviceOptions, WithResetHomeOnStartup(true))
}
if dev.SnapshotMaxDepth != 0 {
deviceOptions = append(deviceOptions, WithSnapshotMaxDepth(dev.SnapshotMaxDepth))
}
if dev.AcceptAlertButtonSelector != "" {
deviceOptions = append(deviceOptions, WithAcceptAlertButtonSelector(dev.AcceptAlertButtonSelector))
}
if dev.DismissAlertButtonSelector != "" {
deviceOptions = append(deviceOptions, WithDismissAlertButtonSelector(dev.DismissAlertButtonSelector))
}
return
}
type DeviceDetail struct {
DeviceName string `json:"deviceName,omitempty"`
DeviceClass string `json:"deviceClass,omitempty"`
ProductVersion string `json:"productVersion,omitempty"`
ProductType string `json:"productType,omitempty"`
ProductName string `json:"productName,omitempty"`
PasswordProtected bool `json:"passwordProtected,omitempty"`
ModelNumber string `json:"modelNumber,omitempty"`
SerialNumber string `json:"serialNumber,omitempty"`
SIMStatus string `json:"simStatus,omitempty"`
PhoneNumber string `json:"phoneNumber,omitempty"`
CPUArchitecture string `json:"cpuArchitecture,omitempty"`
ProtocolVersion string `json:"protocolVersion,omitempty"`
RegionInfo string `json:"regionInfo,omitempty"`
TimeZone string `json:"timeZone,omitempty"`
UniqueDeviceID string `json:"uniqueDeviceID,omitempty"`
WiFiAddress string `json:"wifiAddress,omitempty"`
BuildVersion string `json:"buildVersion,omitempty"`
}
type ApplicationType string
const (
ApplicationTypeSystem ApplicationType = "System"
ApplicationTypeUser ApplicationType = "User"
ApplicationTypeInternal ApplicationType = "internal"
ApplicationTypeAny ApplicationType = "Any"
)
func (dev *IOSDevice) Init() error {
images, err := dev.ListImages()
if err != nil {
return err
}
version, err := dev.getVersion()
if err != nil {
return err
}
if len(images) == 0 && version.LessThan(ios.IOS17()) {
// Notice: iOS 17.0+ does not need to mount developer image
err = dev.AutoMountImage(os.TempDir())
if err != nil {
return err
}
}
return nil
}
func (dev *IOSDevice) Teardown() error {
for _, listener := range dev.listeners {
_ = listener.Close()
}
return nil
}
func (dev *IOSDevice) UUID() string {
return dev.UDID
}
func (dev *IOSDevice) LogEnabled() bool {
return dev.LogOn
}
func (dev *IOSDevice) getAppInfo(packageName string) (appInfo AppInfo, err error) {
apps, err := dev.ListApps(ApplicationTypeAny)
if err != nil {
return AppInfo{}, err
}
for _, app := range apps {
if app.CFBundleIdentifier == packageName {
appInfo.BundleId = app.CFBundleIdentifier
appInfo.AppName = app.CFBundleName
appInfo.PackageName = app.CFBundleIdentifier
appInfo.VersionName = app.CFBundleShortVersionString
appInfo.VersionCode = app.CFBundleVersion
return appInfo, err
}
}
return AppInfo{}, fmt.Errorf("not found App by bundle id: %s", packageName)
}
func (dev *IOSDevice) NewDriver(options ...DriverOption) (driverExt *DriverExt, err error) {
driverOptions := NewDriverOptions()
for _, option := range options {
option(driverOptions)
}
// init WDA driver
capabilities := driverOptions.capabilities
if capabilities == nil {
capabilities = NewCapabilities()
capabilities.WithDefaultAlertAction(AlertActionAccept)
}
var driver IWebDriver
if dev.STUB {
driver, err = dev.NewStubDriver()
if err != nil {
return nil, errors.Wrap(err, "failed to init Stub driver")
}
} else {
driver, err = dev.NewHTTPDriver(capabilities)
if err != nil {
return nil, errors.Wrap(err, "failed to init WDA driver")
}
settings, err := driver.SetAppiumSettings(map[string]interface{}{
"snapshotMaxDepth": dev.SnapshotMaxDepth,
"acceptAlertButtonSelector": dev.AcceptAlertButtonSelector,
})
if err != nil {
return nil, errors.Wrap(err, "failed to set appium WDA settings")
}
log.Info().Interface("appiumWDASettings", settings).Msg("set appium WDA settings")
}
if dev.ResetHomeOnStartup {
log.Info().Msg("go back to home screen")
if err = driver.Homescreen(); err != nil {
return nil, errors.Wrap(code.MobileUIDriverError,
fmt.Sprintf("go back to home screen failed: %v", err))
}
}
driverExt, err = newDriverExt(dev, driver, options...)
if err != nil {
return nil, err
}
settings, err := driverExt.Driver.SetAppiumSettings(map[string]interface{}{
"snapshotMaxDepth": dev.SnapshotMaxDepth,
"acceptAlertButtonSelector": dev.AcceptAlertButtonSelector,
})
if err != nil {
return nil, errors.Wrap(err, "failed to set appium WDA settings")
}
log.Info().Interface("appiumWDASettings", settings).Msg("set appium WDA settings")
if dev.LogOn {
err = driverExt.Driver.StartCaptureLog("hrp_wda_log")
if err != nil {
return nil, err
}
}
return driverExt, nil
}
func (dev *IOSDevice) Install(appPath string, options ...InstallOption) (err error) {
opts := NewInstallOptions(options...)
for i := 0; i <= opts.RetryTimes; i++ {
var conn *zipconduit.Connection
conn, err = zipconduit.New(dev.d)
if err != nil {
return err
}
defer conn.Close()
err = conn.SendFile(appPath)
if err != nil {
log.Error().Err(err).Msg(fmt.Sprintf("failed to install app Retry time %d", i))
}
if err == nil {
return nil
}
}
return err
}
func (dev *IOSDevice) InstallByUrl(url string, options ...InstallOption) (err error) {
appPath, err := builtin.DownloadFileByUrl(url)
if err != nil {
return err
}
err = dev.Install(appPath, options...)
if err != nil {
return err
}
return nil
}
func (dev *IOSDevice) Uninstall(bundleId string) error {
svc, err := installationproxy.New(dev.d)
if err != nil {
return err
}
defer svc.Close()
err = svc.Uninstall(bundleId)
if err != nil {
return err
}
return nil
}
func (dev *IOSDevice) forward(localPort, remotePort int) error {
if dev.listeners[localPort] != nil {
log.Warn().Msg(fmt.Sprintf("local port :%d is already in use", localPort))
_ = dev.listeners[localPort].Close()
}
listener, err := forward.Forward(dev.d, uint16(localPort), uint16(remotePort))
if err != nil {
log.Error().Err(err).Msg(fmt.Sprintf("failed to forward %d to %d", localPort, remotePort))
return err
}
dev.listeners[localPort] = listener
return nil
}
func (dev *IOSDevice) GetDeviceInfo() (*DeviceDetail, error) {
deviceInfo, err := deviceinfo.NewDeviceInfo(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to get device info")
return nil, err
}
defer deviceInfo.Close()
info, err := deviceInfo.GetDisplayInfo()
if err != nil {
log.Error().Err(err).Msg("failed to get device info")
return nil, err
}
jsonData, err := json.Marshal(info)
if err != nil {
return nil, err
}
// 将 JSON 反序列化为结构体
detail := new(DeviceDetail)
err = json.Unmarshal(jsonData, &detail)
if err != nil {
return nil, err
}
return detail, err
}
func (dev *IOSDevice) ListApps(appType ApplicationType) (apps []installationproxy.AppInfo, err error) {
svc, _ := installationproxy.New(dev.d)
defer svc.Close()
switch appType {
case ApplicationTypeSystem:
apps, err = svc.BrowseSystemApps()
case ApplicationTypeAny:
apps, err = svc.BrowseAllApps()
case ApplicationTypeInternal:
apps, err = svc.BrowseFileSharingApps()
case ApplicationTypeUser:
apps, err = svc.BrowseUserApps()
}
if err != nil {
log.Error().Err(err).Msg("failed to list ios apps")
return nil, err
}
return apps, nil
}
func (dev *IOSDevice) GetAppInfo(packageName string) (appInfo installationproxy.AppInfo, err error) {
svc, _ := installationproxy.New(dev.d)
defer svc.Close()
apps, err := svc.BrowseAllApps()
if err != nil {
log.Error().Err(err).Msg("failed to list ios apps")
return installationproxy.AppInfo{}, err
}
for _, app := range apps {
if app.CFBundleIdentifier == packageName {
return app, nil
}
}
return installationproxy.AppInfo{}, nil
}
func (dev *IOSDevice) ListImages() (images []string, err error) {
conn, err := imagemounter.NewImageMounter(dev.d)
if err != nil {
return nil, errors.Wrap(code.DeviceConnectionError, err.Error())
}
defer conn.Close()
signatures, err := conn.ListImages()
if err != nil {
return nil, errors.Wrap(code.DeviceConnectionError, err.Error())
}
for _, sig := range signatures {
images = append(images, fmt.Sprintf("%x", sig))
}
return
}
func (dev *IOSDevice) MountImage(imagePath string) (err error) {
log.Info().Str("imagePath", imagePath).Msg("mount ios developer image")
conn, err := imagemounter.NewImageMounter(dev.d)
if err != nil {
return errors.Wrap(code.DeviceConnectionError, err.Error())
}
defer conn.Close()
err = conn.MountImage(imagePath)
if err != nil {
return errors.Wrapf(code.DeviceConnectionError,
"mount ios developer image failed: %v", err)
}
log.Info().Str("imagePath", imagePath).Msg("mount ios developer image success")
return nil
}
func (dev *IOSDevice) AutoMountImage(baseDir string) (err error) {
log.Info().Str("baseDir", baseDir).Msg("auto mount ios developer image")
imagePath, err := imagemounter.DownloadImageFor(dev.d, baseDir)
if err != nil {
return errors.Wrapf(code.DeviceConnectionError,
"download ios developer image failed: %v", err)
}
return dev.MountImage(imagePath)
}
func (dev *IOSDevice) RunXCTest(ctx context.Context, bundleID, testRunnerBundleID, xctestConfig string) (err error) {
log.Info().Str("bundleID", bundleID).
Str("testRunnerBundleID", testRunnerBundleID).
Str("xctestConfig", xctestConfig).
Msg("run xctest")
listener := testmanagerd.NewTestListener(io.Discard, io.Discard, os.TempDir())
config := testmanagerd.TestConfig{
BundleId: bundleID,
TestRunnerBundleId: testRunnerBundleID,
XctestConfigName: xctestConfig,
Device: dev.d,
Listener: listener,
}
_, err = testmanagerd.RunTestWithConfig(ctx, config)
if err != nil {
log.Error().Err(err).Msg("run xctest failed")
return err
}
return nil
}
func (dev *IOSDevice) RunXCTestDaemon(ctx context.Context, bundleID, testRunnerBundleID, xctestConfig string) {
ctx, stopWda := context.WithCancel(ctx)
go func() {
err := dev.RunXCTest(ctx, bundleID, testRunnerBundleID, xctestConfig)
if err != nil {
log.Error().Err(err).Msg("wda ended")
}
stopWda()
}()
}
func (dev *IOSDevice) getVersion() (version *semver.Version, err error) {
version, err = ios.GetProductVersion(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to get version")
return nil, err
}
log.Info().Str("version", version.String()).Msg("get ios device version")
return version, nil
}
func (dev *IOSDevice) ListProcess(applicationsOnly bool) (processList []instruments.ProcessInfo, err error) {
service, err := instruments.NewDeviceInfoService(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to list process")
return
}
defer service.Close()
processList, err = service.ProcessList()
if applicationsOnly {
var applicationProcessList []instruments.ProcessInfo
for _, processInfo := range processList {
if processInfo.IsApplication {
applicationProcessList = append(applicationProcessList, processInfo)
}
}
processList = applicationProcessList
}
return
}
func (dev *IOSDevice) Reboot() error {
err := diagnostics.Reboot(dev.d)
if err != nil {
log.Error().Err(err).Msg("failed to reboot device")
return err
}
return nil
}
// NewHTTPDriver creates new remote HTTP client, this will also start a new session.
func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver IWebDriver, err error) {
var localPort int
localPort, err = strconv.Atoi(os.Getenv("WDA_LOCAL_PORT"))
if err != nil {
localPort, err = builtin.GetFreePort()
if err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("get free port failed: %v", err))
}
if err = dev.forward(localPort, dev.Port); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("forward tcp port failed: %v", err))
}
} else {
log.Info().Int("WDA_LOCAL_PORT", localPort).Msg("reuse WDA local port")
}
var localMjpegPort int
localMjpegPort, err = strconv.Atoi(os.Getenv("WDA_LOCAL_MJPEG_PORT"))
if err != nil {
localMjpegPort, err = builtin.GetFreePort()
if err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("get free port failed: %v", err))
}
if err = dev.forward(localMjpegPort, dev.MjpegPort); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("forward tcp port failed: %v", err))
}
} else {
log.Info().Int("WDA_LOCAL_MJPEG_PORT", localMjpegPort).
Msg("reuse WDA local mjpeg port")
}
log.Info().Interface("capabilities", capabilities).
Int("localPort", localPort).Int("localMjpegPort", localMjpegPort).
Msg("init WDA HTTP driver")
wd := new(wdaDriver)
wd.device = dev
wd.udid = dev.UDID
wd.client = &http.Client{
Timeout: time.Second * 10, // 设置超时时间为 10 秒
}
host := "localhost"
if wd.urlPrefix, err = url.Parse(fmt.Sprintf("http://%s:%d", host, localPort)); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError, err.Error())
}
// create new session
var sessionInfo SessionInfo
if sessionInfo, err = wd.NewSession(capabilities); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError, err.Error())
}
wd.session.ID = sessionInfo.SessionId
if wd.mjpegHTTPConn, err = net.Dial(
"tcp",
fmt.Sprintf("%s:%d", host, localMjpegPort),
); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError, err.Error())
}
wd.mjpegClient = convertToHTTPClient(wd.mjpegHTTPConn)
wd.mjpegUrl = fmt.Sprintf("%s:%d", host, localMjpegPort)
// init WDA scale
if wd.scale, err = wd.Scale(); err != nil {
return nil, err
}
return wd, nil
}
func (dev *IOSDevice) NewStubDriver() (driver IWebDriver, err error) {
localStubPort, err := builtin.GetFreePort()
if err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("get free port failed: %v", err))
}
if err = dev.forward(localStubPort, defaultBightInsightPort); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("forward tcp port failed: %v", err))
}
localServerPort, err := builtin.GetFreePort()
if err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("get free port failed: %v", err))
}
if err = dev.forward(localServerPort, defaultDouyinServerPort); err != nil {
return nil, errors.Wrap(code.DeviceHTTPDriverError,
fmt.Sprintf("forward tcp port failed: %v", err))
}
host := "localhost"
stubDriver, err := newStubIOSDriver(
fmt.Sprintf("http://%s:%d", host, localStubPort),
fmt.Sprintf("http://%s:%d", host, localServerPort), dev)
if err != nil {
return nil, err
}
return stubDriver, nil
}
func (dev *IOSDevice) GetCurrentWindow() (WindowInfo, error) {
return WindowInfo{}, nil
}
func (dev *IOSDevice) GetPackageInfo(packageName string) (AppInfo, error) {
svc, err := installationproxy.New(dev.d)
if err != nil {
return AppInfo{}, errors.Wrap(code.DeviceGetInfoError, err.Error())
}
defer svc.Close()
apps, err := svc.BrowseAllApps()
if err != nil {
return AppInfo{}, errors.Wrap(code.DeviceGetInfoError, err.Error())
}
for _, app := range apps {
if app.CFBundleIdentifier != packageName {
continue
}
appInfo := AppInfo{
Name: app.CFBundleName,
AppBaseInfo: AppBaseInfo{
BundleId: app.CFBundleIdentifier,
PackageName: app.CFBundleIdentifier,
VersionName: app.CFBundleShortVersionString,
VersionCode: app.CFBundleVersion,
AppName: app.CFBundleDisplayName,
AppPath: app.Path,
},
}
log.Info().Interface("appInfo", appInfo).Msg("get package info")
return appInfo, nil
}
return AppInfo{}, errors.Wrap(code.DeviceAppNotInstalled,
fmt.Sprintf("%s not found", packageName))
}

544
pkg/uixt/ios_stub_driver.go Normal file
View File

@@ -0,0 +1,544 @@
package uixt
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/rs/zerolog/log"
)
type stubIOSDriver struct {
bightInsightPrefix string
serverPrefix string
timeout time.Duration
Driver
*wdaDriver
device *IOSDevice
}
func newStubIOSDriver(bightInsightAddr, serverAddr string, dev *IOSDevice, readTimeout ...time.Duration) (*stubIOSDriver, error) {
timeout := 10 * time.Second
if len(readTimeout) > 0 {
timeout = readTimeout[0]
}
driver := new(stubIOSDriver)
driver.device = dev
driver.bightInsightPrefix = bightInsightAddr
driver.serverPrefix = serverAddr
driver.timeout = timeout
driver.Driver.client = &http.Client{
Timeout: time.Second * 10, // 设置超时时间为 10 秒
}
return driver, nil
}
func (s *stubIOSDriver) setUpWda() (err error) {
if s.wdaDriver == nil {
capabilities := NewCapabilities()
capabilities.WithDefaultAlertAction(AlertActionAccept)
driver, err := s.device.NewHTTPDriver(capabilities)
if err != nil {
log.Error().Err(err).Msg("stub driver failed to init wda driver")
return err
}
s.wdaDriver = driver.(*wdaDriver)
}
return nil
}
// NewSession starts a new session and returns the SessionInfo.
func (s *stubIOSDriver) NewSession(capabilities Capabilities) (SessionInfo, error) {
err := s.setUpWda()
if err != nil {
return SessionInfo{}, err
}
return s.wdaDriver.NewSession(capabilities)
}
// DeleteSession Kills application associated with that session and removes session
// 1. alertsMonitor disable
// 2. testedApplicationBundleId terminate
func (s *stubIOSDriver) DeleteSession() error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.DeleteSession()
}
func (s *stubIOSDriver) Status() (DeviceStatus, error) {
err := s.setUpWda()
if err != nil {
return DeviceStatus{}, err
}
return s.wdaDriver.Status()
}
func (s *stubIOSDriver) DeviceInfo() (DeviceInfo, error) {
err := s.setUpWda()
if err != nil {
return DeviceInfo{}, err
}
return s.wdaDriver.DeviceInfo()
}
func (s *stubIOSDriver) Location() (Location, error) {
err := s.setUpWda()
if err != nil {
return Location{}, err
}
return s.wdaDriver.Location()
}
func (s *stubIOSDriver) BatteryInfo() (BatteryInfo, error) {
err := s.setUpWda()
if err != nil {
return BatteryInfo{}, err
}
return s.wdaDriver.BatteryInfo()
}
// WindowSize Return the width and height in portrait mode.
// when getting the window size in wda/ui2/adb, if the device is in landscape mode,
// the width and height will be reversed.
func (s *stubIOSDriver) WindowSize() (Size, error) {
err := s.setUpWda()
if err != nil {
return Size{}, err
}
return s.wdaDriver.WindowSize()
}
func (s *stubIOSDriver) Screen() (Screen, error) {
err := s.setUpWda()
if err != nil {
return Screen{}, err
}
return s.wdaDriver.Screen()
}
func (s *stubIOSDriver) Scale() (float64, error) {
err := s.setUpWda()
if err != nil {
return 0, err
}
return s.wdaDriver.Scale()
}
// GetTimestamp returns the timestamp of the mobile device
func (s *stubIOSDriver) GetTimestamp() (timestamp int64, err error) {
err = s.setUpWda()
if err != nil {
return 0, err
}
return s.wdaDriver.GetTimestamp()
}
// Homescreen Forces the device under test to switch to the home screen
func (s *stubIOSDriver) Homescreen() error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Homescreen()
}
func (s *stubIOSDriver) Unlock() (err error) {
err = s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Unlock()
}
// AppLaunch Launch an application with given bundle identifier in scope of current session.
// !This method is only available since Xcode9 SDK
func (s *stubIOSDriver) AppLaunch(packageName string) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.AppLaunch(packageName)
}
// AppTerminate Terminate an application with the given package name.
// Either `true` if the app has been successfully terminated or `false` if it was not running
func (s *stubIOSDriver) AppTerminate(packageName string) (bool, error) {
err := s.setUpWda()
if err != nil {
return false, err
}
return s.wdaDriver.AppTerminate(packageName)
}
// GetForegroundApp returns current foreground app package name and activity name
func (s *stubIOSDriver) GetForegroundApp() (app AppInfo, err error) {
err = s.setUpWda()
if err != nil {
return AppInfo{}, err
}
return s.wdaDriver.GetForegroundApp()
}
// AssertForegroundApp returns nil if the given package and activity are in foreground
func (s *stubIOSDriver) AssertForegroundApp(packageName string, activityType ...string) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.AssertForegroundApp(packageName, activityType...)
}
// StartCamera Starts a new camera for recording
func (s *stubIOSDriver) StartCamera() error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.StartCamera()
}
// StopCamera Stops the camera for recording
func (s *stubIOSDriver) StopCamera() error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.StopCamera()
}
func (s *stubIOSDriver) Orientation() (orientation Orientation, err error) {
err = s.setUpWda()
if err != nil {
return OrientationPortrait, err
}
return s.wdaDriver.Orientation()
}
// Tap Sends a tap event at the coordinate.
func (s *stubIOSDriver) Tap(x, y float64, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Tap(x, y, options...)
}
// DoubleTap Sends a double tap event at the coordinate.
func (s *stubIOSDriver) DoubleTap(x, y float64, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.DoubleTap(x, y, options...)
}
// TouchAndHold Initiates a long-press gesture at the coordinate, holding for the specified duration.
//
// second: The default value is 1
func (s *stubIOSDriver) TouchAndHold(x, y float64, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.TouchAndHold(x, y, options...)
}
// Drag Initiates a press-and-hold gesture at the coordinate, then drags to another coordinate.
// WithPressDurationOption option can be used to set pressForDuration (default to 1 second).
func (s *stubIOSDriver) Drag(fromX, fromY, toX, toY float64, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Drag(fromX, fromY, toX, toY, options...)
}
// SetPasteboard Sets data to the general pasteboard
func (s *stubIOSDriver) SetPasteboard(contentType PasteboardType, content string) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.SetPasteboard(contentType, content)
}
// GetPasteboard Gets the data contained in the general pasteboard.
//
// It worked when `WDA` was foreground. https://github.com/appium/WebDriverAgent/issues/330
func (s *stubIOSDriver) GetPasteboard(contentType PasteboardType) (raw *bytes.Buffer, err error) {
err = s.setUpWda()
if err != nil {
return nil, err
}
return s.wdaDriver.GetPasteboard(contentType)
}
func (s *stubIOSDriver) SetIme(ime string) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.SetIme(ime)
}
// SendKeys Types a string into active element. There must be element with keyboard focus,
// otherwise an error is raised.
// WithFrequency option can be used to set frequency of typing (letters per sec). The default value is 60
func (s *stubIOSDriver) SendKeys(text string, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.SendKeys(text, options...)
}
// Input works like SendKeys
func (s *stubIOSDriver) Input(text string, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Input(text, options...)
}
func (s *stubIOSDriver) Clear(packageName string) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.Clear(packageName)
}
// PressButton Presses the corresponding hardware button on the device
func (s *stubIOSDriver) PressButton(devBtn DeviceButton) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.PressButton(devBtn)
}
// PressBack Presses the back button
func (s *stubIOSDriver) PressBack(options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.PressBack(options...)
}
func (s *stubIOSDriver) PressKeyCode(keyCode KeyCode) (err error) {
err = s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.PressKeyCode(keyCode)
}
func (s *stubIOSDriver) Screenshot() (*bytes.Buffer, error) {
err := s.setUpWda()
if err != nil {
return nil, err
}
return s.wdaDriver.Screenshot()
//screenshotService, err := instruments.NewScreenshotService(s.device.d)
//if err != nil {
// log.Error().Err(err).Msg("Starting screenshot service failed")
// return nil, err
//}
//defer screenshotService.Close()
//
//imageBytes, err := screenshotService.TakeScreenshot()
//if err != nil {
// log.Error().Err(err).Msg("failed to task screenshot")
// return nil, err
//}
//return bytes.NewBuffer(imageBytes), nil
}
func (s *stubIOSDriver) TapByText(text string, options ...ActionOption) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.TapByText(text, options...)
}
func (s *stubIOSDriver) TapByTexts(actions ...TapTextAction) error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.TapByTexts(actions...)
}
// AccessibleSource Return application elements accessibility tree
func (s *stubIOSDriver) AccessibleSource() (string, error) {
err := s.setUpWda()
if err != nil {
return "", err
}
return s.wdaDriver.AccessibleSource()
}
// HealthCheck Health check might modify simulator state so it should only be called in-between testing sessions
//
// Checks health of XCTest by:
// 1) Querying application for some elements,
// 2) Triggering some device events.
func (s *stubIOSDriver) HealthCheck() error {
err := s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.HealthCheck()
}
func (s *stubIOSDriver) GetAppiumSettings() (map[string]interface{}, error) {
err := s.setUpWda()
if err != nil {
return nil, err
}
return s.wdaDriver.GetAppiumSettings()
}
func (s *stubIOSDriver) SetAppiumSettings(settings map[string]interface{}) (map[string]interface{}, error) {
err := s.setUpWda()
if err != nil {
return nil, err
}
return s.wdaDriver.SetAppiumSettings(settings)
}
func (s *stubIOSDriver) IsHealthy() (bool, error) {
err := s.setUpWda()
if err != nil {
return false, err
}
return s.wdaDriver.IsHealthy()
}
// triggers the log capture and returns the log entries
func (s *stubIOSDriver) StartCaptureLog(identifier ...string) (err error) {
err = s.setUpWda()
if err != nil {
return err
}
return s.wdaDriver.StartCaptureLog(identifier...)
}
func (s *stubIOSDriver) StopCaptureLog() (result interface{}, err error) {
err = s.setUpWda()
if err != nil {
return nil, err
}
return s.wdaDriver.StopCaptureLog()
}
func (s *stubIOSDriver) GetDriverResults() []*DriverResult {
err := s.setUpWda()
if err != nil {
return nil
}
return s.wdaDriver.GetDriverResults()
}
func (s *stubIOSDriver) Source(srcOpt ...SourceOption) (string, error) {
resp, err := s.Driver.Request(http.MethodGet, fmt.Sprintf("%s/source?format=json&onlyWeb=false", s.bightInsightPrefix), []byte{})
if err != nil {
return "", err
}
return string(resp), nil
}
func (s *stubIOSDriver) LoginNoneUI(packageName, phoneNumber string, captcha, password string) (info AppLoginInfo, err error) {
params := map[string]interface{}{
"phone": phoneNumber,
}
if captcha != "" {
params["captcha"] = captcha
} else if password != "" {
params["password"] = password
} else {
return info, fmt.Errorf("password and capcha is empty")
}
bsJSON, err := json.Marshal(params)
if err != nil {
return info, err
}
resp, err := s.Driver.Request(http.MethodPost, fmt.Sprintf("%s/host/login/account/", s.serverPrefix), bsJSON)
if err != nil {
return info, err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return info, err
}
log.Info().Msgf("%v", res)
// {'isSuccess': True, 'data': '登录成功', 'code': 0}
if res["isSuccess"] != true {
err = fmt.Errorf("falied to logout %s", res["data"])
log.Err(err).Msgf("%v", res)
return info, err
}
time.Sleep(20 * time.Second)
info, err = s.getLoginAppInfo(packageName)
if err != nil || !info.IsLogin {
return info, fmt.Errorf("falied to login %v", info)
}
return info, nil
}
func (s *stubIOSDriver) LogoutNoneUI(packageName string) error {
resp, err := s.Driver.Request(http.MethodGet, fmt.Sprintf("%s/host/loginout/", s.serverPrefix), []byte{})
if err != nil {
return err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return err
}
log.Info().Msgf("%v", res)
if res["isSuccess"] != true {
err = fmt.Errorf("falied to logout %s", res["data"])
log.Err(err).Msgf("%v", res)
return err
}
time.Sleep(10 * time.Second)
return nil
}
func (s *stubIOSDriver) TearDown() error {
s.Driver.client.CloseIdleConnections()
return nil
}
func (s *stubIOSDriver) getLoginAppInfo(packageName string) (info AppLoginInfo, err error) {
resp, err := s.Driver.Request(http.MethodGet, fmt.Sprintf("%s/host/app/info/", s.serverPrefix), []byte{})
if err != nil {
return info, err
}
res, err := resp.valueConvertToJsonObject()
if err != nil {
return info, err
}
log.Info().Msgf("%v", res)
if res["isSuccess"] != true {
err = fmt.Errorf("falied to get is login %s", res["data"])
log.Err(err).Msgf("%v", res)
return info, err
}
err = json.Unmarshal([]byte(res["data"].(string)), &info)
if err != nil {
return info, err
}
return info, nil
}
func (s *stubIOSDriver) GetSession() *DriverSession {
return &s.Driver.session
}

105
pkg/uixt/ios_stub_driver_test.go Normal file
View File

@@ -0,0 +1,105 @@
package uixt
import (
"fmt"
"net"
"os"
"testing"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
var (
iOSStubDriver IWebDriver
iOSDevice *IOSDevice
)
func setupiOSStubDriver(t *testing.T) {
var err error
iOSDevice, err = NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800), WithIOSStub(false))
checkErr(t, err)
iOSStubDriver, err = iOSDevice.NewStubDriver()
checkErr(t, err)
}
func TestIOSLogin(t *testing.T) {
setupiOSStubDriver(t)
info, err := iOSStubDriver.LoginNoneUI("", "12342316231", "8517", "")
checkErr(t, err)
t.Log(info)
}
func TestIOSLogout(t *testing.T) {
setupiOSStubDriver(t)
err := iOSStubDriver.LogoutNoneUI("")
checkErr(t, err)
}
func TestIOSIsLogin(t *testing.T) {
setupiOSStubDriver(t)
err := iOSStubDriver.LogoutNoneUI("")
checkErr(t, err)
}
func TestIOSSource(t *testing.T) {
setupiOSStubDriver(t)
source, err := iOSStubDriver.Source()
checkErr(t, err)
t.Log(source)
}
func TestIOSForeground(t *testing.T) {
setupiOSStubDriver(t)
app, err := iOSStubDriver.GetForegroundApp()
checkErr(t, err)
t.Log(app)
}
func TestIOSSwipe(t *testing.T) {
setupiOSStubDriver(t)
iOSStubDriver.Swipe(540, 0, 540, 1000)
}
func TestIOSSave(t *testing.T) {
setupiOSStubDriver(t)
raw, err := iOSStubDriver.Screenshot()
if err != nil {
t.Fatal(err)
}
source, err := iOSStubDriver.Source()
if err != nil {
t.Fatal(err)
}
step := 7
file, err := os.Create(fmt.Sprintf("/Users/bytedance/workcode/wings_algorithm/testcases/data/cases/ios/4159417_cvcn02okg4g0/%d.jpg", step))
if err != nil {
t.Fatal(err)
}
file.Write(raw.Bytes())
file, err = os.Create(fmt.Sprintf("/Users/bytedance/workcode/wings_algorithm/testcases/data/cases/ios/4159417_cvcn02okg4g0/%d.json", step))
if err != nil {
t.Fatal(err)
}
file.Write([]byte(source))
}
func TestListen(t *testing.T) {
setupiOSStubDriver(t)
localPort, err := builtin.GetFreePort()
if err != nil {
t.Fatal(err)
}
err = iOSDevice.forward(localPort, 8800)
if err != nil {
t.Fatal(err)
}
addr := fmt.Sprintf("0.0.0.0:%d", localPort)
l, err := net.Listen("tcp", addr)
if err == nil {
l.Close() // 端口成功绑定后立即释放，返回该端口号
} else {
t.Fatal(err)
}
}

474
pkg/uixt/ios_test.go Normal file
View File

@@ -0,0 +1,474 @@
//go:build localtest
package uixt
import (
"bytes"
"fmt"
"testing"
"time"
"github.com/rs/zerolog/log"
)
var (
bundleId = "com.apple.Preferences"
driver IWebDriver
iOSDriverExt *DriverExt
)
func setup(t *testing.T) {
device, err := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800), WithWDALogOn(true))
if err != nil {
t.Fatal(err)
}
capabilities := NewCapabilities()
capabilities.WithDefaultAlertAction(AlertActionAccept)
driver, err = device.NewHTTPDriver(capabilities)
if err != nil {
t.Fatal(err)
}
iOSDriverExt, err = newDriverExt(device, driver)
if err != nil {
t.Fatal(err)
}
}
func TestViaUSB(t *testing.T) {
setup(t)
t.Log(driver.Status())
}
func TestInstall(t *testing.T) {
setup(t)
err := iOSDriverExt.Install("/Users/bytedance/Downloads/com.yueyou.cyreader_1387717110_7.54.20.ipa", WithRetryTimes(5))
log.Error().Err(err)
if err != nil {
t.Fatal(err)
}
}
func TestNewIOSDevice(t *testing.T) {
device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
if device != nil {
t.Log(device)
}
device, _ = NewIOSDevice(WithUDID("xxxx"))
if device != nil {
t.Log(device)
}
device, _ = NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800))
if device != nil {
t.Log(device)
}
device, _ = NewIOSDevice(WithUDID("xxxx"), WithWDAPort(8700), WithWDAMjpegPort(8800))
if device != nil {
t.Log(device)
}
}
func TestIOSDevice_GetPackageInfo(t *testing.T) {
device, err := NewIOSDevice(WithWDAPort(8700))
checkErr(t, err)
appInfo, err := device.GetPackageInfo("com.ss.iphone.ugc.Aweme")
checkErr(t, err)
t.Log(appInfo)
}
func TestNewWDAHTTPDriver(t *testing.T) {
device, _ := NewIOSDevice()
var err error
_, err = device.NewHTTPDriver(nil)
if err != nil {
t.Fatal(err)
}
}
func TestNewUSBDriver(t *testing.T) {
setup(t)
// t.Log(driver.IsWdaHealthy())
}
func TestDriver_DeviceScaleRatio(t *testing.T) {
setup(t)
scaleRatio, err := driver.Scale()
if err != nil {
t.Fatal(err)
}
t.Log(scaleRatio)
}
func Test_remoteWD_DeleteSession(t *testing.T) {
setup(t)
err := driver.DeleteSession()
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_HealthCheck(t *testing.T) {
setup(t)
err := driver.HealthCheck()
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_GetAppiumSettings(t *testing.T) {
setup(t)
settings, err := driver.GetAppiumSettings()
if err != nil {
t.Fatal(err)
}
t.Log(settings)
}
func Test_remoteWD_SetAppiumSettings(t *testing.T) {
setup(t)
const _acceptAlertButtonSelector = "**/XCUIElementTypeButton[`label IN {'允许','好','仅在使用应用期间','暂不'}`]"
const _dismissAlertButtonSelector = "**/XCUIElementTypeButton[`label IN {'不允许','暂不'}`]"
key := "acceptAlertButtonSelector"
value := _acceptAlertButtonSelector
// settings, err := driver.SetAppiumSettings(map[string]interface{}{"dismissAlertButtonSelector": "暂不"})
settings, err := driver.SetAppiumSettings(map[string]interface{}{key: value})
if err != nil {
t.Fatal(err)
}
if settings[key] != value {
t.Fatal(settings[key])
}
}
func Test_remoteWD_IsWdaHealthy(t *testing.T) {
setup(t)
healthy, err := driver.IsHealthy()
if err != nil {
t.Fatal(err)
}
if healthy == false {
t.Fatal("healthy =", healthy)
}
}
// func Test_remoteWD_WdaShutdown(t *testing.T) {
// setup(t)
//
// if err := driver.WdaShutdown(); err != nil {
// t.Fatal(err)
// }
// }
func Test_remoteWD_Status(t *testing.T) {
setup(t)
status, err := driver.Status()
if err != nil {
t.Fatal(err)
}
if status.Ready == false {
t.Fatal("deviceStatus =", status)
}
}
func Test_remoteWD_DeviceInfo(t *testing.T) {
setup(t)
info, err := driver.DeviceInfo()
if err != nil {
t.Fatal(err)
}
if len(info.Model) == 0 {
t.Fatal(info)
}
}
func Test_remoteWD_BatteryInfo(t *testing.T) {
setup(t)
batteryInfo, err := driver.BatteryInfo()
if err != nil {
t.Fatal()
}
t.Log(batteryInfo)
}
func Test_remoteWD_WindowSize(t *testing.T) {
setup(t)
size, err := driver.WindowSize()
if err != nil {
t.Fatal()
}
t.Log(size)
}
func Test_remoteWD_Screen(t *testing.T) {
setup(t)
screen, err := driver.Screen()
if err != nil {
t.Fatal(err)
}
t.Log(screen)
}
func Test_remoteWD_Homescreen(t *testing.T) {
setup(t)
err := driver.Homescreen()
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_AppLaunch(t *testing.T) {
setup(t)
err := driver.AppLaunch(bundleId)
// err := driver.AppLaunch(bundleId, NewAppLaunchOption().WithShouldWaitForQuiescence(true))
// err := driver.AppLaunch(bundleId, NewAppLaunchOption().WithArguments([]string{"-AppleLanguages", "(Russian)"}))
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_AppTerminate(t *testing.T) {
setup(t)
_, err := driver.AppTerminate(bundleId)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_Tap(t *testing.T) {
setup(t)
err := driver.Tap(200, 300)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_DoubleTap(t *testing.T) {
setup(t)
err := driver.DoubleTap(200, 300)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_TouchAndHold(t *testing.T) {
setup(t)
// err := driver.TouchAndHold(200, 300)
err := driver.TouchAndHold(200, 300)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_Drag(t *testing.T) {
setup(t)
// err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5))
err := driver.Drag(200, 300, 200, 500, WithPressDuration(2), WithDuration(3))
if err != nil {
t.Fatal(err)
}
}
func Test_Relative_Drag(t *testing.T) {
setup(t)
// err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5))
err := iOSDriverExt.SwipeRelative(0.5, 0.7, 0.5, 0.5)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_SetPasteboard(t *testing.T) {
setup(t)
// err := driver.SetPasteboard(PasteboardTypePlaintext, "gwda")
err := driver.SetPasteboard(PasteboardTypeUrl, "Clock-stopwatch://")
// userHomeDir, _ := os.UserHomeDir()
// bytesImg, _ := ioutil.ReadFile(userHomeDir + "/Pictures/IMG_0806.jpg")
// err := driver.SetPasteboard(PasteboardTypeImage, string(bytesImg))
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_GetPasteboard(t *testing.T) {
setup(t)
var buffer *bytes.Buffer
var err error
buffer, err = driver.GetPasteboard(PasteboardTypePlaintext)
// buffer, err = driver.GetPasteboard(PasteboardTypeUrl)
if err != nil {
t.Fatal(err)
}
t.Log(buffer.String())
// buffer, err = driver.GetPasteboard(PasteboardTypeImage)
// if err != nil {
// t.Fatal(err)
// }
// userHomeDir, _ := os.UserHomeDir()
// if err = ioutil.WriteFile(userHomeDir+"/Desktop/p1.png", buffer.Bytes(), 0600); err != nil {
// t.Error(err)
// }
}
func Test_remoteWD_SendKeys(t *testing.T) {
setup(t)
// driver.StartCaptureLog("hrp_wda_log")
err := driver.SendKeys("test", WithIdentifier("test"))
// result, _ := driver.StopCaptureLog()
// err := driver.SendKeys("App Store", WithFrequency(3))
if err != nil {
t.Fatal(err)
}
// t.Log(result)
}
func Test_remoteWD_PressButton(t *testing.T) {
setup(t)
err := driver.PressButton(DeviceButtonVolumeUp)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second * 1)
err = driver.PressButton(DeviceButtonVolumeDown)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second * 1)
err = driver.PressButton(DeviceButtonHome)
if err != nil {
t.Fatal(err)
}
}
func Test_remoteWD_Screenshot(t *testing.T) {
setup(t)
screenshot, err := driver.Screenshot()
if err != nil {
t.Fatal(err)
}
_ = screenshot
// img, format, err := image.Decode(screenshot)
// if err != nil {
// t.Fatal(err)
// }
// userHomeDir, _ := os.UserHomeDir()
// file, err := os.Create(userHomeDir + "/Desktop/s1." + format)
// if err != nil {
// t.Fatal(err)
// }
// defer func() { _ = file.Close() }()
// switch format {
// case "png":
// err = png.Encode(file, img)
// case "jpeg":
// err = jpeg.Encode(file, img, nil)
// }
// if err != nil {
// t.Fatal(err)
// }
// t.Log(file.Name())
}
func Test_remoteWD_Source(t *testing.T) {
setup(t)
var source string
var err error
// source, err = driver.Source()
// if err != nil {
// t.Fatal(err)
// }
source, err = driver.Source()
if err != nil {
t.Fatal(err)
}
// source, err = driver.Source(NewSourceOption().WithFormatAsJson())
// if err != nil {
// t.Fatal(err)
// }
// source, err = driver.Source(NewSourceOption().WithFormatAsDescription())
// if err != nil {
// t.Fatal(err)
// }
// source, err = driver.Source(NewSourceOption().WithFormatAsXml().WithExcludedAttributes([]string{"label", "type", "index"}))
// if err != nil {
// t.Fatal(err)
// }
_ = source
fmt.Println(source)
}
func TestGetForegroundApp(t *testing.T) {
setup(t)
app, err := driver.GetForegroundApp()
if err != nil {
t.Fatal(err)
}
t.Log(app)
}
func Test_remoteWD_AccessibleSource(t *testing.T) {
setup(t)
source, err := driver.AccessibleSource()
if err != nil {
t.Fatal(err)
}
_ = source
fmt.Println(source)
}
func TestRecord(t *testing.T) {
setup(t)
path, err := driver.(*wdaDriver).RecordScreen("", 5*time.Second)
if err != nil {
t.Fatal(err)
}
println(path)
}
// func Test_Backspace(t *testing.T) {
// setup(t)
// err := driver.Backspace(3)
// if err != nil {
// t.Fatal(err)
// }
// }

1107
pkg/uixt/ios_wda_driver.go Normal file
View File

File diff suppressed because it is too large Load Diff

136
pkg/uixt/live_e2e.go Normal file
View File

@@ -0,0 +1,136 @@
package uixt
import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/rs/zerolog/log"
)
type timeLog struct {
UTCTimeStr string `json:"utc_time_str"`
UTCTime int64 `json:"utc_time"`
LiveTimeStr string `json:"live_time_str"`
LiveTime int64 `json:"live_time"`
Delay float64 `json:"delay"`
}
type EndToEndDelay struct {
driver *DriverExt
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
Interval int `json:"interval"` // seconds
Duration int `json:"duration"` // seconds
Timelines []timeLog `json:"timelines"`
}
func CollectEndToEndDelay(dExt *DriverExt, options ...ActionOption) {
dataOptions := NewActionOptions(options...)
startTime := time.Now()
if dataOptions.Interval == 0 {
dataOptions.Interval = 5
}
if dataOptions.Timeout == 0 {
dataOptions.Timeout = 60
}
endToEndDelay := &EndToEndDelay{
driver: dExt,
Duration: int(dataOptions.Timeout),
Interval: int(dataOptions.Interval),
StartTime: startTime.Format("2006-01-02 15:04:05"),
}
endToEndDelay.Start()
// TODO: remove
dExt.Driver.GetSession().e2eDelay = endToEndDelay.Timelines
}
func (ete *EndToEndDelay) getCurrentLiveTime(utcTime time.Time) error {
utcTimeStr := utcTime.Format("2006-01-02 15:04:05")
ocrTexts, err := ete.driver.GetScreenTexts()
if err != nil {
log.Error().Err(err).Msg("get ocr texts failed")
return err
}
// filter ocr texts with time format
var liveTimeTexts []string
for _, ocrText := range ocrTexts {
if len(ocrText.Text) < 13 || strings.Contains(ocrText.Text, ":") {
continue
}
// exclude digit(s) recognized as letter(s)
_, errParseInt := strconv.ParseInt(ocrText.Text[:13], 10, 64)
if errParseInt != nil {
continue
}
liveTimeTexts = append(liveTimeTexts, ocrText.Text)
}
var liveTimeText string
if len(liveTimeTexts) != 0 {
liveTimeText = liveTimeTexts[0]
} else {
log.Warn().Msg("no time text found")
return nil
}
liveTimeInt, err := strconv.Atoi(liveTimeText)
if err != nil {
liveTimeInt = 0
}
liveTimeSInt, err := strconv.Atoi(liveTimeText[:10])
if err != nil {
liveTimeSInt = 0
}
liveTimeNSInt, err := strconv.Atoi(liveTimeText[10:13])
if err != nil {
liveTimeNSInt = 0
}
liveTimeStr := time.Unix(int64(liveTimeSInt), int64(liveTimeNSInt*1000*1000)).Format("2006-01-02 15:04:05")
log.Info().
Str("utcTime", utcTimeStr).
Int64("utcTimeInt", utcTime.UnixMilli()).
Str("liveTime", liveTimeStr).
Int64("liveTimeInt", int64(liveTimeInt)).
Float64("delay", float64(utcTime.UnixMilli()-int64(liveTimeInt))/1000).
Msg("log live time")
ete.Timelines = append(ete.Timelines, timeLog{
UTCTimeStr: utcTimeStr,
UTCTime: utcTime.UnixMilli(),
LiveTimeStr: liveTimeStr,
LiveTime: int64(liveTimeInt),
Delay: float64(utcTime.UnixMilli()-int64(liveTimeInt)) / 1000,
})
return nil
}
func (ete *EndToEndDelay) Start() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
timer := time.NewTimer(time.Duration(ete.Duration) * time.Second)
for {
select {
case <-timer.C:
ete.EndTime = time.Now().Format("2006-01-02 15:04:05")
return
case <-c:
ete.EndTime = time.Now().Format("2006-01-02 15:04:05")
return
default:
utcTime := time.Now()
if utcTime.Unix()%int64(ete.Interval) == 0 {
_ = ete.getCurrentLiveTime(utcTime)
} else {
time.Sleep(500 * time.Millisecond)
}
}
}
}

148
pkg/uixt/popups.go Normal file
View File

@@ -0,0 +1,148 @@
package uixt
import (
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
)
// TODO: add more popup texts
var popups = [][]string{
{".*青少年.*", "我知道了"}, // 青少年弹窗
{".*个人信息保护.*", "同意"},
{".*通讯录.*", "拒绝"},
{".*更新.*", "以后再说|稍后|取消"},
{".*升级.*", "以后再说|稍后|取消"},
{".*定位.*", "仅.*允许"},
{".*拍照.*", "仅.*允许"},
{".*录音.*", "仅.*允许"},
{".*位置.*", "仅.*允许"},
{".*权限.*", "仅.*允许|始终允许"},
{".*允许.*", "仅.*允许|始终允许"},
{".*风险.*", "继续使用"},
{"管理使用时间", ".*忽略.*"},
}
func findTextPopup(screenTexts OCRTexts) (closePoint *OCRText) {
for _, popup := range popups {
if len(popup) != 2 {
continue
}
points, err := screenTexts.FindTexts([]string{popup[0], popup[1]}, WithRegex(true))
if err == nil {
log.Warn().Interface("popup", popup).
Interface("texts", screenTexts).Msg("text popup found")
closePoint = &points[1]
break
}
}
return
}
func (dExt *DriverExt) handleTextPopup(screenTexts OCRTexts) error {
closePoint := findTextPopup(screenTexts)
if closePoint == nil {
// no popup found
return nil
}
log.Info().Str("text", closePoint.Text).Msg("close popup")
pointCenter := closePoint.Center()
if err := dExt.TapAbsXY(pointCenter.X, pointCenter.Y); err != nil {
log.Error().Err(err).Msg("tap popup failed")
return errors.Wrap(code.MobileUIPopupError, err.Error())
}
// tap popup success
return nil
}
func (dExt *DriverExt) AutoPopupHandler() error {
// TODO: check popup by activity type
// check popup by screenshot
screenResult, err := dExt.GetScreenResult(
WithScreenShotOCR(true),
WithScreenShotUpload(true),
WithScreenShotFileName("check_popup"),
)
if err != nil {
return errors.Wrap(err, "get screen result failed for popup handler")
}
return dExt.handleTextPopup(screenResult.Texts)
}
type PopupInfo struct {
*ClosePopupsResult
ClosePoints []PointF `json:"close_points,omitempty"` // CV 识别的所有关闭按钮（仅关闭按钮，可能存在多个）
PicName string `json:"pic_name"`
PicURL string `json:"pic_url"`
}
func (p *PopupInfo) ClosePoint() *PointF {
closeResult := p.ClosePopupsResult
if closeResult == nil {
return nil
}
// 弹窗关闭按钮不存在
if closeResult.CloseArea.IsEmpty() {
return nil
}
closePoint := closeResult.CloseArea.Center()
return &closePoint
}
func (dExt *DriverExt) CheckPopup() (popup *PopupInfo, err error) {
screenResult, err := dExt.GetScreenResult(
WithScreenShotUpload(true),
WithScreenShotClosePopups(true), // get popup area and close area
WithScreenShotFileName("check_popup"),
)
if err != nil {
return nil, errors.Wrap(err, "get screen result failed for popup handler")
}
popup = screenResult.Popup
if popup == nil {
// popup not found
log.Debug().Msg("check popup, no found")
return nil, nil
}
closePoint := popup.ClosePoint()
if closePoint == nil {
// close point not found
return nil, errors.Wrap(code.MobileUIPopupError, "popup close point not found")
}
log.Info().Interface("popup", popup).Msg("found popup")
return popup, nil
}
func (dExt *DriverExt) ClosePopupsHandler() (err error) {
log.Info().Msg("try to find and close popups")
popup, err := dExt.CheckPopup()
if err != nil {
// check popup failed
return err
} else if popup == nil {
// no popup found
return nil
}
// found popup
closePoint := popup.ClosePoint()
log.Info().
Interface("closePoint", closePoint).
Interface("popup", popup).
Msg("tap to close popup")
if err := dExt.TapAbsXY(closePoint.X, closePoint.Y); err != nil {
log.Error().Err(err).Msg("tap popup failed")
return errors.Wrap(code.MobileUIPopupError, err.Error())
}
return nil
}

49
pkg/uixt/popups_test.go Normal file
View File

@@ -0,0 +1,49 @@
//go:build localtest
package uixt
import (
"regexp"
"testing"
)
func TestCheckPopup(t *testing.T) {
setupAndroidAdbDriver(t)
popup, err := driverExt.CheckPopup()
if err != nil {
t.Logf("check popup failed, err: %v", err)
} else if popup == nil {
t.Log("no popup found")
} else {
t.Logf("found popup: %v", popup)
}
}
func TestClosePopup(t *testing.T) {
setupAndroidAdbDriver(t)
if err := driverExt.ClosePopupsHandler(); err != nil {
t.Fatal(err)
}
}
func matchPopup(text string) bool {
for _, popup := range popups {
if regexp.MustCompile(popup[1]).MatchString(text) {
return true
}
}
return false
}
func TestMatchRegex(t *testing.T) {
testData := []string{
"以后再说", "我知道了", "同意", "拒绝", "稍后",
"始终允许", "继续使用", "仅在使用中允许",
}
for _, text := range testData {
if !matchPopup(text) {
t.Fatal(text)
}
}
}

274
pkg/uixt/screenshot.go Normal file
View File

@@ -0,0 +1,274 @@
package uixt
import (
"bytes"
"fmt"
"image"
"image/gif"
"image/jpeg"
_ "image/png"
"os"
"path/filepath"
"strings"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/internal/config"
)
type ScreenResult struct {
bufSource *bytes.Buffer // raw image buffer bytes
ImagePath string `json:"image_path"` // image file path
Resolution Size `json:"resolution"`
UploadedURL string `json:"uploaded_url"` // uploaded image url
Texts OCRTexts `json:"texts"` // dumped raw OCRTexts
Icons UIResultMap `json:"icons"` // CV 识别的图标
Tags []string `json:"tags"` // tags for image, e.g. ["feed", "ad", "live"]
Popup *PopupInfo `json:"popup,omitempty"`
}
func (s *ScreenResult) FilterTextsByScope(x1, y1, x2, y2 float64) OCRTexts {
if x1 > 1 || y1 > 1 || x2 > 1 || y2 > 1 {
log.Warn().Msg("x1, y1, x2, y2 should be in percentage, skip filter scope")
return s.Texts
}
return s.Texts.FilterScope(AbsScope{
int(float64(s.Resolution.Width) * x1), int(float64(s.Resolution.Height) * y1),
int(float64(s.Resolution.Width) * x2), int(float64(s.Resolution.Height) * y2),
})
}
// GetScreenResult takes a screenshot, returns the image recognition result
func (dExt *DriverExt) GetScreenResult(options ...ActionOption) (screenResult *ScreenResult, err error) {
actionOptions := NewActionOptions(options...)
if actionOptions.MaxRetryTimes == 0 {
actionOptions.MaxRetryTimes = 1
}
var fileName string
screenshotActions := actionOptions.screenshotActions()
if actionOptions.ScreenShotFileName != "" {
fileName = builtin.GenNameWithTimestamp("%d_" + actionOptions.ScreenShotFileName)
} else if len(screenshotActions) != 0 {
fileName = builtin.GenNameWithTimestamp("%d_" + strings.Join(screenshotActions, "_"))
} else {
fileName = builtin.GenNameWithTimestamp("%d_screenshot")
}
var bufSource *bytes.Buffer
var imageResult *ImageResult
var imagePath string
var windowSize Size
var lastErr error
// get screenshot info with retry
for i := 0; i <= actionOptions.MaxRetryTimes; i++ {
bufSource, imagePath, err = dExt.GetScreenShot(fileName)
if err != nil {
lastErr = err
time.Sleep(time.Second * 1)
continue
}
windowSize, err = dExt.Driver.WindowSize()
if err != nil {
lastErr = errors.Wrap(code.DeviceGetInfoError, err.Error())
continue
}
screenResult = &ScreenResult{
bufSource: bufSource,
ImagePath: imagePath,
Tags: nil,
Resolution: windowSize,
}
imageResult, err = dExt.ImageService.GetImage(bufSource, options...)
if err != nil {
log.Error().Err(err).Msg("GetImage from ImageService failed")
lastErr = err
continue
}
// success, break the loop
lastErr = nil
break
}
if lastErr != nil {
return nil, lastErr
}
// cache screen result
dExt.Driver.GetSession().addScreenResult(screenResult)
if imageResult != nil {
screenResult.Texts = imageResult.OCRResult.ToOCRTexts()
screenResult.UploadedURL = imageResult.URL
screenResult.Icons = imageResult.UIResult
if actionOptions.ScreenShotWithClosePopups && imageResult.ClosePopupsResult != nil {
screenResult.Popup = &PopupInfo{
ClosePopupsResult: imageResult.ClosePopupsResult,
PicName: imagePath,
PicURL: imageResult.URL,
}
closeAreas, _ := imageResult.UIResult.FilterUIResults([]string{"close"})
for _, closeArea := range closeAreas {
screenResult.Popup.ClosePoints = append(screenResult.Popup.ClosePoints, closeArea.Center())
}
}
}
log.Debug().
Str("imagePath", imagePath).
Str("imageUrl", screenResult.UploadedURL).
Msg("log screenshot")
return screenResult, nil
}
func (dExt *DriverExt) GetScreenTexts(options ...ActionOption) (ocrTexts OCRTexts, err error) {
actionOptions := NewActionOptions(options...)
if actionOptions.ScreenShotFileName == "" {
options = append(options, WithScreenShotFileName("get_screen_texts"))
}
options = append(options, WithScreenShotOCR(true), WithScreenShotUpload(true))
screenResult, err := dExt.GetScreenResult(options...)
if err != nil {
return
}
return screenResult.Texts, nil
}
func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...ActionOption) (point PointF, err error) {
// find text using OCR
if !builtin.IsPathExists(search) {
return dExt.FindScreenText(search, options...)
}
// TODO: find image using CV
err = errors.New("ocr text not found")
return
}
func (dExt *DriverExt) FindScreenText(text string, options ...ActionOption) (point PointF, err error) {
actionOptions := NewActionOptions(options...)
if actionOptions.ScreenShotFileName == "" {
options = append(options, WithScreenShotFileName(fmt.Sprintf("find_screen_text_%s", text)))
}
ocrTexts, err := dExt.GetScreenTexts(options...)
if err != nil {
return
}
result, err := ocrTexts.FindText(text, dExt.ParseActionOptions(options...)...)
if err != nil {
log.Warn().Msgf("FindText failed: %s", err.Error())
return
}
point = result.Center()
log.Info().Str("text", text).
Interface("point", point).Msgf("FindScreenText success")
return
}
func (dExt *DriverExt) FindUIResult(options ...ActionOption) (point PointF, err error) {
actionOptions := NewActionOptions(options...)
if actionOptions.ScreenShotFileName == "" {
options = append(options, WithScreenShotFileName(
fmt.Sprintf("find_ui_result_%s", strings.Join(actionOptions.ScreenShotWithUITypes, "_"))))
}
screenResult, err := dExt.GetScreenResult(options...)
if err != nil {
return
}
uiResults, err := screenResult.Icons.FilterUIResults(actionOptions.ScreenShotWithUITypes)
if err != nil {
return
}
uiResult, err := uiResults.GetUIResult(dExt.ParseActionOptions(options...)...)
point = uiResult.Center()
log.Info().Interface("text", actionOptions.ScreenShotWithUITypes).
Interface("point", point).Msg("FindUIResult success")
return
}
// GetScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
func (dExt *DriverExt) GetScreenShot(fileName string) (raw *bytes.Buffer, path string, err error) {
if raw, err = dExt.Driver.Screenshot(); err != nil {
log.Error().Err(err).Msg("capture screenshot data failed")
return nil, "", errors.Wrap(code.DeviceScreenShotError, err.Error())
}
// save screenshot to file
path = filepath.Join(config.ScreenShotsPath, fileName)
path, err = saveScreenShot(raw, path)
if err != nil {
log.Error().Err(err).Msg("save screenshot file failed")
return nil, "", errors.Wrap(code.DeviceScreenShotError,
fmt.Sprintf("save screenshot file failed: %s", err.Error()))
}
return raw, path, nil
}
// saveScreenShot saves compressed image file with file name
func saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) {
// notice: screenshot data is a stream, so we need to copy it to a new buffer
copiedBuffer := &bytes.Buffer{}
if _, err := copiedBuffer.Write(raw.Bytes()); err != nil {
log.Error().Err(err).Msg("copy screenshot buffer failed")
}
img, format, err := image.Decode(copiedBuffer)
if err != nil {
return "", errors.Wrap(err, "decode screenshot image failed")
}
// The default format uses jpeg for compression
screenshotPath := filepath.Join(fmt.Sprintf("%s.%s", fileName, "jpeg"))
file, err := os.Create(screenshotPath)
if err != nil {
return "", errors.Wrap(err, "create screenshot image file failed")
}
defer func() {
_ = file.Close()
}()
// compress image and save to file
switch format {
case "jpeg", "png":
jpegOptions := &jpeg.Options{Quality: 95}
err = jpeg.Encode(file, img, jpegOptions)
// case "png":
// encoder := png.Encoder{
// CompressionLevel: png.BestCompression,
// }
// err = encoder.Encode(file, img)
case "gif":
gifOptions := &gif.Options{
NumColors: 256,
}
err = gif.Encode(file, img, gifOptions)
default:
return "", fmt.Errorf("unsupported image format %s", format)
}
if err != nil {
return "", errors.Wrap(err, "save image file failed")
}
var fileSize int64
fileInfo, err := file.Stat()
if err == nil {
fileSize = fileInfo.Size()
}
log.Info().Str("path", screenshotPath).
Int("rawBytes", raw.Len()).Int64("saveBytes", fileSize).
Msg("save screenshot file success")
return screenshotPath, nil
}

23
pkg/uixt/screenshot_test.go Normal file
View File

@@ -0,0 +1,23 @@
//go:build localtest
package uixt
import (
"testing"
)
func TestGetScreenShot(t *testing.T) {
setupAndroidAdbDriver(t)
fileName := "test_screenshot"
_, path, err := driverExt.GetScreenShot(fileName)
if err != nil {
t.Fatalf("GetScreenShot failed: %v", err)
}
if path == "" {
t.Fatal("screenshot path is empty")
}
t.Logf("screenshot saved at: %s", path)
}

215
pkg/uixt/swipe.go Normal file
View File

@@ -0,0 +1,215 @@
package uixt
import (
"fmt"
"strings"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
)
func assertRelative(p float64) bool {
return p >= 0 && p <= 1
}
// SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY]
func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...ActionOption) error {
if !assertRelative(fromX) || !assertRelative(fromY) ||
!assertRelative(toX) || !assertRelative(toY) {
return errors.Wrap(code.InvalidCaseError,
fmt.Sprintf("fromX(%f), fromY(%f), toX(%f), toY(%f) must be less than 1",
fromX, fromY, toX, toY))
}
windowSize, err := dExt.Driver.WindowSize()
if err != nil {
return errors.Wrap(code.DeviceGetInfoError, err.Error())
}
width := windowSize.Width
height := windowSize.Height
fromX = float64(width) * fromX
fromY = float64(height) * fromY
toX = float64(width) * toX
toY = float64(height) * toY
err = dExt.Driver.Swipe(fromX, fromY, toX, toY, options...)
if err != nil {
return errors.Wrap(code.MobileUISwipeError, err.Error())
}
return nil
}
func (dExt *DriverExt) SwipeUp(options ...ActionOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, options...)
}
func (dExt *DriverExt) SwipeDown(options ...ActionOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, options...)
}
func (dExt *DriverExt) SwipeLeft(options ...ActionOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, options...)
}
func (dExt *DriverExt) SwipeRight(options ...ActionOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, options...)
}
type Action func(driver *DriverExt) error
func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
maxRetryTimes := actionOptions.MaxRetryTimes
interval := actionOptions.Interval
for i := 0; i < maxRetryTimes; i++ {
// wait interval between each findAction
time.Sleep(time.Duration(interval) * time.Second)
if err := findCondition(dExt); err == nil {
// do action after found
return foundAction(dExt)
}
if err := findAction(dExt); err != nil {
log.Error().Err(err).Msgf("find action failed")
}
}
return errors.Wrap(code.LoopActionNotFoundError,
fmt.Sprintf("loop %d times, match find condition failed", maxRetryTimes))
}
func (dExt *DriverExt) prepareSwipeAction(params interface{}, options ...ActionOption) func(d *DriverExt) error {
actionOptions := NewActionOptions(options...)
var swipeDirection interface{}
// priority: params > actionOptions.Direction, default swipe up
if params != nil {
swipeDirection = params
} else if actionOptions.Direction != nil {
swipeDirection = actionOptions.Direction
} else {
swipeDirection = "up" // default swipe up
}
if actionOptions.Steps == 0 {
actionOptions.Steps = 10
}
return func(d *DriverExt) error {
defer func() {
// wait for swipe action to completed and content to load completely
time.Sleep(time.Duration(1000*actionOptions.Interval) * time.Millisecond)
}()
if d, ok := swipeDirection.(string); ok {
// enum direction: up, down, left, right
switch d {
case "up":
return dExt.SwipeUp(options...)
case "down":
return dExt.SwipeDown(options...)
case "left":
return dExt.SwipeLeft(options...)
case "right":
return dExt.SwipeRight(options...)
default:
return errors.Wrap(code.InvalidParamError,
fmt.Sprintf("get unexpected swipe direction: %s", d))
}
} else if params, err := builtin.ConvertToFloat64Slice(swipeDirection); err == nil && len(params) == 4 {
// custom direction: [fromX, fromY, toX, toY]
if err := dExt.SwipeRelative(params[0], params[1], params[2], params[3], options...); err != nil {
log.Error().Err(err).Msgf("swipe from (%v, %v) to (%v, %v) failed",
params[0], params[1], params[2], params[3])
return err
}
} else {
return fmt.Errorf("invalid swipe params %v", swipeDirection)
}
return nil
}
}
func (dExt *DriverExt) swipeToTapTexts(texts []string, options ...ActionOption) error {
if len(texts) == 0 {
return errors.New("no text to tap")
}
options = append(options, WithMatchOne(true), WithRegex(true))
actionOptions := NewActionOptions(options...)
actionOptions.Identifier = ""
optionsWithoutIdentifier := actionOptions.Options()
var point PointF
findTexts := func(d *DriverExt) error {
var err error
screenResult, err := d.GetScreenResult(
WithScreenShotOCR(true),
WithScreenShotUpload(true),
WithScreenShotFileName(
fmt.Sprintf("swipe_to_tap_texts_%s", strings.Join(texts, "_")),
),
)
if err != nil {
return err
}
points, err := screenResult.Texts.FindTexts(texts,
dExt.ParseActionOptions(optionsWithoutIdentifier...)...)
if err != nil {
log.Error().Err(err).Strs("texts", texts).Msg("find texts failed")
return err
}
log.Info().Strs("texts", texts).Interface("results", points).Msg("swipeToTapTexts successful")
// target texts found, pick the first one
point = points[0].Center() // FIXME
return nil
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, options...)
}
findAction := dExt.prepareSwipeAction(nil, optionsWithoutIdentifier...)
return dExt.LoopUntil(findAction, findTexts, foundTextAction, optionsWithoutIdentifier...)
}
func (dExt *DriverExt) swipeToTapApp(appName string, options ...ActionOption) error {
// go to home screen
if err := dExt.Driver.Homescreen(); err != nil {
return errors.Wrap(err, "go to home screen failed")
}
// automatic handling popups before swipe
if err := dExt.ClosePopupsHandler(); err != nil {
log.Error().Err(err).Msg("auto handle popup failed")
}
// swipe to first screen
for i := 0; i < 5; i++ {
dExt.SwipeRight()
}
options = append(options, WithDirection("left"))
actionOptions := NewActionOptions(options...)
// default to retry 5 times
if actionOptions.MaxRetryTimes == 0 {
options = append(options, WithMaxRetryTimes(5))
}
// tap app icon above the text
if len(actionOptions.Offset) == 0 {
options = append(options, WithTapOffset(0, -25))
}
// set default swipe interval to 1 second
if builtin.IsZeroFloat64(actionOptions.Interval) {
options = append(options, WithInterval(1))
}
return dExt.swipeToTapTexts([]string{appName}, options...)
}

36
pkg/uixt/swipe_test.go Normal file
View File

@@ -0,0 +1,36 @@
//go:build localtest
package uixt
import (
"testing"
)
func TestAndroidSwipeAction(t *testing.T) {
setupAndroidAdbDriver(t)
swipeAction := driverExt.prepareSwipeAction("up", WithDirection("down"))
err := swipeAction(driverExt)
checkErr(t, err)
swipeAction = driverExt.prepareSwipeAction("up", WithCustomDirection(0.5, 0.5, 0.5, 0.9))
err = swipeAction(driverExt)
checkErr(t, err)
}
func TestAndroidSwipeToTapApp(t *testing.T) {
setupAndroidAdbDriver(t)
err := driverExt.swipeToTapApp("抖音")
checkErr(t, err)
}
func TestAndroidSwipeToTapTexts(t *testing.T) {
setupAndroidAdbDriver(t)
err := driverExt.Driver.AppLaunch("com.ss.android.ugc.aweme")
checkErr(t, err)
err = driverExt.swipeToTapTexts([]string{"点击进入直播间", "直播中"}, WithDirection("up"))
checkErr(t, err)
}

118
pkg/uixt/tap.go Normal file
View File

@@ -0,0 +1,118 @@
package uixt
import (
"fmt"
"github.com/pkg/errors"
"github.com/httprunner/httprunner/v5/code"
)
func (dExt *DriverExt) TapAbsXY(x, y float64, options ...ActionOption) error {
// tap on absolute coordinate [x, y]
err := dExt.Driver.Tap(x, y, options...)
if err != nil {
return errors.Wrap(code.MobileUITapError, err.Error())
}
return nil
}
func (dExt *DriverExt) TapXY(x, y float64, options ...ActionOption) error {
// tap on [x, y] percent of window size
if x > 1 || y > 1 {
return fmt.Errorf("x, y percentage should be <= 1, got x=%v, y=%v", x, y)
}
windowSize, err := dExt.Driver.WindowSize()
if err != nil {
return err
}
x = x * float64(windowSize.Width)
y = y * float64(windowSize.Height)
return dExt.TapAbsXY(x, y, options...)
}
func (dExt *DriverExt) TapByOCR(ocrText string, options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
if actionOptions.ScreenShotFileName == "" {
options = append(options, WithScreenShotFileName(fmt.Sprintf("tap_by_ocr_%s", ocrText)))
}
point, err := dExt.FindScreenText(ocrText, options...)
if err != nil {
if actionOptions.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(point.X, point.Y, options...)
}
func (dExt *DriverExt) TapByUIDetection(options ...ActionOption) error {
actionOptions := NewActionOptions(options...)
point, err := dExt.FindUIResult(options...)
if err != nil {
if actionOptions.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(point.X, point.Y, options...)
}
func (dExt *DriverExt) Tap(param string, options ...ActionOption) error {
return dExt.TapOffset(param, 0, 0, options...)
}
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...ActionOption) (err error) {
actionOptions := NewActionOptions(options...)
point, err := dExt.FindUIRectInUIKit(param, options...)
if err != nil {
if actionOptions.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(point.X+xOffset, point.Y+yOffset, options...)
}
func (dExt *DriverExt) DoubleTapXY(x, y float64, options ...ActionOption) error {
// double tap on coordinate: [x, y] should be relative
if x > 1 || y > 1 {
return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
}
windowSize, err := dExt.Driver.WindowSize()
if err != nil {
return err
}
x = x * float64(windowSize.Width)
y = y * float64(windowSize.Height)
err = dExt.Driver.DoubleTap(x, y, options...)
if err != nil {
return errors.Wrap(code.MobileUITapError, err.Error())
}
return nil
}
func (dExt *DriverExt) DoubleTap(param string, options ...ActionOption) (err error) {
return dExt.DoubleTapOffset(param, 0, 0, options...)
}
func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64, options ...ActionOption) (err error) {
point, err := dExt.FindUIRectInUIKit(param)
if err != nil {
return err
}
err = dExt.Driver.DoubleTap(point.X+xOffset, point.Y+yOffset, options...)
if err != nil {
return errors.Wrap(code.MobileUITapError, err.Error())
}
return nil
}

38
pkg/uixt/tap_test.go Normal file
View File

@@ -0,0 +1,38 @@
//go:build localtest
package uixt
import (
"testing"
)
var iosDevice *IOSDevice
func init() {
iosDevice, _ = NewIOSDevice()
}
func TestDriverExt_TapXY(t *testing.T) {
driverExt, err := iosDevice.NewDriver()
checkErr(t, err)
err = driverExt.TapXY(0.4, 0.5)
checkErr(t, err)
}
func TestDriverExt_TapAbsXY(t *testing.T) {
driverExt, err := iosDevice.NewDriver()
checkErr(t, err)
err = driverExt.TapAbsXY(100, 300)
checkErr(t, err)
}
func TestDriverExt_TapWithOCR(t *testing.T) {
driverExt, err := iosDevice.NewDriver()
checkErr(t, err)
// 需要点击文字上方的图标
err = driverExt.TapOffset("抖音", 0, -20)
checkErr(t, err)
}