refactor: split action to separate file

This commit is contained in:
lilong.129
2023-04-25 14:47:26 +08:00
parent 556050367e
commit f9a488c7e7
2 changed files with 527 additions and 514 deletions

524
hrp/pkg/uixt/action.go Normal file
View File

@@ -0,0 +1,524 @@
package uixt
import (
"encoding/json"
"fmt"
"math/rand"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
)
type ActionMethod string
const (
ACTION_AppInstall ActionMethod = "install"
ACTION_AppUninstall ActionMethod = "uninstall"
ACTION_AppStart ActionMethod = "app_start"
ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成
ACTION_AppTerminate ActionMethod = "app_terminate"
ACTION_AppStop ActionMethod = "app_stop"
ACTION_ScreenShot ActionMethod = "screenshot"
ACTION_Sleep ActionMethod = "sleep"
ACTION_SleepRandom ActionMethod = "sleep_random"
ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera
ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera
// UI validation
// selectors
SelectorName string = "ui_name"
SelectorLabel string = "ui_label"
SelectorOCR string = "ui_ocr"
SelectorImage string = "ui_image"
SelectorForegroundApp string = "ui_foreground_app"
// assertions
AssertionEqual string = "equal"
AssertionNotEqual string = "not_equal"
AssertionExists string = "exists"
AssertionNotExists string = "not_exists"
// UI handling
ACTION_Home ActionMethod = "home"
ACTION_TapXY ActionMethod = "tap_xy"
ACTION_TapAbsXY ActionMethod = "tap_abs_xy"
ACTION_TapByOCR ActionMethod = "tap_ocr"
ACTION_TapByCV ActionMethod = "tap_cv"
ACTION_Tap ActionMethod = "tap"
ACTION_DoubleTapXY ActionMethod = "double_tap_xy"
ACTION_DoubleTap ActionMethod = "double_tap"
ACTION_Swipe ActionMethod = "swipe"
ACTION_Input ActionMethod = "input"
ACTION_Back ActionMethod = "back"
// custom actions
ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap
ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap
ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap
ACTION_VideoCrawler ActionMethod = "video_crawler"
)
type MobileAction struct {
Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"`
Params interface{} `json:"params,omitempty" yaml:"params,omitempty"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second
Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action
Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app
Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope
Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Text string `json:"text,omitempty" yaml:"text,omitempty"`
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
}
type ActionOption func(o *MobileAction)
func WithIdentifier(identifier string) ActionOption {
return func(o *MobileAction) {
o.Identifier = identifier
}
}
func WithIndex(index int) ActionOption {
return func(o *MobileAction) {
o.Index = index
}
}
func WithWaitTime(sec float64) ActionOption {
return func(o *MobileAction) {
o.WaitTime = sec
}
}
func WithDuration(duration float64) ActionOption {
return func(o *MobileAction) {
o.Duration = duration
}
}
func WithSteps(steps int) ActionOption {
return func(o *MobileAction) {
o.Steps = steps
}
}
// WithDirection inputs direction (up, down, left, right)
func WithDirection(direction string) ActionOption {
return func(o *MobileAction) {
o.Direction = direction
}
}
// WithCustomDirection inputs sx, sy, ex, ey
func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
return func(o *MobileAction) {
o.Direction = []float64{sx, sy, ex, ey}
}
}
// WithScope inputs area of [(x1,y1), (x2,y2)]
func WithScope(x1, y1, x2, y2 float64) ActionOption {
return func(o *MobileAction) {
o.Scope = []float64{x1, y1, x2, y2}
}
}
func WithOffset(offsetX, offsetY int) ActionOption {
return func(o *MobileAction) {
o.Offset = []int{offsetX, offsetY}
}
}
func WithText(text string) ActionOption {
return func(o *MobileAction) {
o.Text = text
}
}
func WithID(id string) ActionOption {
return func(o *MobileAction) {
o.ID = id
}
}
func WithDescription(description string) ActionOption {
return func(o *MobileAction) {
o.Description = description
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *MobileAction) {
o.MaxRetryTimes = maxRetryTimes
}
}
func WithTimeout(timeout int) ActionOption {
return func(o *MobileAction) {
o.Timeout = timeout
}
}
func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
return func(o *MobileAction) {
o.IgnoreNotFoundError = ignoreError
}
}
func (dExt *DriverExt) DoAction(action MobileAction) error {
log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action")
switch action.Method {
case ACTION_AppInstall:
// TODO
return errActionNotImplemented
case ACTION_AppLaunch:
if bundleId, ok := action.Params.(string); ok {
return dExt.Driver.AppLaunch(bundleId)
}
return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
ACTION_AppLaunch, action.Params)
case ACTION_SwipeToTapApp:
if appName, ok := action.Params.(string); ok {
return dExt.swipeToTapApp(appName, action)
}
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
// TODO: merge to LoopUntil
if text, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
identifierOption := WithDataIdentifier(action.Identifier)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
indexOption := WithDataIndex(action.Index)
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
waitTimeOption := WithDataWaitTime(action.WaitTime)
var point PointF
// findTextAction := func(d *DriverExt) error {
// return nil
// }
findTextCondition := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(text, indexOption, scopeOption)
return err
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
}
// swipe until found
return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
case ACTION_SwipeToTapTexts:
// TODO: merge to LoopUntil
if texts, ok := action.Params.([]interface{}); ok {
var textList []string
for _, t := range texts {
textList = append(textList, t.(string))
}
action.Params = textList
}
if texts, ok := action.Params.([]string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
identifierOption := WithDataIdentifier(action.Identifier)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
waitTimeOption := WithDataWaitTime(action.WaitTime)
var point PointF
findTexts := func(d *DriverExt) error {
var err error
points, err := d.GetTextXYs(texts, scopeOption)
if err != nil {
return err
}
for _, point = range points {
if point != (PointF{X: 0, Y: 0}) {
return nil
}
}
return errors.New("failed to find text position")
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
}
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption)
}
// swipe until found
return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption)
}
return fmt.Errorf("invalid %s params, should be app text([]string), got %v",
ACTION_SwipeToTapText, action.Params)
case ACTION_AppTerminate:
if bundleId, ok := action.Params.(string); ok {
success, err := dExt.Driver.AppTerminate(bundleId)
if err != nil {
return errors.Wrap(err, "failed to terminate app")
}
if !success {
log.Warn().Str("bundleId", bundleId).Msg("app was not running")
}
return nil
}
return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params)
case ACTION_Home:
return dExt.Driver.Homescreen()
case ACTION_TapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_TapAbsXY:
if location, ok := action.Params.([]interface{}); ok {
// absolute coordinates x,y of window size: [100, 300]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier), WithDataOffset(action.Offset[0], action.Offset[1]))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
indexOption := WithDataIndex(action.Index)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
identifierOption := WithDataIdentifier(action.Identifier)
IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError)
return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption, offsetOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.DoubleTapXY(x, y)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params)
case ACTION_DoubleTap:
if param, ok := action.Params.(string); ok {
return dExt.DoubleTap(param)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
case ACTION_Swipe:
identifierOption := WithDataIdentifier(action.Identifier)
durationOption := WithDataPressDuration(action.Duration)
if action.Steps == 0 {
action.Steps = 10
}
stepsOption := WithDataSteps(action.Steps)
if positions, ok := action.Params.([]interface{}); ok {
// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
if len(positions) != 4 {
return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions)
}
fromX, _ := positions[0].(float64)
fromY, _ := positions[1].(float64)
toX, _ := positions[2].(float64)
toY, _ := positions[3].(float64)
return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption)
}
if direction, ok := action.Params.(string); ok {
return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
case ACTION_Input:
// input text on current active element
// append \n to send text with enter
// send \b\b\b to delete 3 chars
param := fmt.Sprintf("%v", action.Params)
options := []DataOption{}
if action.Text != "" {
options = append(options, WithCustomOption("textview", action.Text))
}
if action.ID != "" {
options = append(options, WithCustomOption("id", action.ID))
}
if action.Description != "" {
options = append(options, WithCustomOption("description", action.Description))
}
if action.Identifier != "" {
options = append(options, WithDataIdentifier(action.Identifier))
}
return dExt.Driver.Input(param, options...)
case ACTION_Back:
return dExt.Driver.PressBack()
case ACTION_Sleep:
if param, ok := action.Params.(json.Number); ok {
seconds, _ := param.Float64()
time.Sleep(time.Duration(seconds*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(float64); ok {
time.Sleep(time.Duration(param*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(int64); ok {
time.Sleep(time.Duration(param) * time.Second)
return nil
}
return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
case ACTION_SleepRandom:
params, ok := action.Params.([]interface{})
if !ok {
return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
}
return sleepRandom(params)
case ACTION_ScreenShot:
// take screenshot
log.Info().Msg("take screenshot for current screen")
_, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot"))
return err
case ACTION_StartCamera:
return dExt.Driver.StartCamera()
case ACTION_StopCamera:
return dExt.Driver.StopCamera()
case ACTION_VideoCrawler:
params, ok := action.Params.(map[string]interface{})
if !ok {
return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params)
}
return dExt.VideoCrawler(params)
}
return nil
}
var errActionNotImplemented = errors.New("UI action not implemented")
func convertToFloat64(val interface{}) (float64, error) {
switch v := val.(type) {
case float64:
return v, nil
case int:
return float64(v), nil
case int64:
return float64(v), nil
default:
return 0, fmt.Errorf("invalid type for conversion to float64: %T, value: %+v", val, val)
}
}
func sleepRandom(params []interface{}) error {
// append default weight 1
if len(params) == 2 {
params = append(params, 1.0)
}
var sections []struct {
min, max, weight float64
}
totalProb := 0.0
for i := 0; i+3 <= len(params); i += 3 {
min, err := convertToFloat64(params[i])
if err != nil {
return errors.Wrapf(err, "invalid minimum time: %v", params[i])
}
max, err := convertToFloat64(params[i+1])
if err != nil {
return errors.Wrapf(err, "invalid maximum time: %v", params[i+1])
}
weight, err := convertToFloat64(params[i+2])
if err != nil {
return errors.Wrapf(err, "invalid weight value: %v", params[i+2])
}
totalProb += weight
sections = append(sections,
struct{ min, max, weight float64 }{min, max, weight},
)
}
if totalProb == 0 {
log.Warn().Msg("total weight is 0, skip sleep")
return nil
}
r := rand.Float64()
accProb := 0.0
for _, s := range sections {
accProb += s.weight / totalProb
if r < accProb {
n := s.min + rand.Float64()*(s.max-s.min)
log.Info().Float64("duration", n).Msg("sleep random seconds")
time.Sleep(time.Duration(n*1000) * time.Millisecond)
return nil
}
}
return nil
}

View File

@@ -2,7 +2,6 @@ package uixt
import (
"bytes"
"encoding/json"
"fmt"
"image"
"image/gif"
@@ -17,175 +16,13 @@ import (
"strings"
"time"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/httprunner/httprunner/v4/hrp/internal/env"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/httprunner/httprunner/v4/hrp/internal/env"
)
type ActionMethod string
const (
ACTION_AppInstall ActionMethod = "install"
ACTION_AppUninstall ActionMethod = "uninstall"
ACTION_AppStart ActionMethod = "app_start"
ACTION_AppLaunch ActionMethod = "app_launch" // 启动 app 并堵塞等待 app 首屏加载完成
ACTION_AppTerminate ActionMethod = "app_terminate"
ACTION_AppStop ActionMethod = "app_stop"
ACTION_ScreenShot ActionMethod = "screenshot"
ACTION_Sleep ActionMethod = "sleep"
ACTION_SleepRandom ActionMethod = "sleep_random"
ACTION_StartCamera ActionMethod = "camera_start" // alias for app_launch camera
ACTION_StopCamera ActionMethod = "camera_stop" // alias for app_terminate camera
// UI validation
// selectors
SelectorName string = "ui_name"
SelectorLabel string = "ui_label"
SelectorOCR string = "ui_ocr"
SelectorImage string = "ui_image"
SelectorForegroundApp string = "ui_foreground_app"
// assertions
AssertionEqual string = "equal"
AssertionNotEqual string = "not_equal"
AssertionExists string = "exists"
AssertionNotExists string = "not_exists"
// UI handling
ACTION_Home ActionMethod = "home"
ACTION_TapXY ActionMethod = "tap_xy"
ACTION_TapAbsXY ActionMethod = "tap_abs_xy"
ACTION_TapByOCR ActionMethod = "tap_ocr"
ACTION_TapByCV ActionMethod = "tap_cv"
ACTION_Tap ActionMethod = "tap"
ACTION_DoubleTapXY ActionMethod = "double_tap_xy"
ACTION_DoubleTap ActionMethod = "double_tap"
ACTION_Swipe ActionMethod = "swipe"
ACTION_Input ActionMethod = "input"
ACTION_Back ActionMethod = "back"
// custom actions
ACTION_SwipeToTapApp ActionMethod = "swipe_to_tap_app" // swipe left & right to find app and tap
ACTION_SwipeToTapText ActionMethod = "swipe_to_tap_text" // swipe up & down to find text and tap
ACTION_SwipeToTapTexts ActionMethod = "swipe_to_tap_texts" // swipe up & down to find text and tap
ACTION_VideoCrawler ActionMethod = "video_crawler"
)
type MobileAction struct {
Method ActionMethod `json:"method,omitempty" yaml:"method,omitempty"`
Params interface{} `json:"params,omitempty" yaml:"params,omitempty"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second
Duration float64 `json:"duration,omitempty" yaml:"duration,omitempty"` // used to set duration of ios swipe action
Steps int `json:"steps,omitempty" yaml:"steps,omitempty"` // used to set steps of android swipe action
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app
Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope
Offset []int `json:"offset,omitempty" yaml:"offset,omitempty"` // used to tap offset of point
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Text string `json:"text,omitempty" yaml:"text,omitempty"`
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
}
type ActionOption func(o *MobileAction)
func WithIdentifier(identifier string) ActionOption {
return func(o *MobileAction) {
o.Identifier = identifier
}
}
func WithIndex(index int) ActionOption {
return func(o *MobileAction) {
o.Index = index
}
}
func WithWaitTime(sec float64) ActionOption {
return func(o *MobileAction) {
o.WaitTime = sec
}
}
func WithDuration(duration float64) ActionOption {
return func(o *MobileAction) {
o.Duration = duration
}
}
func WithSteps(steps int) ActionOption {
return func(o *MobileAction) {
o.Steps = steps
}
}
// WithDirection inputs direction (up, down, left, right)
func WithDirection(direction string) ActionOption {
return func(o *MobileAction) {
o.Direction = direction
}
}
// WithCustomDirection inputs sx, sy, ex, ey
func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
return func(o *MobileAction) {
o.Direction = []float64{sx, sy, ex, ey}
}
}
// WithScope inputs area of [(x1,y1), (x2,y2)]
func WithScope(x1, y1, x2, y2 float64) ActionOption {
return func(o *MobileAction) {
o.Scope = []float64{x1, y1, x2, y2}
}
}
func WithOffset(offsetX, offsetY int) ActionOption {
return func(o *MobileAction) {
o.Offset = []int{offsetX, offsetY}
}
}
func WithText(text string) ActionOption {
return func(o *MobileAction) {
o.Text = text
}
}
func WithID(id string) ActionOption {
return func(o *MobileAction) {
o.ID = id
}
}
func WithDescription(description string) ActionOption {
return func(o *MobileAction) {
o.Description = description
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *MobileAction) {
o.MaxRetryTimes = maxRetryTimes
}
}
func WithTimeout(timeout int) ActionOption {
return func(o *MobileAction) {
o.Timeout = timeout
}
}
func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
return func(o *MobileAction) {
o.IgnoreNotFoundError = ignoreError
}
}
// TemplateMatchMode is the type of the template matching operation.
type TemplateMatchMode int
@@ -376,354 +213,6 @@ func (dExt *DriverExt) IsAppInForeground(packageName string) bool {
return true
}
var errActionNotImplemented = errors.New("UI action not implemented")
func convertToFloat64(val interface{}) (float64, error) {
switch v := val.(type) {
case float64:
return v, nil
case int:
return float64(v), nil
case int64:
return float64(v), nil
default:
return 0, fmt.Errorf("invalid type for conversion to float64: %T, value: %+v", val, val)
}
}
func (dExt *DriverExt) DoAction(action MobileAction) error {
log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action")
switch action.Method {
case ACTION_AppInstall:
// TODO
return errActionNotImplemented
case ACTION_AppLaunch:
if bundleId, ok := action.Params.(string); ok {
return dExt.Driver.AppLaunch(bundleId)
}
return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
ACTION_AppLaunch, action.Params)
case ACTION_SwipeToTapApp:
if appName, ok := action.Params.(string); ok {
return dExt.swipeToTapApp(appName, action)
}
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
// TODO: merge to LoopUntil
if text, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
identifierOption := WithDataIdentifier(action.Identifier)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
indexOption := WithDataIndex(action.Index)
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
waitTimeOption := WithDataWaitTime(action.WaitTime)
var point PointF
// findTextAction := func(d *DriverExt) error {
// return nil
// }
findTextCondition := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(text, indexOption, scopeOption)
return err
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
}
// swipe until found
return dExt.SwipeUntil("up", findTextCondition, foundTextAction, maxRetryOption, waitTimeOption)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
case ACTION_SwipeToTapTexts:
// TODO: merge to LoopUntil
if texts, ok := action.Params.([]interface{}); ok {
var textList []string
for _, t := range texts {
textList = append(textList, t.(string))
}
action.Params = textList
}
if texts, ok := action.Params.([]string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
identifierOption := WithDataIdentifier(action.Identifier)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
maxRetryOption := WithDataMaxRetryTimes(action.MaxRetryTimes)
waitTimeOption := WithDataWaitTime(action.WaitTime)
var point PointF
findTexts := func(d *DriverExt) error {
var err error
points, err := d.GetTextXYs(texts, scopeOption)
if err != nil {
return err
}
for _, point = range points {
if point != (PointF{X: 0, Y: 0}) {
return nil
}
}
return errors.New("failed to find text position")
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, identifierOption, offsetOption)
}
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findTexts, foundTextAction, maxRetryOption, waitTimeOption)
}
// swipe until found
return dExt.SwipeUntil("up", findTexts, foundTextAction, maxRetryOption, waitTimeOption)
}
return fmt.Errorf("invalid %s params, should be app text([]string), got %v",
ACTION_SwipeToTapText, action.Params)
case ACTION_AppTerminate:
if bundleId, ok := action.Params.(string); ok {
success, err := dExt.Driver.AppTerminate(bundleId)
if err != nil {
return errors.Wrap(err, "failed to terminate app")
}
if !success {
log.Warn().Str("bundleId", bundleId).Msg("app was not running")
}
return nil
}
return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params)
case ACTION_Home:
return dExt.Driver.Homescreen()
case ACTION_TapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_TapAbsXY:
if location, ok := action.Params.([]interface{}); ok {
// absolute coordinates x,y of window size: [100, 300]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier), WithDataOffset(action.Offset[0], action.Offset[1]))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
if len(action.Offset) != 2 {
action.Offset = []int{0, 0}
}
indexOption := WithDataIndex(action.Index)
offsetOption := WithDataOffset(action.Offset[0], action.Offset[1])
scopeOption := WithDataScope(dExt.getAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
identifierOption := WithDataIdentifier(action.Identifier)
IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError)
return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption, offsetOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.DoubleTapXY(x, y)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params)
case ACTION_DoubleTap:
if param, ok := action.Params.(string); ok {
return dExt.DoubleTap(param)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
case ACTION_Swipe:
identifierOption := WithDataIdentifier(action.Identifier)
durationOption := WithDataPressDuration(action.Duration)
if action.Steps == 0 {
action.Steps = 10
}
stepsOption := WithDataSteps(action.Steps)
if positions, ok := action.Params.([]interface{}); ok {
// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
if len(positions) != 4 {
return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions)
}
fromX, _ := positions[0].(float64)
fromY, _ := positions[1].(float64)
toX, _ := positions[2].(float64)
toY, _ := positions[3].(float64)
return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption, durationOption, stepsOption)
}
if direction, ok := action.Params.(string); ok {
return dExt.SwipeTo(direction, identifierOption, durationOption, stepsOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
case ACTION_Input:
// input text on current active element
// append \n to send text with enter
// send \b\b\b to delete 3 chars
param := fmt.Sprintf("%v", action.Params)
options := []DataOption{}
if action.Text != "" {
options = append(options, WithCustomOption("textview", action.Text))
}
if action.ID != "" {
options = append(options, WithCustomOption("id", action.ID))
}
if action.Description != "" {
options = append(options, WithCustomOption("description", action.Description))
}
if action.Identifier != "" {
options = append(options, WithDataIdentifier(action.Identifier))
}
return dExt.Driver.Input(param, options...)
case ACTION_Back:
return dExt.Driver.PressBack()
case ACTION_Sleep:
if param, ok := action.Params.(json.Number); ok {
seconds, _ := param.Float64()
time.Sleep(time.Duration(seconds*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(float64); ok {
time.Sleep(time.Duration(param*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(int64); ok {
time.Sleep(time.Duration(param) * time.Second)
return nil
}
return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
case ACTION_SleepRandom:
params, ok := action.Params.([]interface{})
if !ok {
return fmt.Errorf("invalid sleep random params: %v(%T)", action.Params, action.Params)
}
return sleepRandom(params)
case ACTION_ScreenShot:
// take screenshot
log.Info().Msg("take screenshot for current screen")
_, err := dExt.TakeScreenShot(builtin.GenNameWithTimestamp("step_%d_screenshot"))
return err
case ACTION_StartCamera:
return dExt.Driver.StartCamera()
case ACTION_StopCamera:
return dExt.Driver.StopCamera()
case ACTION_VideoCrawler:
params, ok := action.Params.(map[string]interface{})
if !ok {
return fmt.Errorf("invalid video crawler params: %v(%T)", action.Params, action.Params)
}
return dExt.VideoCrawler(params)
}
return nil
}
func sleepRandom(params []interface{}) error {
// append default weight 1
if len(params) == 2 {
params = append(params, 1.0)
}
var sections []struct {
min, max, weight float64
}
totalProb := 0.0
for i := 0; i+3 <= len(params); i += 3 {
min, err := convertToFloat64(params[i])
if err != nil {
return errors.Wrapf(err, "invalid minimum time: %v", params[i])
}
max, err := convertToFloat64(params[i+1])
if err != nil {
return errors.Wrapf(err, "invalid maximum time: %v", params[i+1])
}
weight, err := convertToFloat64(params[i+2])
if err != nil {
return errors.Wrapf(err, "invalid weight value: %v", params[i+2])
}
totalProb += weight
sections = append(sections,
struct{ min, max, weight float64 }{min, max, weight},
)
}
if totalProb == 0 {
log.Warn().Msg("total weight is 0, skip sleep")
return nil
}
r := rand.Float64()
accProb := 0.0
for _, s := range sections {
accProb += s.weight / totalProb
if r < accProb {
n := s.min + rand.Float64()*(s.max-s.min)
log.Info().Float64("duration", n).Msg("sleep random seconds")
time.Sleep(time.Duration(n*1000) * time.Millisecond)
return nil
}
}
return nil
}
func (dExt *DriverExt) getAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) {
return int(x1 * float64(dExt.windowSize.Width) * dExt.scale),
int(y1 * float64(dExt.windowSize.Height) * dExt.scale),