refactor: move uixt from hrp internal to pkg

This commit is contained in:
debugtalk
2022-10-10 21:50:07 +08:00
parent f4584db139
commit 59b06fcf65
44 changed files with 54 additions and 21 deletions

593
hrp/pkg/uixt/ext.go Normal file
View File

@@ -0,0 +1,593 @@
package uixt
import (
"bytes"
"encoding/json"
"fmt"
"image"
"image/jpeg"
"image/png"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type MobileMethod string
const (
AppInstall MobileMethod = "install"
AppUninstall MobileMethod = "uninstall"
AppStart MobileMethod = "app_start"
AppLaunch MobileMethod = "app_launch" // 等待 app 打开并堵塞到 app 首屏加载完成,可以传入 app 的启动参数、环境变量
AppLaunchUnattached MobileMethod = "app_launch_unattached" // 只负责通知打开 app不堵塞等待不可传入启动参数
AppTerminate MobileMethod = "app_terminate"
AppStop MobileMethod = "app_stop"
CtlScreenShot MobileMethod = "screenshot"
CtlSleep MobileMethod = "sleep"
CtlStartCamera MobileMethod = "camera_start" // alias for app_launch camera
CtlStopCamera MobileMethod = "camera_stop" // alias for app_terminate camera
RecordStart MobileMethod = "record_start"
RecordStop MobileMethod = "record_stop"
// UI validation
SelectorName string = "ui_name"
SelectorLabel string = "ui_label"
SelectorOCR string = "ui_ocr"
SelectorImage string = "ui_image"
AssertionExists string = "exists"
AssertionNotExists string = "not_exists"
// UI handling
ACTION_Home MobileMethod = "home"
ACTION_TapXY MobileMethod = "tap_xy"
ACTION_TapAbsXY MobileMethod = "tap_abs_xy"
ACTION_TapByOCR MobileMethod = "tap_ocr"
ACTION_TapByCV MobileMethod = "tap_cv"
ACTION_Tap MobileMethod = "tap"
ACTION_DoubleTapXY MobileMethod = "double_tap_xy"
ACTION_DoubleTap MobileMethod = "double_tap"
ACTION_Swipe MobileMethod = "swipe"
ACTION_Input MobileMethod = "input"
// custom actions
ACTION_SwipeToTapApp MobileMethod = "swipe_to_tap_app" // swipe left & right to find app and tap
ACTION_SwipeToTapText MobileMethod = "swipe_to_tap_text" // swipe up & down to find text and tap
)
type MobileAction struct {
Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"`
Params interface{} `json:"params,omitempty" yaml:"params,omitempty"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Text string `json:"text,omitempty" yaml:"text,omitempty"`
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
}
type ActionOption func(o *MobileAction)
func WithIdentifier(identifier string) ActionOption {
return func(o *MobileAction) {
o.Identifier = identifier
}
}
func WithIndex(index int) ActionOption {
return func(o *MobileAction) {
o.Index = index
}
}
func WithText(text string) ActionOption {
return func(o *MobileAction) {
o.Text = text
}
}
func WithID(id string) ActionOption {
return func(o *MobileAction) {
o.ID = id
}
}
func WithDescription(description string) ActionOption {
return func(o *MobileAction) {
o.Description = description
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *MobileAction) {
o.MaxRetryTimes = maxRetryTimes
}
}
func WithTimeout(timeout int) ActionOption {
return func(o *MobileAction) {
o.Timeout = timeout
}
}
func WithIgnoreNotFoundError(ignoreError bool) ActionOption {
return func(o *MobileAction) {
o.IgnoreNotFoundError = ignoreError
}
}
// TemplateMatchMode is the type of the template matching operation.
type TemplateMatchMode int
type CVArgs struct {
matchMode TemplateMatchMode
threshold float64
}
type CVOption func(*CVArgs)
func WithTemplateMatchMode(mode TemplateMatchMode) CVOption {
return func(args *CVArgs) {
args.matchMode = mode
}
}
func WithThreshold(threshold float64) CVOption {
return func(args *CVArgs) {
args.threshold = threshold
}
}
type DriverExt struct {
UUID string // ios udid or android serial
Driver WebDriver
windowSize Size
frame *bytes.Buffer
doneMjpegStream chan bool
scale float64
StartTime time.Time // used to associate screenshots name
ScreenShots []string // save screenshots path
perfStop chan struct{} // stop performance monitor
perfData []string // save perf data
CVArgs
}
func extend(driver WebDriver) (dExt *DriverExt, err error) {
dExt = &DriverExt{Driver: driver}
dExt.doneMjpegStream = make(chan bool, 1)
// get device window size
dExt.windowSize, err = dExt.Driver.WindowSize()
if err != nil {
return nil, errors.Wrap(err, "failed to get windows size")
}
if dExt.scale, err = dExt.Driver.Scale(); err != nil {
return nil, err
}
return dExt, nil
}
func (dExt *DriverExt) GetPerfData() []string {
if dExt.perfStop == nil {
return nil
}
close(dExt.perfStop)
return dExt.perfData
}
func (dExt *DriverExt) takeScreenShot() (raw *bytes.Buffer, err error) {
// wait for action done
time.Sleep(500 * time.Millisecond)
// 优先使用 MJPEG 流进行截图,性能最优
// 如果 MJPEG 流未开启,则使用 WebDriver 的截图接口
if dExt.frame != nil {
return dExt.frame, nil
}
if raw, err = dExt.Driver.Screenshot(); err != nil {
log.Error().Err(err).Msg("takeScreenShot failed")
return nil, err
}
return raw, nil
}
// saveScreenShot saves image file to $CWD/screenshots/ folder
func (dExt *DriverExt) saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) {
img, format, err := image.Decode(raw)
if err != nil {
return "", errors.Wrap(err, "decode screenshot image failed")
}
dir, _ := os.Getwd()
screenshotsDir := filepath.Join(dir, "screenshots")
if err = os.MkdirAll(screenshotsDir, os.ModePerm); err != nil {
return "", errors.Wrap(err, "create screenshots directory failed")
}
screenshotPath := filepath.Join(screenshotsDir,
fmt.Sprintf("%s.%s", fileName, format))
file, err := os.Create(screenshotPath)
if err != nil {
return "", errors.Wrap(err, "create screenshot image file failed")
}
defer func() {
_ = file.Close()
}()
switch format {
case "png":
err = png.Encode(file, img)
case "jpeg":
err = jpeg.Encode(file, img, nil)
default:
return "", fmt.Errorf("unsupported image format: %s", format)
}
if err != nil {
return "", errors.Wrap(err, "encode screenshot image failed")
}
return screenshotPath, nil
}
// ScreenShot takes screenshot and saves image file to $CWD/screenshots/ folder
func (dExt *DriverExt) ScreenShot(fileName string) (string, error) {
raw, err := dExt.takeScreenShot()
if err != nil {
return "", errors.Wrap(err, "screenshot by WDA failed")
}
path, err := dExt.saveScreenShot(raw, fileName)
if err != nil {
return "", errors.Wrap(err, "save screenshot failed")
}
return path, nil
}
// isPathExists returns true if path exists, whether path is file or dir
func isPathExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return false
}
return true
}
func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
var selector BySelector
if strings.HasPrefix(param, "/") {
// xpath
selector = BySelector{
XPath: param,
}
} else if strings.HasPrefix(param, "com.") {
// name
selector = BySelector{
ResourceIdID: param,
}
} else {
// name
selector = BySelector{
LinkText: NewElementAttribute().WithName(param),
}
}
return dExt.Driver.FindElement(selector)
}
func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
// click on text, using OCR
if !isPathExists(search) {
return dExt.FindTextByOCR(search, index...)
}
// click on image, using opencv
return dExt.FindImageRectInUIKit(search, index...)
}
func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale
width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale
return
}
func (dExt *DriverExt) PerformTouchActions(touchActions *TouchActions) error {
return dExt.Driver.PerformAppiumTouchActions(touchActions)
}
func (dExt *DriverExt) PerformActions(actions *W3CActions) error {
return dExt.Driver.PerformW3CActions(actions)
}
func (dExt *DriverExt) IsNameExist(name string) bool {
selector := BySelector{
LinkText: NewElementAttribute().WithName(name),
}
_, err := dExt.Driver.FindElement(selector)
return err == nil
}
func (dExt *DriverExt) IsLabelExist(label string) bool {
selector := BySelector{
LinkText: NewElementAttribute().WithLabel(label),
}
_, err := dExt.Driver.FindElement(selector)
return err == nil
}
func (dExt *DriverExt) IsOCRExist(text string) bool {
_, _, _, _, err := dExt.FindTextByOCR(text)
return err == nil
}
func (dExt *DriverExt) IsImageExist(text string) bool {
_, _, _, _, err := dExt.FindImageRectInUIKit(text)
return err == nil
}
var errActionNotImplemented = errors.New("UI action not implemented")
func (dExt *DriverExt) DoAction(action MobileAction) error {
log.Info().Str("method", string(action.Method)).Interface("params", action.Params).Msg("start UI action")
switch action.Method {
case AppInstall:
// TODO
return errActionNotImplemented
case AppLaunch:
if bundleId, ok := action.Params.(string); ok {
return dExt.Driver.AppLaunch(bundleId)
}
return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
AppLaunch, action.Params)
case AppLaunchUnattached:
if bundleId, ok := action.Params.(string); ok {
return dExt.Driver.AppLaunchUnattached(bundleId)
}
return fmt.Errorf("invalid %s params, should be bundleId(string), got %v",
AppLaunchUnattached, action.Params)
case ACTION_SwipeToTapApp:
if appName, ok := action.Params.(string); ok {
var point PointF
findApp := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(appName, action.Index)
return err
}
foundAppAction := func(d *DriverExt) error {
// click app to launch
return d.TapAbsXY(point.X, point.Y-20, action.Identifier)
}
// go to home screen
if err := dExt.Driver.Homescreen(); err != nil {
return errors.Wrap(err, "go to home screen failed")
}
// swipe to first screen
for i := 0; i < 5; i++ {
dExt.SwipeRight()
}
// default to retry 5 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 5
}
// swipe next screen until app found
return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes)
}
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
if text, ok := action.Params.(string); ok {
var point PointF
findText := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(text, action.Index)
return err
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, action.Identifier)
}
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
// swipe until live room found
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
case AppTerminate:
if bundleId, ok := action.Params.(string); ok {
success, err := dExt.Driver.AppTerminate(bundleId)
if err != nil {
return errors.Wrap(err, "failed to terminate app")
}
if !success {
log.Warn().Str("bundleId", bundleId).Msg("app was not running")
}
return nil
}
return fmt.Errorf("app_terminate params should be bundleId(string), got %v", action.Params)
case ACTION_Home:
return dExt.Driver.Homescreen()
case ACTION_TapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapXY(x, y, action.Identifier)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_TapAbsXY:
if location, ok := action.Params.([]interface{}); ok {
// absolute coordinates x,y of window size: [100, 300]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapAbsXY(x, y, action.Identifier)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
if location, ok := action.Params.([]interface{}); ok {
// relative x,y of window size: [0.5, 0.5]
if len(location) != 2 {
return fmt.Errorf("invalid tap location params: %v", location)
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.DoubleTapXY(x, y)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTapXY, action.Params)
case ACTION_DoubleTap:
if param, ok := action.Params.(string); ok {
return dExt.DoubleTap(param)
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
case ACTION_Swipe:
if positions, ok := action.Params.([]interface{}); ok {
// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
if len(positions) != 4 {
return fmt.Errorf("invalid swipe params [fromX, fromY, toX, toY]: %v", positions)
}
fromX, _ := positions[0].(float64)
fromY, _ := positions[1].(float64)
toX, _ := positions[2].(float64)
toY, _ := positions[3].(float64)
return dExt.SwipeRelative(fromX, fromY, toX, toY, action.Identifier)
}
if direction, ok := action.Params.(string); ok {
return dExt.SwipeTo(direction, action.Identifier)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
case ACTION_Input:
// input text on current active element
// append \n to send text with enter
// send \b\b\b to delete 3 chars
param := fmt.Sprintf("%v", action.Params)
options := []DataOption{}
if action.Text != "" {
options = append(options, WithCustomOption("text", action.Text))
}
if action.ID != "" {
options = append(options, WithCustomOption("id", action.ID))
}
if action.Description != "" {
options = append(options, WithCustomOption("description", action.Description))
}
if action.Identifier != "" {
options = append(options, WithCustomOption("log", map[string]interface{}{
"enable": true,
"data": action.Identifier,
}))
}
return dExt.Driver.Input(param, options...)
case CtlSleep:
if param, ok := action.Params.(json.Number); ok {
seconds, _ := param.Float64()
time.Sleep(time.Duration(seconds*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(float64); ok {
time.Sleep(time.Duration(param*1000) * time.Millisecond)
return nil
} else if param, ok := action.Params.(int64); ok {
time.Sleep(time.Duration(param) * time.Second)
return nil
}
return fmt.Errorf("invalid sleep params: %v(%T)", action.Params, action.Params)
case CtlScreenShot:
// take snapshot
log.Info().Msg("take snapshot for current screen")
screenshotPath, err := dExt.ScreenShot(fmt.Sprintf("%d_screenshot_%d",
dExt.StartTime.Unix(), time.Now().Unix()))
if err != nil {
return errors.Wrap(err, "take screenshot failed")
}
log.Info().Str("path", screenshotPath).Msg("take screenshot")
dExt.ScreenShots = append(dExt.ScreenShots, screenshotPath)
return err
case CtlStartCamera:
return dExt.Driver.StartCamera()
case CtlStopCamera:
return dExt.Driver.StopCamera()
}
return nil
}
func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool {
var exists bool
if assert == AssertionExists {
exists = true
} else {
exists = false
}
var result bool
switch check {
case SelectorName:
result = (dExt.IsNameExist(expected) == exists)
case SelectorLabel:
result = (dExt.IsLabelExist(expected) == exists)
case SelectorOCR:
result = (dExt.IsOCRExist(expected) == exists)
case SelectorImage:
result = (dExt.IsImageExist(expected) == exists)
}
if !result {
if message == nil {
message = []string{""}
}
log.Error().
Str("assert", assert).
Str("expect", expected).
Str("msg", message[0]).
Msg("validate UI failed")
return false
}
log.Info().
Str("assert", assert).
Str("expect", expected).
Msg("validate UI success")
return true
}
func checkErr(t *testing.T, err error, msg ...string) {
if err != nil {
if len(msg) == 0 {
t.Fatal(err)
} else {
t.Fatal(msg, err)
}
}
}