GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-14 06:37:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1498 from httprunner/dev-v4.3-bugfix

Browse Source 
feat: get ocr position by given recognition area
This commit is contained in:
debugtalk
2022-10-17 16:42:25 +08:00
committed by GitHub
parent 3ccdb00fc2 9b0d1a4ed0
commit a1ed98cdd9
15 changed files with 294 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
hrp/pkg/uixt/android_driver.go
View File

@@ -491,12 +491,10 @@ func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) {
"x": x,
"y": y,
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
_, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap")
_, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "appium/tap")
return
}
@@ -551,16 +549,10 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp
"endY": toY,
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
if _, ok := data["steps"]; !ok {
data["steps"] = 12 // default steps
}
return ud._drag(data)
return ud._drag(d.Data)
}
func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...DataOption) (err error) {
@@ -572,16 +564,10 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D
"endY": endY,
}
// append options in post data for extra uiautomator configurations
// e.g. use WithPressDuration to set pressForDuration
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
if _, ok := data["steps"]; !ok {
data["steps"] = 12 // default steps
}
_, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/perform")
_, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "touch/perform")
return
}
@@ -590,7 +576,7 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D
// per step. So for a 100 steps, the swipe will take about 1/2 second to complete.
// `steps` is the number of move steps sent to the system
func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error {
options = append(options, WithSteps(12))
options = append(options, WithDataSteps(12))
return ud.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
}
@@ -670,16 +656,10 @@ func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) {
data := map[string]interface{}{
"text": text,
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
if _, ok := data["isReplace"]; !ok {
data["isReplace"] = true // default true
}
_, err = ud.httpPOST(data, "/session", ud.sessionId, "keys")
_, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "keys")
return
}
@@ -687,17 +667,15 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) {
data := map[string]interface{}{
"view": text,
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
var element WebElement
if valuetext, ok := data["textview"]; ok {
if valuetext, ok := d.Data["textview"]; ok {
element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()})
} else if valueid, ok := data["id"]; ok {
} else if valueid, ok := d.Data["id"]; ok {
element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)})
} else if valuedesc, ok := data["description"]; ok {
} else if valuedesc, ok := d.Data["description"]; ok {
element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().Description(fmt.Sprintf("%v", valuedesc)).String()})
} else {
element, err = ud.FindElement(BySelector{ClassName: ElementType{EditText: true}})

14
hrp/pkg/uixt/android_elment.go
View File

@@ -29,16 +29,10 @@ func (ue uiaElement) SendKeys(text string, options ...DataOption) (err error) {
"text": text,
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
if _, ok := data["isReplace"]; !ok {
data["isReplace"] = true // default true
}
_, err = ue.parent.httpPOST(data, "/session", ue.parent.sessionId, "/element", ue.id, "/value")
_, err = ue.parent.httpPOST(d.Data, "/session", ue.parent.sessionId, "/element", ue.id, "/value")
return
}
@@ -113,7 +107,7 @@ func (ue uiaElement) Swipe(fromX, fromY, toX, toY int) error {
func (ue uiaElement) SwipeFloat(fromX, fromY, toX, toY float64) error {
options := []DataOption{
WithSteps(12),
WithDataSteps(12),
WithCustomOption("elementId", ue.id),
}
return ue.parent._swipe(fromX, fromY, toX, toY, options...)

2
hrp/pkg/uixt/demo/main_test.go
View File

@@ -38,7 +38,7 @@ func TestIOSDemo(t *testing.T) {
continue
}
err = driverExt.TapAbsXY(points[1].X, points[1].Y, "")
err = driverExt.TapAbsXY(points[1].X, points[1].Y)
if err != nil {
t.Fatal(err)
}

2
hrp/pkg/uixt/drag.go
View File

@@ -26,5 +26,5 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs
fromY := y + height*yOffset
return dExt.Driver.DragFloat(fromX, fromY, toX, toY,
WithPressDuration(pressForDuration[0]))
WithDataPressDuration(pressForDuration[0]))
}

100
hrp/pkg/uixt/ext.go
View File

@@ -66,7 +66,9 @@ type MobileAction struct {
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app
Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
@@ -89,6 +91,12 @@ func WithIndex(index int) ActionOption {
}
}
func WithWaitTime(sec float64) ActionOption {
return func(o *MobileAction) {
o.WaitTime = sec
}
}
// WithDirection inputs direction (up, down, left, right)
func WithDirection(direction string) ActionOption {
return func(o *MobileAction) {
@@ -103,6 +111,13 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption {
}
}
// WithScope inputs area of [(x1,y1), (x2,y2)]
func WithScope(x1, y1, x2, y2 float64) ActionOption {
return func(o *MobileAction) {
o.Scope = []float64{x1, y1, x2, y2}
}
}
func WithText(text string) ActionOption {
return func(o *MobileAction) {
o.Text = text
@@ -299,13 +314,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
return dExt.Driver.FindElement(selector)
}
func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (x, y, width, height float64, err error) {
// click on text, using OCR
if !isPathExists(search) {
return dExt.FindTextByOCR(search, index...)
return dExt.FindTextByOCR(search, options...)
}
// click on image, using opencv
return dExt.FindImageRectInUIKit(search, index...)
return dExt.FindImageRectInUIKit(search, options...)
}
func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
@@ -371,15 +386,23 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
AppLaunchUnattached, action.Params)
case ACTION_SwipeToTapApp:
if appName, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
identifierOption := WithDataIdentifier(action.Identifier)
indexOption := WithDataIndex(action.Index)
scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
var point PointF
findApp := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(appName, action.Index)
point, err = d.GetTextXY(appName, scopeOption, indexOption)
return err
}
foundAppAction := func(d *DriverExt) error {
// click app to launch
return d.TapAbsXY(point.X, point.Y-25, action.Identifier)
return d.TapAbsXY(point.X, point.Y-25, identifierOption)
}
// go to home screen
@@ -397,21 +420,29 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
action.MaxRetryTimes = 5
}
// swipe next screen until app found
return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes)
return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes, action.WaitTime)
}
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
if text, ok := action.Params.(string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
identifierOption := WithDataIdentifier(action.Identifier)
indexOption := WithDataIndex(action.Index)
scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
var point PointF
findText := func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(text, action.Index)
point, err = d.GetTextXY(text, indexOption, scopeOption)
return err
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, action.Identifier)
return d.TapAbsXY(point.X, point.Y, identifierOption)
}
// default to retry 10 times
@@ -420,10 +451,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes)
return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes, action.WaitTime)
}
// swipe until found
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes, action.WaitTime)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
@@ -436,10 +467,16 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
action.Params = textList
}
if texts, ok := action.Params.([]string); ok {
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
var point PointF
findText := func(d *DriverExt) error {
var err error
points, err := d.GetTextXYs(texts)
points, err := d.GetTextXYs(texts, scopeOption)
if err != nil {
return err
}
@@ -452,7 +489,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, action.Identifier)
return d.TapAbsXY(point.X, point.Y, WithDataIdentifier(action.Identifier))
}
// default to retry 10 times
@@ -461,10 +498,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes)
return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes, action.WaitTime)
}
// swipe until found
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes, action.WaitTime)
}
return fmt.Errorf("invalid %s params, should be app text([]string), got %v",
ACTION_SwipeToTapText, action.Params)
@@ -490,7 +527,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapXY(x, y, action.Identifier)
return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_TapAbsXY:
@@ -501,22 +538,30 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
x, _ := location[0].(float64)
y, _ := location[1].(float64)
return dExt.TapAbsXY(x, y, action.Identifier)
return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
if len(action.Scope) != 4 {
action.Scope = []float64{0, 0, 1, 1}
}
indexOption := WithDataIndex(action.Index)
scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3]))
identifierOption := WithDataIdentifier(action.Identifier)
IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError)
return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index))
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
@@ -536,6 +581,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params)
case ACTION_Swipe:
identifierOption := WithDataIdentifier(action.Identifier)
if positions, ok := action.Params.([]interface{}); ok {
// relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1]
if len(positions) != 4 {
@@ -545,10 +591,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
fromY, _ := positions[1].(float64)
toX, _ := positions[2].(float64)
toY, _ := positions[3].(float64)
return dExt.SwipeRelative(fromX, fromY, toX, toY, action.Identifier)
return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption)
}
if direction, ok := action.Params.(string); ok {
return dExt.SwipeTo(direction, action.Identifier)
return dExt.SwipeTo(direction, identifierOption)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params)
case ACTION_Input:
@@ -567,10 +613,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
options = append(options, WithCustomOption("description", action.Description))
}
if action.Identifier != "" {
options = append(options, WithCustomOption("log", map[string]interface{}{
"enable": true,
"data": action.Identifier,
}))
options = append(options, WithDataIdentifier(action.Identifier))
}
return dExt.Driver.Input(param, options...)
case CtlSleep:
@@ -605,6 +648,13 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
return nil
}
func (dExt *DriverExt) GetAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) {
return int(x1 * float64(dExt.windowSize.Width) * dExt.scale),
int(y1 * float64(dExt.windowSize.Height) * dExt.scale),
int(x2 * float64(dExt.windowSize.Width) * dExt.scale),
int(y2 * float64(dExt.windowSize.Height) * dExt.scale)
}
func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool {
var exists bool
if assert == AssertionExists {

95
hrp/pkg/uixt/interface.go
View File

@@ -3,6 +3,7 @@ package uixt
import (
"bytes"
"fmt"
"math"
"reflect"
"strconv"
"strings"
@@ -773,32 +774,100 @@ type Rect struct {
Size
}
type DataOption func(data map[string]interface{})
type DataOption func(data *DataOptions)
func WithCustomOption(key string, value interface{}) DataOption {
return func(data map[string]interface{}) {
data[key] = value
return func(data *DataOptions) {
data.Data[key] = value
}
}
func WithPressDuration(duraion float64) DataOption {
return func(data map[string]interface{}) {
data["duration"] = duraion
func WithDataPressDuration(duration float64) DataOption {
return func(data *DataOptions) {
data.Data["duration"] = duration
}
}
func WithSteps(steps int) DataOption {
return func(data map[string]interface{}) {
data["steps"] = steps
func WithDataSteps(steps int) DataOption {
return func(data *DataOptions) {
data.Data["steps"] = steps
}
}
func WithFrequency(frequency int) DataOption {
return func(data map[string]interface{}) {
data["frequency"] = frequency
func WithDataFrequency(frequency int) DataOption {
return func(data *DataOptions) {
data.Data["frequency"] = frequency
}
}
func WithDataIndex(index int) DataOption {
return func(data *DataOptions) {
data.Index = index
}
}
func WithDataScope(x1, x2, y1, y2 int) DataOption {
return func(data *DataOptions) {
data.Scope = []int{x1, x2, y1, y2}
}
}
func WithDataIdentifier(identifier string) DataOption {
if identifier == "" {
return func(data *DataOptions) {}
}
return func(data *DataOptions) {
data.Data["log"] = map[string]interface{}{
"enable": true,
"data": identifier,
}
}
}
func WithDataIgnoreNotFoundError(ignoreError bool) DataOption {
return func(data *DataOptions) {
data.IgnoreNotFoundError = ignoreError
}
}
type DataOptions struct {
Data map[string]interface{} // configurations used by ios/android driver
Scope []int // used by ocr to get text position in the scope
Index int // index of the target element, should start from 1
IgnoreNotFoundError bool // ignore error if target element not found
}
func NewData(d map[string]interface{}, options ...DataOption) *DataOptions {
data := &DataOptions{
Data: d,
}
for _, option := range options {
option(data)
}
if len(data.Scope) == 0 {
data.Scope = []int{0, 0, math.MaxInt64, math.MaxInt64} // default scope
}
if _, ok := data.Data["steps"]; !ok {
data.Data["steps"] = 12 // default steps
}
if _, ok := data.Data["duration"]; !ok {
data.Data["duration"] = 1.0 // default duration
}
if _, ok := data.Data["frequency"]; !ok {
data.Data["frequency"] = 60 // default frequency
}
if _, ok := data.Data["isReplace"]; !ok {
data.Data["isReplace"] = true // default true
}
return data
}
// current implemeted device: IOSDevice, AndroidDevice
type Device interface {
UUID() string
@@ -905,7 +974,7 @@ type WebDriver interface {
TouchAndHoldFloat(x, y float64, second ...float64) error
// Drag Initiates a press-and-hold gesture at the coordinate, then drags to another coordinate.
// WithPressDuration option can be used to set pressForDuration (default to 1 second).
// WithPressDurationOption option can be used to set pressForDuration (default to 1 second).
Drag(fromX, fromY, toX, toY int, options ...DataOption) error
DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) error

38
hrp/pkg/uixt/ios_driver.go
View File

@@ -379,12 +379,10 @@ func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) {
"x": x,
"y": y,
}
// append options in post data for extra WDA configurations
// e.g. add identifier in tap event logs
for _, option := range options {
option(data)
}
_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0")
// new data options in post data for extra WDA configurations
d := NewData(data, options...)
_, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/tap/0")
return
}
@@ -433,26 +431,20 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp
"toY": toY,
}
// append options in post data for extra WDA configurations
// e.g. use WithPressDuration to set pressForDuration
for _, option := range options {
option(data)
}
// new data options in post data for extra WDA configurations
d := NewData(data, options...)
if _, ok := data["duration"]; !ok {
data["duration"] = 1.0 // default duration
}
_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
_, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/dragfromtoforduration")
return
}
func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error {
options = append(options, WithPressDuration(0))
options = append(options, WithDataPressDuration(0))
return wd.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...)
}
func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error {
options = append(options, WithPressDuration(0))
options = append(options, WithDataPressDuration(0))
return wd.DragFloat(fromX, fromY, toX, toY, options...)
}
@@ -514,16 +506,10 @@ func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) {
// [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)]
data := map[string]interface{}{"value": strings.Split(text, "")}
// append options in post data for extra WDA configurations
// e.g. use WithFrequency to set frequency of typing
for _, option := range options {
option(data)
}
// new data options in post data for extra WDA configurations
d := NewData(data, options...)
if _, ok := data["frequency"]; !ok {
data["frequency"] = 60 // default frequency
}
_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys")
_, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/keys")
return
}

12
hrp/pkg/uixt/ios_element.go
View File

@@ -30,16 +30,10 @@ func (we wdaElement) SendKeys(text string, options ...DataOption) (err error) {
data := map[string]interface{}{
"value": strings.Split(text, ""),
}
// append options in post data for extra uiautomator configurations
for _, option := range options {
option(data)
}
// new data options in post data for extra uiautomator configurations
d := NewData(data, options...)
if _, ok := data["frequency"]; !ok {
data["frequency"] = 60
}
_, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/element", we.id, "/value")
_, err = we.parent.httpPOST(d.Data, "/session", we.parent.sessionId, "/element", we.id, "/value")
return
}

2
hrp/pkg/uixt/ios_test.go
View File

@@ -443,7 +443,7 @@ func Test_remoteWD_TouchAndHold(t *testing.T) {
func Test_remoteWD_Drag(t *testing.T) {
setup(t)
// err := driver.Drag(200, 300, 200, 500, WithPressDuration(0.5))
// err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5))
err := driver.Swipe(200, 300, 200, 500)
if err != nil {
t.Fatal(err)

72
hrp/pkg/uixt/ocr_vedem.go
View File

@@ -109,10 +109,8 @@ func getLogID(header http.Header) string {
return logID[0]
}
func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) {
if len(index) == 0 {
index = []int{0} // index not specified
}
func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) {
data := NewData(map[string]interface{}{}, options...)
ocrResults, err := s.getOCRResult(imageBuf)
if err != nil {
@@ -123,13 +121,6 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
var rects []image.Rectangle
var ocrTexts []string
for _, ocrResult := range ocrResults {
ocrTexts = append(ocrTexts, ocrResult.Text)
// not contains text
if !strings.Contains(ocrResult.Text, text) {
continue
}
rect = image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
@@ -141,7 +132,16 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
Y: int(ocrResult.Points[2].Y),
},
}
rects = append(rects, rect)
if rect.Min.X > data.Scope[0] && rect.Max.X < data.Scope[2] && rect.Min.Y > data.Scope[1] && rect.Max.Y < data.Scope[3] {
ocrTexts = append(ocrTexts, ocrResult.Text)
// not contains text
if !strings.Contains(ocrResult.Text, text) {
continue
}
rects = append(rects, rect)
}
// contains text while not match exactly
if ocrResult.Text != text {
@@ -149,7 +149,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
}
// match exactly, and not specify index, return the first one
if index[0] == 0 {
if data.Index == 0 {
return rect, nil
}
}
@@ -160,7 +160,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
}
// get index
idx := index[0]
idx := data.Index
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
@@ -177,23 +177,22 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
return rects[idx], nil
}
func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []image.Rectangle, err error) {
func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ...DataOption) (rects []image.Rectangle, err error) {
ocrResults, err := s.getOCRResult(imageBuf)
if err != nil {
log.Error().Err(err).Msg("getOCRResult failed")
return
}
data := NewData(map[string]interface{}{}, options...)
ocrTexts := map[string]bool{}
var success bool
var rect image.Rectangle
for _, text := range texts {
var found bool
for _, ocrResult := range ocrResults {
// not contains text
if !strings.Contains(ocrResult.Text, text) {
continue
}
found = true
rect := image.Rectangle{
rect = image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(ocrResult.Points[0].X),
@@ -204,12 +203,29 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []im
Y: int(ocrResult.Points[2].Y),
},
}
rects = append(rects, rect)
break
if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] {
ocrTexts[ocrResult.Text] = true
// not contains text
if !strings.Contains(ocrResult.Text, text) {
continue
}
found = true
rects = append(rects, rect)
break
}
}
if !found {
rects = append(rects, image.Rectangle{})
}
success = found || success
}
if !success {
return rects,
fmt.Errorf("texts %s not found in %v", texts, ocrTexts)
}
return rects, nil
@@ -219,7 +235,7 @@ type OCRService interface {
FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
}
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -227,7 +243,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width,
}
service := &veDEMOCRService{}
rect, err := service.FindText(ocrText, bufSource.Bytes(), index...)
rect, err := service.FindText(ocrText, bufSource.Bytes(), options...)
if err != nil {
log.Warn().Msgf("FindText failed: %s", err.Error())
err = fmt.Errorf("FindText failed: %v", err)
@@ -240,7 +256,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width,
return
}
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (points [][]float64, err error) {
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -248,7 +264,7 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (points [][]float64, er
}
service := &veDEMOCRService{}
rects, err := service.FindTexts(ocrTexts, bufSource.Bytes())
rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), options...)
if err != nil {
log.Warn().Msgf("FindTexts failed: %s", err.Error())
err = fmt.Errorf("FindTexts failed: %v", err)

2
hrp/pkg/uixt/opencv_off.go
View File

@@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
log.Fatal().Msg("opencv is not supported")
return
}

2
hrp/pkg/uixt/opencv_on.go
View File

@@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err

42
hrp/pkg/uixt/swipe.go
View File

@@ -2,6 +2,7 @@ package uixt
import (
"fmt"
"time"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/rs/zerolog/log"
@@ -12,7 +13,7 @@ func assertRelative(p float64) bool {
}
// SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY]
func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, identifier ...string) error {
func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...DataOption) error {
width := dExt.windowSize.Width
height := dExt.windowSize.Height
@@ -27,44 +28,37 @@ func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, identifier
toX = float64(width) * toX
toY = float64(height) * toY
if len(identifier) > 0 && identifier[0] != "" {
option := WithCustomOption("log", map[string]interface{}{
"enable": true,
"data": identifier[0],
})
return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, option)
}
return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY)
return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, options...)
}
func (dExt *DriverExt) SwipeTo(direction string, identifier ...string) (err error) {
func (dExt *DriverExt) SwipeTo(direction string, options ...DataOption) (err error) {
switch direction {
case "up":
return dExt.SwipeUp(identifier...)
return dExt.SwipeUp(options...)
case "down":
return dExt.SwipeDown(identifier...)
return dExt.SwipeDown(options...)
case "left":
return dExt.SwipeLeft(identifier...)
return dExt.SwipeLeft(options...)
case "right":
return dExt.SwipeRight(identifier...)
return dExt.SwipeRight(options...)
}
return fmt.Errorf("unexpected direction: %s", direction)
}
func (dExt *DriverExt) SwipeUp(identifier ...string) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, identifier...)
func (dExt *DriverExt) SwipeUp(options ...DataOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, options...)
}
func (dExt *DriverExt) SwipeDown(identifier ...string) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, identifier...)
func (dExt *DriverExt) SwipeDown(options ...DataOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, options...)
}
func (dExt *DriverExt) SwipeLeft(identifier ...string) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, identifier...)
func (dExt *DriverExt) SwipeLeft(options ...DataOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, options...)
}
func (dExt *DriverExt) SwipeRight(identifier ...string) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, identifier...)
func (dExt *DriverExt) SwipeRight(options ...DataOption) (err error) {
return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, options...)
}
// FindCondition indicates the condition to find a UI element
@@ -73,7 +67,7 @@ type FindCondition func(driver *DriverExt) error
// FoundAction indicates the action to do after a UI element is found
type FoundAction func(driver *DriverExt) error
func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int) error {
func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int, waitTime float64) error {
for i := 0; i < maxTimes; i++ {
if err := condition(dExt); err == nil {
// do action after found
@@ -96,6 +90,8 @@ func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition
log.Error().Err(err).Msgf("swipe (%v, %v) to (%v, %v) failed", sx, sy, ex, ey)
}
}
// wait for swipe action to completed and content to load completely
time.Sleep(time.Duration(1000*waitTime) * time.Millisecond)
}
return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes)
}

61
hrp/pkg/uixt/tap.go
View File

@@ -4,19 +4,12 @@ import (
"fmt"
)
func (dExt *DriverExt) TapAbsXY(x, y float64, identifier string) error {
func (dExt *DriverExt) TapAbsXY(x, y float64, options ...DataOption) error {
// tap on absolute coordinate [x, y]
if len(identifier) > 0 {
option := WithCustomOption("log", map[string]interface{}{
"enable": true,
"data": identifier,
})
return dExt.Driver.TapFloat(x, y, option)
}
return dExt.Driver.TapFloat(x, y)
return dExt.Driver.TapFloat(x, y, options...)
}
func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error {
// tap on [x, y] percent of window size
if x > 1 || y > 1 {
return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y)
@@ -25,11 +18,11 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
x = x * float64(dExt.windowSize.Width)
y = y * float64(dExt.windowSize.Height)
return dExt.TapAbsXY(x, y, identifier)
return dExt.TapAbsXY(x, y, options...)
}
func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, err error) {
x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) {
x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...)
if err != nil {
return PointF{}, err
}
@@ -41,8 +34,8 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er
return point, nil
}
func (dExt *DriverExt) GetTextXYs(ocrText []string) (points []PointF, err error) {
ps, err := dExt.FindTextsByOCR(ocrText)
func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) {
ps, err := dExt.FindTextsByOCR(ocrText, options...)
if err != nil {
return nil, err
}
@@ -58,8 +51,8 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string) (points []PointF, err error)
return points, nil
}
func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) {
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...)
func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) {
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...)
if err != nil {
return PointF{}, err
}
@@ -71,50 +64,56 @@ func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF,
return point, nil
}
func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
point, err := dExt.GetTextXY(ocrText, index...)
func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error {
data := NewData(map[string]interface{}{}, options...)
point, err := dExt.GetTextXY(ocrText, options...)
if err != nil {
if ignoreNotFoundError {
if data.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(point.X, point.Y, identifier)
return dExt.TapAbsXY(point.X, point.Y, options...)
}
func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
point, err := dExt.GetImageXY(imagePath, index...)
func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error {
data := NewData(map[string]interface{}{}, options...)
point, err := dExt.GetImageXY(imagePath, options...)
if err != nil {
if ignoreNotFoundError {
if data.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(point.X, point.Y, identifier)
return dExt.TapAbsXY(point.X, point.Y, options...)
}
func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...)
func (dExt *DriverExt) Tap(param string, options ...DataOption) error {
return dExt.TapOffset(param, 0.5, 0.5, options...)
}
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) {
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...DataOption) (err error) {
// click on element, find by name attribute
ele, err := dExt.FindUIElement(param)
if err == nil {
return ele.Click()
}
x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...)
data := NewData(map[string]interface{}{}, options...)
x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...)
if err != nil {
if ignoreNotFoundError {
if data.IgnoreNotFoundError {
return nil
}
return err
}
return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, identifier)
return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, options...)
}
func (dExt *DriverExt) DoubleTapXY(x, y float64) error {

2
hrp/step.go
View File

@@ -23,6 +23,7 @@ const (
var (
WithIdentifier = uixt.WithIdentifier
WithMaxRetryTimes = uixt.WithMaxRetryTimes
WithWaitTime = uixt.WithWaitTime
WithIndex = uixt.WithIndex
WithTimeout = uixt.WithTimeout
WithIgnoreNotFoundError = uixt.WithIgnoreNotFoundError
@@ -31,6 +32,7 @@ var (
WithDescription = uixt.WithDescription
WithDirection = uixt.WithDirection
WithCustomDirection = uixt.WithCustomDirection
WithScope = uixt.WithScope
)
var (