GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-16 18:17:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: tap the first one matches text from given texts by ocr

Browse Source
This commit is contained in:
xucong.053
2022-10-11 14:53:21 +08:00
parent 0ff5fb762f
commit cd0ab6dfdb
12 changed files with 485 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hrp/pkg/uixt/android_driver.go
View File

@@ -693,7 +693,7 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) {
}
var element WebElement
if valuetext, ok := data["text"]; ok {
if valuetext, ok := data["textview"]; ok {
element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()})
} else if valueid, ok := data["id"]; ok {
element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)})

81
hrp/pkg/uixt/ext.go
View File

@@ -63,14 +63,15 @@ type MobileAction struct {
Method MobileMethod `json:"method,omitempty" yaml:"method,omitempty"`
Params interface{} `json:"params,omitempty" yaml:"params,omitempty"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Text string `json:"text,omitempty" yaml:"text,omitempty"`
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
Text string `json:"text,omitempty" yaml:"text,omitempty"`
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
}
type ActionOption func(o *MobileAction)
@@ -87,6 +88,13 @@ func WithIndex(index int) ActionOption {
}
}
// WithDirection inputs direction (up, down, left, right, []float64{sx, sy, ex, ey})
func WithDirection(direction interface{}) ActionOption {
return func(o *MobileAction) {
o.Direction = direction
}
}
func WithText(text string) ActionOption {
return func(o *MobileAction) {
o.Text = text
@@ -363,7 +371,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
}
foundAppAction := func(d *DriverExt) error {
// click app to launch
return d.TapAbsXY(point.X, point.Y-20, action.Identifier)
return d.TapAbsXY(point.X, point.Y-25, action.Identifier)
}
// go to home screen
@@ -386,27 +394,52 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
return fmt.Errorf("invalid %s params, should be app name(string), got %v",
ACTION_SwipeToTapApp, action.Params)
case ACTION_SwipeToTapText:
var point PointF
var findText func(d *DriverExt) error
if text, ok := action.Params.(string); ok {
var point PointF
findText := func(d *DriverExt) error {
findText = func(d *DriverExt) error {
var err error
point, err = d.GetTextXY(text, action.Index)
return err
}
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, action.Identifier)
} else if texts, ok := action.Params.([]interface{}); ok {
findText = func(d *DriverExt) error {
var err error
var ts []string
for _, t := range texts {
ts = append(ts, t.(string))
}
points, err := d.GetTextXYs(ts)
if err != nil {
return err
}
for _, point = range points {
if point != (PointF{}) {
return nil
}
}
return errors.New("failed to find text position")
}
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
// swipe until live room found
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
} else {
return fmt.Errorf("invalid %s params, should be app text(string or []string), got %v",
ACTION_SwipeToTapText, action.Params)
}
return fmt.Errorf("invalid %s params, should be app text(string), got %v",
ACTION_SwipeToTapText, action.Params)
foundTextAction := func(d *DriverExt) error {
// tap text
return d.TapAbsXY(point.X, point.Y, action.Identifier)
}
// default to retry 10 times
if action.MaxRetryTimes == 0 {
action.MaxRetryTimes = 10
}
if action.Direction != nil {
return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes)
}
// swipe until live room found
return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes)
case AppTerminate:
if bundleId, ok := action.Params.(string); ok {
success, err := dExt.Driver.AppTerminate(bundleId)
@@ -497,7 +530,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
param := fmt.Sprintf("%v", action.Params)
options := []DataOption{}
if action.Text != "" {
options = append(options, WithCustomOption("text", action.Text))
options = append(options, WithCustomOption("textview", action.Text))
}
if action.ID != "" {
options = append(options, WithCustomOption("id", action.ID))

9
hrp/pkg/uixt/ocr_off.go
View File

@@ -2,9 +2,16 @@
package uixt
import "github.com/rs/zerolog/log"
import (
"github.com/rs/zerolog/log"
)
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
log.Fatal().Msg("OCR is not supported")
return
}
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
log.Fatal().Msg("OCR is not supported")
return
}

71
hrp/pkg/uixt/ocr_on.go
View File

@@ -159,6 +159,48 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (
return rects[idx], nil
}
func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects map[string]image.Rectangle, err error) {
ocrResults, err := s.getOCRResult(imageBuf)
if err != nil {
log.Error().Err(err).Msg("getOCRResult failed")
return
}
var ocrTexts []string
rects = map[string]image.Rectangle{}
for _, text := range texts {
for _, ocrResult := range ocrResults {
ocrTexts = append(ocrTexts, ocrResult.Text)
// not contains text
if !strings.Contains(ocrResult.Text, text) {
continue
}
rect := image.Rectangle{
// ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下
Min: image.Point{
X: int(ocrResult.Points[0].X),
Y: int(ocrResult.Points[0].Y),
},
Max: image.Point{
X: int(ocrResult.Points[2].X),
Y: int(ocrResult.Points[2].Y),
},
}
rects[text] = rect
break
}
if _, ok := rects[text]; !ok {
rects[text] = image.Rectangle{}
}
}
return rects, nil
}
type OCRService interface {
FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
}
@@ -182,3 +224,32 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width,
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (ps map[string][]float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
return
}
service := &veDEMOCRService{}
rects, err := service.FindTexts(ocrTexts, bufSource.Bytes())
if err != nil {
log.Warn().Msgf("FindTexts failed: %s", err.Error())
err = fmt.Errorf("FindTexts failed: %v", err)
return
}
ps = map[string][]float64{}
log.Info().Interface("ocrTexts", ocrTexts).Msgf("FindTexts success")
for text, rect := range rects {
if rect == (image.Rectangle{}) {
ps[text] = []float64{}
continue
}
x, y, width, height := dExt.MappingToRectInUIKit(rect)
ps[text] = []float64{x, y, width, height}
}
return
}

12
hrp/pkg/uixt/swipe.go
View File

@@ -72,14 +72,20 @@ type FindCondition func(driver *DriverExt) error
// FoundAction indicates the action to do after a UI element is found
type FoundAction func(driver *DriverExt) error
func (dExt *DriverExt) SwipeUntil(direction string, condition FindCondition, action FoundAction, maxTimes int) error {
func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int) error {
for i := 0; i < maxTimes; i++ {
if err := condition(dExt); err == nil {
// do action after found
return action(dExt)
}
if err := dExt.SwipeTo(direction); err != nil {
log.Error().Err(err).Msgf("swipe %s failed", direction)
if d, ok := direction.(string); ok {
if err := dExt.SwipeTo(d); err != nil {
log.Error().Err(err).Msgf("swipe %s failed", d)
}
} else if d, ok := direction.([]float64); ok {
if err := dExt.SwipeRelative(d[0], d[1], d[2], d[3]); err != nil {
log.Error().Err(err).Msgf("swipe %s failed", d)
}
}
}
return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes)

21
hrp/pkg/uixt/tap.go
View File

@@ -41,6 +41,27 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er
return point, nil
}
func (dExt *DriverExt) GetTextXYs(ocrText []string) (points map[string]PointF, err error) {
ps, err := dExt.FindTextsByOCR(ocrText)
if err != nil {
return map[string]PointF{}, err
}
points = map[string]PointF{}
for text, point := range ps {
if len(point) == 0 {
points[text] = PointF{}
continue
}
points[text] = PointF{
X: point[0] + point[2]*0.5,
Y: point[1] + point[3]*0.5,
}
}
return points, nil
}
func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) {
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...)
if err != nil {