GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-05 07:26:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: locate ocr text with index

Browse Source
This commit is contained in:
debugtalk
2022-09-26 22:26:13 +08:00
parent 1de68a1847
commit 1122e7db5b
14 changed files with 249 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hrp/internal/uixt/ext.go
+14 -7
View File
@@ -63,6 +63,7 @@ type MobileAction struct {
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
}
@@ -75,6 +76,12 @@ func WithIdentifier(identifier string) ActionOption {
}
}
func WithIndex(index int) ActionOption {
return func(o *MobileAction) {
o.Index = index
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *MobileAction) {
o.MaxRetryTimes = maxRetryTimes
@@ -235,13 +242,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
return dExt.Driver.FindElement(selector)
}
func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
// click on text, using OCR
if !isPathExists(search) {
return dExt.FindTextByOCR(search)
return dExt.FindTextByOCR(search, index...)
}
// click on image, using opencv
return dExt.FindImageRectInUIKit(search)
return dExt.FindImageRectInUIKit(search, index...)
}
func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
@@ -310,7 +317,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
var x, y, width, height float64
findApp := func(d *DriverExt) error {
var err error
x, y, width, height, err = d.FindTextByOCR(appName)
x, y, width, height, err = d.FindTextByOCR(appName, action.Index)
return err
}
foundAppAction := func(d *DriverExt) error {
@@ -384,17 +391,17 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError)
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError)
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError)
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:
hrp/internal/uixt/gesture_test.go
+2 -1
View File
@@ -3,6 +3,7 @@
package uixt
import (
"strconv"
"strings"
"testing"
)
@@ -14,7 +15,7 @@ func TestDriverExt_GesturePassword(t *testing.T) {
password[i], _ = strconv.Atoi(split[i])
}
driverExt, err := InitWDAClient()
driverExt, err := InitWDAClient(nil)
checkErr(t, err)
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
hrp/internal/uixt/ocr_off.go
+1 -1
View File
@@ -4,7 +4,7 @@ package uixt
import "github.com/rs/zerolog/log"
func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
log.Fatal().Msg("OCR is not supported")
return
}
hrp/internal/uixt/ocr_on.go
+31 -11
View File
@@ -86,7 +86,11 @@ func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) {
return ocrResult.OCRResult, nil
}
func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) {
func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) {
if len(index) == 0 {
index = []int{0} // index not specified
}
ocrResults, err := s.getOCRResult(imageBuf)
if err != nil {
return
@@ -110,30 +114,46 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rec
Y: int(ocrResult.Points[2].Y),
},
}
rects = append(rects, rect)
// contains text while not match exactly
if ocrResult.Text != text {
rects = append(rects, rect)
continue
}
// match exactly
return rect, nil
// match exactly, and not specify index, return the first one
if index[0] == 0 {
return rect, nil
}
}
// only find the first matched one
if len(rects) > 0 {
return rects[0], nil
if len(rects) == 0 {
return image.Rectangle{}, fmt.Errorf("text %s not found", text)
}
return image.Rectangle{}, fmt.Errorf("text %s not found", text)
// get index
idx := index[0]
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
} else if idx < 0 {
idx = len(rects) + idx
}
// index out of range
if idx >= len(rects) {
return image.Rectangle{}, fmt.Errorf("text %s found %d, index %d out of range",
text, len(rects), idx)
}
return rects[idx], nil
}
type OCRService interface {
FindText(text string, imageBuf []byte) (rect image.Rectangle, err error)
FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
}
func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -141,7 +161,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float6
}
service := &veDEMOCRService{}
rect, err := service.FindText(ocrText, bufSource.Bytes())
rect, err := service.FindText(ocrText, bufSource.Bytes(), index...)
if err != nil {
log.Warn().Err(err).Msg("FindText failed")
err = fmt.Errorf("FindText failed: %v", err)
hrp/internal/uixt/opencv_off.go
+1 -1
View File
@@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
log.Fatal().Msg("opencv is not supported")
return
}
hrp/internal/uixt/opencv_on.go
+1 -1
View File
@@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err
hrp/internal/uixt/tap.go
+7 -7
View File
@@ -27,8 +27,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
return dExt.tapFloat(x, y, identifier)
}
func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool) error {
x, y, width, height, err := dExt.FindTextByOCR(ocrText)
func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
if err != nil {
if ignoreNotFoundError {
return nil
@@ -39,7 +39,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun
return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
}
func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool) error {
func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath)
if err != nil {
if ignoreNotFoundError {
@@ -51,18 +51,18 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou
return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
}
func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool) error {
return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError)
func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...)
}
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool) (err error) {
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) {
// click on element, find by name attribute
ele, err := dExt.FindUIElement(param)
if err == nil {
return ele.Click()
}
x, y, width, height, err := dExt.FindUIRectInUIKit(param)
x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...)
if err != nil {
if ignoreNotFoundError {
return nil