GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-13 08:39:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: locate ocr text with index

Browse Source
This commit is contained in:
debugtalk
2022-09-26 22:25:13 +08:00
parent 0fd271f19e
commit 498cc9432a
14 changed files with 249 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
examples/uitest/demo_douyin_live.json Normal file
View File

@@ -0,0 +1,89 @@
{
"config": {
"name": "通过 feed 卡片进入微信直播间",
"ios": [
{
"port": 8700,
"mjpeg_port": 8800,
"log_on": true
}
]
},
"teststeps": [
{
"name": "启动抖音",
"ios": {
"actions": [
{
"method": "home"
},
{
"method": "app_terminate",
"params": "com.ss.iphone.ugc.Aweme"
},
{
"method": "swipe_to_tap_app",
"params": "抖音",
"max_retry_times": 5
},
{
"method": "sleep",
"params": 5
}
]
},
"validate": [
{
"check": "ui_ocr",
"assert": "exists",
"expect": "推荐",
"msg": "抖音启动失败，「推荐」不存在"
}
]
},
{
"name": "在推荐页上划，直到出现「点击进入直播间」",
"ios": {
"actions": [
{
"method": "swipe_to_tap_text",
"params": "点击进入直播间",
"identifier": "进入直播间",
"max_retry_times": 100
}
]
}
},
{
"name": "向上滑动，等待 10s",
"ios": {
"actions": [
{
"method": "swipe",
"params": "up",
"identifier": "第一次上划"
},
{
"method": "sleep",
"params": 2
},
{
"method": "screenshot"
},
{
"method": "swipe",
"params": "up",
"identifier": "第二次上划"
},
{
"method": "sleep",
"params": 2
},
{
"method": "screenshot"
}
]
}
}
]
}

45
examples/uitest/demo_douyin_live.yaml Normal file
View File

@@ -0,0 +1,45 @@
config:
name: 通过 feed 卡片进入微信直播间
ios:
- port: 8700
mjpeg_port: 8800
log_on: true
teststeps:
- name: 启动抖音
ios:
actions:
- method: home
- method: app_terminate
params: com.ss.iphone.ugc.Aweme
- method: swipe_to_tap_app
params: 抖音
max_retry_times: 5
- method: sleep
params: 5
validate:
- check: ui_ocr
assert: exists
expect: 推荐
msg: 抖音启动失败，「推荐」不存在
- name: 在推荐页上划，直到出现「点击进入直播间」
ios:
actions:
- method: swipe_to_tap_text
params: 点击进入直播间
identifier: 进入直播间
max_retry_times: 100
- name: 向上滑动，等待 10s
ios:
actions:
- method: swipe
params: up
identifier: 第一次上划
- method: sleep
params: 2
- method: screenshot
- method: swipe
params: up
identifier: 第二次上划
- method: sleep
params: 2
- method: screenshot

52
examples/uitest/demo_douyin_test.go Normal file
View File

@@ -0,0 +1,52 @@
package uitest
import (
"fmt"
"testing"
"github.com/httprunner/httprunner/v4/hrp"
)
func TestIOSDouyinLive(t *testing.T) {
testCase := &hrp.TestCase{
Config: hrp.NewConfig("通过 feed 卡片进入微信直播间").
SetIOS(hrp.WithLogOn(true), hrp.WithPort(8700), hrp.WithMjpegPort(8800)),
TestSteps: []hrp.IStep{
hrp.NewStep("启动抖音").
IOS().
Home().
AppTerminate("com.ss.iphone.ugc.Aweme"). // 关闭已运行的抖音
SwipeToTapApp("抖音", hrp.WithMaxRetryTimes(5)).Sleep(5).
Validate().
AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"),
// hrp.NewStep("处理青少年弹窗").
// IOS().
// TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)),
hrp.NewStep("在推荐页上划，直到出现「点击进入直播间」").
IOS().
SwipeToTapText("点击进入直播间", hrp.WithMaxRetryTimes(100), hrp.WithIdentifier("进入直播间")),
hrp.NewStep("向上滑动，等待 10s").
IOS().
SwipeUp(hrp.WithIdentifier("第一次上划")).Sleep(2).ScreenShot(). // 上划 1 次，等待 2s，截图保存
SwipeUp(hrp.WithIdentifier("第二次上划")).Sleep(2).ScreenShot(), // 再上划 1 次，等待 2s，截图保存
},
}
if err := testCase.Dump2JSON("demo_douyin_live.json"); err != nil {
t.Fatal(err)
}
if err := testCase.Dump2YAML("demo_douyin_live.yaml"); err != nil {
t.Fatal(err)
}
runner := hrp.NewRunner(t)
sessionRunner, err := runner.NewSessionRunner(testCase)
if err != nil {
t.Fatal(err)
}
if err := sessionRunner.Start(nil); err != nil {
t.Fatal(err)
}
summary := sessionRunner.GetSummary()
fmt.Println(summary)
}

3
examples/uitest/demo_weixin_live.json
View File

@@ -48,7 +48,8 @@
{
"method": "tap_ocr",
"params": "视频号",
"identifier": "进入视频号"
"identifier": "进入视频号",
"index": -1
}
]
}

1
examples/uitest/demo_weixin_live.yaml
View File

@@ -27,6 +27,7 @@ teststeps:
- method: tap_ocr
params: 视频号
identifier: 进入视频号
index: -1
- name: 处理青少年弹窗
ios:
actions:

4
examples/uitest/demo_weixin_test.go
View File

@@ -21,8 +21,8 @@ func TestIOSWeixinLive(t *testing.T) {
AssertLabelExists("通讯录", "微信启动失败，「通讯录」不存在"),
hrp.NewStep("进入直播页").
IOS().
Tap("发现"). // 进入「发现页」
TapByOCR("视频号", hrp.WithIdentifier("进入视频号")), // 通过 OCR 识别「视频号」
Tap("发现"). // 进入「发现页」
TapByOCR("视频号", hrp.WithIdentifier("进入视频号"), hrp.WithIndex(-1)), // 通过 OCR 识别「视频号」
hrp.NewStep("处理青少年弹窗").
IOS().
TapByOCR("我知道了", hrp.WithIgnoreNotFoundError(true)),

21
hrp/internal/uixt/ext.go
View File

@@ -63,6 +63,7 @@ type MobileAction struct {
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log
MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times
Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action
IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found
}
@@ -75,6 +76,12 @@ func WithIdentifier(identifier string) ActionOption {
}
}
func WithIndex(index int) ActionOption {
return func(o *MobileAction) {
o.Index = index
}
}
func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
return func(o *MobileAction) {
o.MaxRetryTimes = maxRetryTimes
@@ -235,13 +242,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) {
return dExt.Driver.FindElement(selector)
}
func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) {
// click on text, using OCR
if !isPathExists(search) {
return dExt.FindTextByOCR(search)
return dExt.FindTextByOCR(search, index...)
}
// click on image, using opencv
return dExt.FindImageRectInUIKit(search)
return dExt.FindImageRectInUIKit(search, index...)
}
func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) {
@@ -310,7 +317,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
var x, y, width, height float64
findApp := func(d *DriverExt) error {
var err error
x, y, width, height, err = d.FindTextByOCR(appName)
x, y, width, height, err = d.FindTextByOCR(appName, action.Index)
return err
}
foundAppAction := func(d *DriverExt) error {
@@ -384,17 +391,17 @@ func (dExt *DriverExt) DoAction(action MobileAction) error {
return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params)
case ACTION_Tap:
if param, ok := action.Params.(string); ok {
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError)
return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params)
case ACTION_TapByOCR:
if ocrText, ok := action.Params.(string); ok {
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError)
return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params)
case ACTION_TapByCV:
if imagePath, ok := action.Params.(string); ok {
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError)
return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index)
}
return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params)
case ACTION_DoubleTapXY:

3
hrp/internal/uixt/gesture_test.go
View File

@@ -3,6 +3,7 @@
package uixt
import (
"strconv"
"strings"
"testing"
)
@@ -14,7 +15,7 @@ func TestDriverExt_GesturePassword(t *testing.T) {
password[i], _ = strconv.Atoi(split[i])
}
driverExt, err := InitWDAClient()
driverExt, err := InitWDAClient(nil)
checkErr(t, err)
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"

2
hrp/internal/uixt/ocr_off.go
View File

@@ -4,7 +4,7 @@ package uixt
import "github.com/rs/zerolog/log"
func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
log.Fatal().Msg("OCR is not supported")
return
}

42
hrp/internal/uixt/ocr_on.go
View File

@@ -86,7 +86,11 @@ func (s *veDEMOCRService) getOCRResult(imageBuf []byte) ([]OCRResult, error) {
return ocrResult.OCRResult, nil
}
func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rectangle, err error) {
func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) {
if len(index) == 0 {
index = []int{0} // index not specified
}
ocrResults, err := s.getOCRResult(imageBuf)
if err != nil {
return
@@ -110,30 +114,46 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte) (rect image.Rec
Y: int(ocrResult.Points[2].Y),
},
}
rects = append(rects, rect)
// contains text while not match exactly
if ocrResult.Text != text {
rects = append(rects, rect)
continue
}
// match exactly
return rect, nil
// match exactly, and not specify index, return the first one
if index[0] == 0 {
return rect, nil
}
}
// only find the first matched one
if len(rects) > 0 {
return rects[0], nil
if len(rects) == 0 {
return image.Rectangle{}, fmt.Errorf("text %s not found", text)
}
return image.Rectangle{}, fmt.Errorf("text %s not found", text)
// get index
idx := index[0]
if idx > 0 {
// NOTICE: index start from 1
idx = idx - 1
} else if idx < 0 {
idx = len(rects) + idx
}
// index out of range
if idx >= len(rects) {
return image.Rectangle{}, fmt.Errorf("text %s found %d, index %d out of range",
text, len(rects), idx)
}
return rects[idx], nil
}
type OCRService interface {
FindText(text string, imageBuf []byte) (rect image.Rectangle, err error)
FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error)
}
func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) {
var bufSource *bytes.Buffer
if bufSource, err = dExt.takeScreenShot(); err != nil {
err = fmt.Errorf("takeScreenShot error: %v", err)
@@ -141,7 +161,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float6
}
service := &veDEMOCRService{}
rect, err := service.FindText(ocrText, bufSource.Bytes())
rect, err := service.FindText(ocrText, bufSource.Bytes(), index...)
if err != nil {
log.Warn().Err(err).Msg("FindText failed")
err = fmt.Errorf("FindText failed: %v", err)

2
hrp/internal/uixt/opencv_off.go
View File

@@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
log.Fatal().Msg("opencv is not supported")
return
}

2
hrp/internal/uixt/opencv_on.go
View File

@@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle,
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) {
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err

14
hrp/internal/uixt/tap.go
View File

@@ -27,8 +27,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error {
return dExt.tapFloat(x, y, identifier)
}
func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool) error {
x, y, width, height, err := dExt.FindTextByOCR(ocrText)
func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error {
x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...)
if err != nil {
if ignoreNotFoundError {
return nil
@@ -39,7 +39,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoun
return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
}
func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool) error {
func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error {
x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath)
if err != nil {
if ignoreNotFoundError {
@@ -51,18 +51,18 @@ func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFou
return dExt.tapFloat(x+width*0.5, y+height*0.5, identifier)
}
func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool) error {
return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError)
func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error {
return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...)
}
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool) (err error) {
func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) {
// click on element, find by name attribute
ele, err := dExt.FindUIElement(param)
if err == nil {
return ele.Click()
}
x, y, width, height, err := dExt.FindUIRectInUIKit(param)
x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...)
if err != nil {
if ignoreNotFoundError {
return nil

1
hrp/step.go
View File

@@ -19,6 +19,7 @@ const (
var (
WithIdentifier = uixt.WithIdentifier
WithMaxRetryTimes = uixt.WithMaxRetryTimes
WithIndex = uixt.WithIndex
WithTimeout = uixt.WithTimeout
WithIgnoreNotFoundError = uixt.WithIgnoreNotFoundError
)