From 7b0a442a7aecd9f3276d664ed8c57f643141ad71 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Sat, 15 Oct 2022 23:50:31 +0800 Subject: [PATCH 1/6] feat: get ocr position by given recognition area --- hrp/pkg/uixt/demo/main_test.go | 2 +- hrp/pkg/uixt/ext.go | 20 ++++-- hrp/pkg/uixt/ocr_vedem.go | 117 ++++++++++++++++++++++++++------- hrp/pkg/uixt/tap.go | 12 ++-- hrp/step.go | 1 + 5 files changed, 116 insertions(+), 36 deletions(-) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 6dac96d2..9456a550 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -32,7 +32,7 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) + points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}, nil) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 564bad59..4696c656 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -67,6 +67,7 @@ type MobileAction struct { Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app + RecognitionArea []float64 `json:"recognition_area,omitempty" yaml:"recognition_area,omitempty"` // used by ocr to get text position in the recognition area Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found @@ -103,6 +104,13 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { } } +// WithRecognitionArea inputs area of [(x1,y1), (x2,y2)] +func WithRecognitionArea(x1, y1, x2, y2 float64) ActionOption { + return func(o *MobileAction) { + o.RecognitionArea = []float64{x1, y1, x2, y2} + } +} + func WithText(text string) ActionOption { return func(o *MobileAction) { o.Text = text @@ -302,7 +310,7 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) { func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search, index...) + return dExt.FindTextByOCR(search, nil, index...) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, index...) @@ -339,7 +347,7 @@ func (dExt *DriverExt) IsLabelExist(label string) bool { } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, _, _, _, err := dExt.FindTextByOCR(text) + _, _, _, _, err := dExt.FindTextByOCR(text, nil) return err == nil } @@ -374,7 +382,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(appName, action.Index) + point, err = d.GetTextXY(appName, action.RecognitionArea, action.Index) return err } foundAppAction := func(d *DriverExt) error { @@ -406,7 +414,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findText := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(text, action.Index) + point, err = d.GetTextXY(text, action.RecognitionArea, action.Index) return err } foundTextAction := func(d *DriverExt) error { @@ -439,7 +447,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { var point PointF findText := func(d *DriverExt) error { var err error - points, err := d.GetTextXYs(texts) + points, err := d.GetTextXYs(texts, action.RecognitionArea) if err != nil { return err } @@ -511,7 +519,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.Index) + return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.RecognitionArea, action.Index) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 82d142dc..11388ba6 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -5,6 +5,7 @@ import ( "fmt" "image" "io/ioutil" + "math" "mime/multipart" "net/http" "os" @@ -109,7 +110,7 @@ func getLogID(header http.Header) string { return logID[0] } -func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) { +func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []int, index ...int) (rect image.Rectangle, err error) { if len(index) == 0 { index = []int{0} // index not specified } @@ -120,16 +121,25 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) ( return } + if len(recAbsArea) != 4 { + recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} + } + + var minX, minY, maxX, maxY int + if recAbsArea[0] < recAbsArea[2] { + minX, maxX = recAbsArea[0], recAbsArea[2] + } else { + minX, maxX = recAbsArea[2], recAbsArea[0] + } + if recAbsArea[1] < recAbsArea[3] { + minY, maxY = recAbsArea[1], recAbsArea[3] + } else { + minY, maxY = recAbsArea[3], recAbsArea[1] + } + var rects []image.Rectangle var ocrTexts []string for _, ocrResult := range ocrResults { - ocrTexts = append(ocrTexts, ocrResult.Text) - - // not contains text - if !strings.Contains(ocrResult.Text, text) { - continue - } - rect = image.Rectangle{ // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 Min: image.Point{ @@ -141,7 +151,16 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) ( Y: int(ocrResult.Points[2].Y), }, } - rects = append(rects, rect) + if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + ocrTexts = append(ocrTexts, ocrResult.Text) + + // not contains text + if !strings.Contains(ocrResult.Text, text) { + continue + } + + rects = append(rects, rect) + } // contains text while not match exactly if ocrResult.Text != text { @@ -177,23 +196,36 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, index ...int) ( return rects[idx], nil } -func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []image.Rectangle, err error) { +func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, recAbsArea []int) (rects []image.Rectangle, err error) { ocrResults, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return } + if len(recAbsArea) != 4 { + recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} + } + + var minX, minY, maxX, maxY int + if recAbsArea[0] < recAbsArea[2] { + minX, maxX = recAbsArea[0], recAbsArea[2] + } else { + minX, maxX = recAbsArea[2], recAbsArea[0] + } + if recAbsArea[1] < recAbsArea[3] { + minY, maxY = recAbsArea[1], recAbsArea[3] + } else { + minY, maxY = recAbsArea[3], recAbsArea[1] + } + + var success bool + var rect image.Rectangle + var ocrTexts []string for _, text := range texts { var found bool for _, ocrResult := range ocrResults { - // not contains text - if !strings.Contains(ocrResult.Text, text) { - continue - } - - found = true - rect := image.Rectangle{ + rect = image.Rectangle{ // ocrResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 Min: image.Point{ X: int(ocrResult.Points[0].X), @@ -204,12 +236,29 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte) (rects []im Y: int(ocrResult.Points[2].Y), }, } - rects = append(rects, rect) - break + + if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + ocrTexts = append(ocrTexts, ocrResult.Text) + + // not contains text + if !strings.Contains(ocrResult.Text, text) { + continue + } + + found = true + rects = append(rects, rect) + break + } } if !found { rects = append(rects, image.Rectangle{}) } + success = found || success + } + + if !success { + return rects, + fmt.Errorf("texts %s not found in %v", texts, ocrTexts) } return rects, nil @@ -219,15 +268,26 @@ type OCRService interface { FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) } -func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string, recognitionArea []float64, index ...int) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } + if len(recognitionArea) != 4 { + recognitionArea = []float64{0, 0, 1, 1} + } + + absArea := []int{ + int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), + } + service := &veDEMOCRService{} - rect, err := service.FindText(ocrText, bufSource.Bytes(), index...) + rect, err := service.FindText(ocrText, bufSource.Bytes(), absArea, index...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) err = fmt.Errorf("FindText failed: %v", err) @@ -240,15 +300,26 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, index ...int) (x, y, width, return } -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string) (points [][]float64, err error) { +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, recognitionArea []float64) (points [][]float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } + if len(recognitionArea) != 4 { + recognitionArea = []float64{0, 0, 1, 1} + } + + absArea := []int{ + int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), + } + service := &veDEMOCRService{} - rects, err := service.FindTexts(ocrTexts, bufSource.Bytes()) + rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), absArea) if err != nil { log.Warn().Msgf("FindTexts failed: %s", err.Error()) err = fmt.Errorf("FindTexts failed: %v", err) diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 7957f3dd..611a5174 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -28,8 +28,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { return dExt.TapAbsXY(x, y, identifier) } -func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, err error) { - x, y, width, height, err := dExt.FindTextByOCR(ocrText, index...) +func (dExt *DriverExt) GetTextXY(ocrText string, recognitionArea []float64, index ...int) (point PointF, err error) { + x, y, width, height, err := dExt.FindTextByOCR(ocrText, recognitionArea, index...) if err != nil { return PointF{}, err } @@ -41,8 +41,8 @@ func (dExt *DriverExt) GetTextXY(ocrText string, index ...int) (point PointF, er return point, nil } -func (dExt *DriverExt) GetTextXYs(ocrText []string) (points []PointF, err error) { - ps, err := dExt.FindTextsByOCR(ocrText) +func (dExt *DriverExt) GetTextXYs(ocrText []string, recognitionArea []float64) (points []PointF, err error) { + ps, err := dExt.FindTextsByOCR(ocrText, recognitionArea) if err != nil { return nil, err } @@ -71,8 +71,8 @@ func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, return point, nil } -func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, index ...int) error { - point, err := dExt.GetTextXY(ocrText, index...) +func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, recognitionArea []float64, index ...int) error { + point, err := dExt.GetTextXY(ocrText, recognitionArea, index...) if err != nil { if ignoreNotFoundError { return nil diff --git a/hrp/step.go b/hrp/step.go index ac7481f1..8873f947 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -31,6 +31,7 @@ var ( WithDescription = uixt.WithDescription WithDirection = uixt.WithDirection WithCustomDirection = uixt.WithCustomDirection + WithRecognitionArea = uixt.WithRecognitionArea ) var ( From 1ed4fcd1e0d19cfa6382d0b49c505a46fdfe94e2 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Sun, 16 Oct 2022 13:11:13 +0800 Subject: [PATCH 2/6] fix: modify function call parameters --- hrp/pkg/uixt/demo/main_test.go | 2 +- hrp/pkg/uixt/ext.go | 78 +++++++++++++++++++++++---- hrp/pkg/uixt/ocr_vedem.go | 99 ++++++++++++++-------------------- hrp/pkg/uixt/tap.go | 12 ++--- hrp/step.go | 2 +- 5 files changed, 116 insertions(+), 77 deletions(-) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 9456a550..6dac96d2 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -32,7 +32,7 @@ func TestIOSDemo(t *testing.T) { // 持续监测手机屏幕，直到出现青少年模式弹窗后，点击「我知道了」 for { - points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}, nil) + points, err := driverExt.GetTextXYs([]string{"青少年模式", "我知道了"}) if err != nil { time.Sleep(1 * time.Second) continue diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 4696c656..d61aad8b 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -67,7 +67,7 @@ type MobileAction struct { Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app - RecognitionArea []float64 `json:"recognition_area,omitempty" yaml:"recognition_area,omitempty"` // used by ocr to get text position in the recognition area + Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` // TODO: wait timeout in seconds for mobile action IgnoreNotFoundError bool `json:"ignore_NotFoundError,omitempty" yaml:"ignore_NotFoundError,omitempty"` // ignore error if target element not found @@ -104,10 +104,10 @@ func WithCustomDirection(sx, sy, ex, ey float64) ActionOption { } } -// WithRecognitionArea inputs area of [(x1,y1), (x2,y2)] -func WithRecognitionArea(x1, y1, x2, y2 float64) ActionOption { +// WithScope inputs area of [(x1,y1), (x2,y2)] +func WithScope(x1, y1, x2, y2 float64) ActionOption { return func(o *MobileAction) { - o.RecognitionArea = []float64{x1, y1, x2, y2} + o.Scope = []float64{x1, y1, x2, y2} } } @@ -310,7 +310,7 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) { func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search, nil, index...) + return dExt.FindTextByOCR(search, WithCustomOption("index", index)) } // click on image, using opencv return dExt.FindImageRectInUIKit(search, index...) @@ -347,7 +347,7 @@ func (dExt *DriverExt) IsLabelExist(label string) bool { } func (dExt *DriverExt) IsOCRExist(text string) bool { - _, _, _, _, err := dExt.FindTextByOCR(text, nil) + _, _, _, _, err := dExt.FindTextByOCR(text) return err == nil } @@ -379,10 +379,25 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { AppLaunchUnattached, action.Params) case ACTION_SwipeToTapApp: if appName, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(appName, action.RecognitionArea, action.Index) + point, err = d.GetTextXY(appName, options...) return err } foundAppAction := func(d *DriverExt) error { @@ -411,10 +426,25 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { ACTION_SwipeToTapApp, action.Params) case ACTION_SwipeToTapText: if text, ok := action.Params.(string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findText := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(text, action.RecognitionArea, action.Index) + point, err = d.GetTextXY(text, options...) return err } foundTextAction := func(d *DriverExt) error { @@ -444,10 +474,24 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Params = textList } if texts, ok := action.Params.([]string); ok { + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + var point PointF findText := func(d *DriverExt) error { var err error - points, err := d.GetTextXYs(texts, action.RecognitionArea) + points, err := d.GetTextXYs(texts, options...) if err != nil { return err } @@ -519,7 +563,21 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: if ocrText, ok := action.Params.(string); ok { - return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, action.RecognitionArea, action.Index) + if len(action.Scope) != 4 { + action.Scope = []float64{0, 0, 1, 1} + } + + var options []DataOption + options = append(options, + WithCustomOption("index", []int{action.Index}), + WithCustomOption("scope", []int{ + int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), + int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), + int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), + }), + ) + return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, options...) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 11388ba6..cc79dcb5 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -110,9 +110,29 @@ func getLogID(header http.Header) string { return logID[0] } -func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []int, index ...int) (rect image.Rectangle, err error) { - if len(index) == 0 { - index = []int{0} // index not specified +func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := map[string]interface{}{} + for _, option := range options { + option(data) + } + + if _, ok := data["index"]; !ok { + data["index"] = []int{0} // index not specified + } + + index, ok := data["index"].([]int) + if !ok || len(index) == 0 { + index = []int{0} + } + + _, ok = data["scope"] + if !ok { + data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified + } + + scope, ok := data["scope"].([]int) + if !ok || len(scope) != 4 { + scope = []int{0, 0, math.MaxInt64, math.MaxInt64} } ocrResults, err := s.getOCRResult(imageBuf) @@ -121,22 +141,6 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in return } - if len(recAbsArea) != 4 { - recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} - } - - var minX, minY, maxX, maxY int - if recAbsArea[0] < recAbsArea[2] { - minX, maxX = recAbsArea[0], recAbsArea[2] - } else { - minX, maxX = recAbsArea[2], recAbsArea[0] - } - if recAbsArea[1] < recAbsArea[3] { - minY, maxY = recAbsArea[1], recAbsArea[3] - } else { - minY, maxY = recAbsArea[3], recAbsArea[1] - } - var rects []image.Rectangle var ocrTexts []string for _, ocrResult := range ocrResults { @@ -151,7 +155,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in Y: int(ocrResult.Points[2].Y), }, } - if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { ocrTexts = append(ocrTexts, ocrResult.Text) // not contains text @@ -196,27 +200,26 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, recAbsArea []in return rects[idx], nil } -func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, recAbsArea []int) (rects []image.Rectangle, err error) { +func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ...DataOption) (rects []image.Rectangle, err error) { ocrResults, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") return } - if len(recAbsArea) != 4 { - recAbsArea = []int{0, 0, math.MaxInt64, math.MaxInt64} + data := map[string]interface{}{} + for _, option := range options { + option(data) } - var minX, minY, maxX, maxY int - if recAbsArea[0] < recAbsArea[2] { - minX, maxX = recAbsArea[0], recAbsArea[2] - } else { - minX, maxX = recAbsArea[2], recAbsArea[0] + _, ok := data["scope"] + if !ok { + data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified } - if recAbsArea[1] < recAbsArea[3] { - minY, maxY = recAbsArea[1], recAbsArea[3] - } else { - minY, maxY = recAbsArea[3], recAbsArea[1] + + scope, ok := data["scope"].([]int) + if !ok || len(scope) != 4 { + scope = []int{0, 0, math.MaxInt64, math.MaxInt64} } var success bool @@ -237,7 +240,7 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, recAbsArea }, } - if rect.Min.X > minX && rect.Max.X < maxX && rect.Min.Y < maxY && rect.Max.Y > minY { + if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { ocrTexts = append(ocrTexts, ocrResult.Text) // not contains text @@ -268,26 +271,15 @@ type OCRService interface { FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) } -func (dExt *DriverExt) FindTextByOCR(ocrText string, recognitionArea []float64, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } - if len(recognitionArea) != 4 { - recognitionArea = []float64{0, 0, 1, 1} - } - - absArea := []int{ - int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), - } - service := &veDEMOCRService{} - rect, err := service.FindText(ocrText, bufSource.Bytes(), absArea, index...) + rect, err := service.FindText(ocrText, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) err = fmt.Errorf("FindText failed: %v", err) @@ -300,26 +292,15 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, recognitionArea []float64, return } -func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, recognitionArea []float64) (points [][]float64, err error) { +func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) (points [][]float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { err = fmt.Errorf("takeScreenShot error: %v", err) return } - if len(recognitionArea) != 4 { - recognitionArea = []float64{0, 0, 1, 1} - } - - absArea := []int{ - int(recognitionArea[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(recognitionArea[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(recognitionArea[3] * float64(dExt.windowSize.Height) * dExt.scale), - } - service := &veDEMOCRService{} - rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), absArea) + rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindTexts failed: %s", err.Error()) err = fmt.Errorf("FindTexts failed: %v", err) diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 611a5174..d46c34aa 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -28,8 +28,8 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { return dExt.TapAbsXY(x, y, identifier) } -func (dExt *DriverExt) GetTextXY(ocrText string, recognitionArea []float64, index ...int) (point PointF, err error) { - x, y, width, height, err := dExt.FindTextByOCR(ocrText, recognitionArea, index...) +func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) { + x, y, width, height, err := dExt.FindTextByOCR(ocrText, options...) if err != nil { return PointF{}, err } @@ -41,8 +41,8 @@ func (dExt *DriverExt) GetTextXY(ocrText string, recognitionArea []float64, inde return point, nil } -func (dExt *DriverExt) GetTextXYs(ocrText []string, recognitionArea []float64) (points []PointF, err error) { - ps, err := dExt.FindTextsByOCR(ocrText, recognitionArea) +func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (points []PointF, err error) { + ps, err := dExt.FindTextsByOCR(ocrText, options...) if err != nil { return nil, err } @@ -71,8 +71,8 @@ func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, return point, nil } -func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, recognitionArea []float64, index ...int) error { - point, err := dExt.GetTextXY(ocrText, recognitionArea, index...) +func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, options ...DataOption) error { + point, err := dExt.GetTextXY(ocrText, options...) if err != nil { if ignoreNotFoundError { return nil diff --git a/hrp/step.go b/hrp/step.go index 8873f947..1f3cb5ee 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -31,7 +31,7 @@ var ( WithDescription = uixt.WithDescription WithDirection = uixt.WithDirection WithCustomDirection = uixt.WithCustomDirection - WithRecognitionArea = uixt.WithRecognitionArea + WithScope = uixt.WithScope ) var ( From 2df2792fe4d7b31262c74c1f5ec3fe0cc2546ef3 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Sun, 16 Oct 2022 23:31:13 +0800 Subject: [PATCH 3/6] change: update function optional parameters to DataOptions --- hrp/pkg/uixt/android_driver.go | 4 +- hrp/pkg/uixt/android_elment.go | 2 +- hrp/pkg/uixt/demo/main_test.go | 2 +- hrp/pkg/uixt/drag.go | 2 +- hrp/pkg/uixt/ext.go | 95 +++++++++++++--------------------- hrp/pkg/uixt/interface.go | 38 ++++++++++++-- hrp/pkg/uixt/ios_driver.go | 6 +-- hrp/pkg/uixt/ios_test.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 37 +++++-------- hrp/pkg/uixt/opencv_off.go | 2 +- hrp/pkg/uixt/opencv_on.go | 2 +- hrp/pkg/uixt/swipe.go | 37 ++++++------- hrp/pkg/uixt/tap.go | 70 ++++++++++++++----------- 13 files changed, 149 insertions(+), 150 deletions(-) diff --git a/hrp/pkg/uixt/android_driver.go b/hrp/pkg/uixt/android_driver.go index b4a2b713..f9f065ce 100644 --- a/hrp/pkg/uixt/android_driver.go +++ b/hrp/pkg/uixt/android_driver.go @@ -573,7 +573,7 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D } // append options in post data for extra uiautomator configurations - // e.g. use WithPressDuration to set pressForDuration + // e.g. use WithPressDurationOption to set pressForDuration for _, option := range options { option(data) } @@ -590,7 +590,7 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D // per step. So for a 100 steps, the swipe will take about 1/2 second to complete. // `steps` is the number of move steps sent to the system func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { - options = append(options, WithSteps(12)) + options = append(options, WithStepsOption(12)) return ud.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } diff --git a/hrp/pkg/uixt/android_elment.go b/hrp/pkg/uixt/android_elment.go index e03c792d..e8fa67b1 100644 --- a/hrp/pkg/uixt/android_elment.go +++ b/hrp/pkg/uixt/android_elment.go @@ -113,7 +113,7 @@ func (ue uiaElement) Swipe(fromX, fromY, toX, toY int) error { func (ue uiaElement) SwipeFloat(fromX, fromY, toX, toY float64) error { options := []DataOption{ - WithSteps(12), + WithStepsOption(12), WithCustomOption("elementId", ue.id), } return ue.parent._swipe(fromX, fromY, toX, toY, options...) diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 6dac96d2..1ff036ce 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -38,7 +38,7 @@ func TestIOSDemo(t *testing.T) { continue } - err = driverExt.TapAbsXY(points[1].X, points[1].Y, "") + err = driverExt.TapAbsXY(points[1].X, points[1].Y) if err != nil { t.Fatal(err) } diff --git a/hrp/pkg/uixt/drag.go b/hrp/pkg/uixt/drag.go index 27a13501..c049d71b 100644 --- a/hrp/pkg/uixt/drag.go +++ b/hrp/pkg/uixt/drag.go @@ -26,5 +26,5 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs fromY := y + height*yOffset return dExt.Driver.DragFloat(fromX, fromY, toX, toY, - WithPressDuration(pressForDuration[0])) + WithPressDurationOption(pressForDuration[0])) } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index d61aad8b..516eda63 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -307,13 +307,13 @@ func (dExt *DriverExt) FindUIElement(param string) (ele WebElement, err error) { return dExt.Driver.FindElement(selector) } -func (dExt *DriverExt) FindUIRectInUIKit(search string, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindUIRectInUIKit(search string, options ...DataOption) (x, y, width, height float64, err error) { // click on text, using OCR if !isPathExists(search) { - return dExt.FindTextByOCR(search, WithCustomOption("index", index)) + return dExt.FindTextByOCR(search, options...) } // click on image, using opencv - return dExt.FindImageRectInUIKit(search, index...) + return dExt.FindImageRectInUIKit(search, options...) } func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { @@ -383,26 +383,19 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - var options []DataOption - options = append(options, - WithCustomOption("index", []int{action.Index}), - WithCustomOption("scope", []int{ - int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), - }), - ) + identifierOption := WithIdentifierOption(action.Identifier) + indexOption := WithIndexOption(action.Index) + scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findApp := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(appName, options...) + point, err = d.GetTextXY(appName, scopeOption, indexOption) return err } foundAppAction := func(d *DriverExt) error { // click app to launch - return d.TapAbsXY(point.X, point.Y-25, action.Identifier) + return d.TapAbsXY(point.X, point.Y-25, identifierOption) } // go to home screen @@ -430,26 +423,19 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - var options []DataOption - options = append(options, - WithCustomOption("index", []int{action.Index}), - WithCustomOption("scope", []int{ - int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), - }), - ) + identifierOption := WithIdentifierOption(action.Identifier) + indexOption := WithIndexOption(action.Index) + scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findText := func(d *DriverExt) error { var err error - point, err = d.GetTextXY(text, options...) + point, err = d.GetTextXY(text, indexOption, scopeOption) return err } foundTextAction := func(d *DriverExt) error { // tap text - return d.TapAbsXY(point.X, point.Y, action.Identifier) + return d.TapAbsXY(point.X, point.Y, identifierOption) } // default to retry 10 times @@ -478,20 +464,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - var options []DataOption - options = append(options, - WithCustomOption("scope", []int{ - int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), - }), - ) + scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findText := func(d *DriverExt) error { var err error - points, err := d.GetTextXYs(texts, options...) + points, err := d.GetTextXYs(texts, scopeOption) if err != nil { return err } @@ -504,7 +482,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } foundTextAction := func(d *DriverExt) error { // tap text - return d.TapAbsXY(point.X, point.Y, action.Identifier) + return d.TapAbsXY(point.X, point.Y, WithIdentifierOption(action.Identifier)) } // default to retry 10 times @@ -542,7 +520,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - return dExt.TapXY(x, y, action.Identifier) + return dExt.TapXY(x, y, WithIdentifierOption(action.Identifier)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) case ACTION_TapAbsXY: @@ -553,12 +531,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - return dExt.TapAbsXY(x, y, action.Identifier) + return dExt.TapAbsXY(x, y, WithIdentifierOption(action.Identifier)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) case ACTION_Tap: if param, ok := action.Params.(string); ok { - return dExt.Tap(param, action.Identifier, action.IgnoreNotFoundError, action.Index) + return dExt.Tap(param, WithIdentifierOption(action.Identifier), WithIgnoreNotFoundErrorOption(true), WithIndexOption(action.Index)) } return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: @@ -567,22 +545,16 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - var options []DataOption - options = append(options, - WithCustomOption("index", []int{action.Index}), - WithCustomOption("scope", []int{ - int(action.Scope[0] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[1] * float64(dExt.windowSize.Height) * dExt.scale), - int(action.Scope[2] * float64(dExt.windowSize.Width) * dExt.scale), - int(action.Scope[3] * float64(dExt.windowSize.Height) * dExt.scale), - }), - ) - return dExt.TapByOCR(ocrText, action.Identifier, action.IgnoreNotFoundError, options...) + indexOption := WithIndexOption(action.Index) + scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + identifierOption := WithIdentifierOption(action.Identifier) + IgnoreNotFoundErrorOption := WithIgnoreNotFoundErrorOption(action.IgnoreNotFoundError) + return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: if imagePath, ok := action.Params.(string); ok { - return dExt.TapByCV(imagePath, action.Identifier, action.IgnoreNotFoundError, action.Index) + return dExt.TapByCV(imagePath, WithIdentifierOption(action.Identifier), WithIgnoreNotFoundErrorOption(true), WithIndexOption(action.Index)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: @@ -602,6 +574,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) case ACTION_Swipe: + identifierOption := WithIdentifierOption(action.Identifier) if positions, ok := action.Params.([]interface{}); ok { // relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1] if len(positions) != 4 { @@ -611,10 +584,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { fromY, _ := positions[1].(float64) toX, _ := positions[2].(float64) toY, _ := positions[3].(float64) - return dExt.SwipeRelative(fromX, fromY, toX, toY, action.Identifier) + return dExt.SwipeRelative(fromX, fromY, toX, toY, identifierOption) } if direction, ok := action.Params.(string); ok { - return dExt.SwipeTo(direction, action.Identifier) + return dExt.SwipeTo(direction, identifierOption) } return fmt.Errorf("invalid %s params: %v", ACTION_Swipe, action.Params) case ACTION_Input: @@ -633,10 +606,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { options = append(options, WithCustomOption("description", action.Description)) } if action.Identifier != "" { - options = append(options, WithCustomOption("log", map[string]interface{}{ - "enable": true, - "data": action.Identifier, - })) + options = append(options, WithIdentifierOption(action.Identifier)) } return dExt.Driver.Input(param, options...) case CtlSleep: @@ -671,6 +641,13 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { return nil } +func (dExt *DriverExt) GetAbsScope(x1, y1, x2, y2 float64) (int, int, int, int) { + return int(x1 * float64(dExt.windowSize.Width) * dExt.scale), + int(y1 * float64(dExt.windowSize.Height) * dExt.scale), + int(x2 * float64(dExt.windowSize.Width) * dExt.scale), + int(y2 * float64(dExt.windowSize.Height) * dExt.scale) +} + func (dExt *DriverExt) DoValidation(check, assert, expected string, message ...string) bool { var exists bool if assert == AssertionExists { diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 93b07179..1bf107ec 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -781,24 +781,54 @@ func WithCustomOption(key string, value interface{}) DataOption { } } -func WithPressDuration(duraion float64) DataOption { +func WithPressDurationOption(duraion float64) DataOption { return func(data map[string]interface{}) { data["duration"] = duraion } } -func WithSteps(steps int) DataOption { +func WithStepsOption(steps int) DataOption { return func(data map[string]interface{}) { data["steps"] = steps } } -func WithFrequency(frequency int) DataOption { +func WithFrequencyOption(frequency int) DataOption { return func(data map[string]interface{}) { data["frequency"] = frequency } } +func WithIndexOption(index int) DataOption { + return func(data map[string]interface{}) { + data["index"] = index + } +} + +func WithScopeOption(x1, x2, y1, y2 int) DataOption { + return func(data map[string]interface{}) { + data["scope"] = []int{x1, x2, y1, y2} + } +} + +func WithIdentifierOption(identifier string) DataOption { + if identifier == "" { + return func(data map[string]interface{}) {} + } + return func(data map[string]interface{}) { + data["log"] = map[string]interface{}{ + "enable": true, + "data": identifier, + } + } +} + +func WithIgnoreNotFoundErrorOption(ignoreError bool) DataOption { + return func(data map[string]interface{}) { + data["ignoreNotFoundError"] = ignoreError + } +} + // current implemeted device: IOSDevice, AndroidDevice type Device interface { UUID() string @@ -905,7 +935,7 @@ type WebDriver interface { TouchAndHoldFloat(x, y float64, second ...float64) error // Drag Initiates a press-and-hold gesture at the coordinate, then drags to another coordinate. - // WithPressDuration option can be used to set pressForDuration (default to 1 second). + // WithPressDurationOption option can be used to set pressForDuration (default to 1 second). Drag(fromX, fromY, toX, toY int, options ...DataOption) error DragFloat(fromX, fromY, toX, toY float64, options ...DataOption) error diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index 4e4e131f..e0a97c6f 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -434,7 +434,7 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // append options in post data for extra WDA configurations - // e.g. use WithPressDuration to set pressForDuration + // e.g. use WithPressDurationOption to set pressForDuration for _, option := range options { option(data) } @@ -447,12 +447,12 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { - options = append(options, WithPressDuration(0)) + options = append(options, WithPressDurationOption(0)) return wd.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error { - options = append(options, WithPressDuration(0)) + options = append(options, WithPressDurationOption(0)) return wd.DragFloat(fromX, fromY, toX, toY, options...) } diff --git a/hrp/pkg/uixt/ios_test.go b/hrp/pkg/uixt/ios_test.go index 136d4876..a635e305 100644 --- a/hrp/pkg/uixt/ios_test.go +++ b/hrp/pkg/uixt/ios_test.go @@ -443,7 +443,7 @@ func Test_remoteWD_TouchAndHold(t *testing.T) { func Test_remoteWD_Drag(t *testing.T) { setup(t) - // err := driver.Drag(200, 300, 200, 500, WithPressDuration(0.5)) + // err := driver.Drag(200, 300, 200, 500, WithPressDurationOption(0.5)) err := driver.Swipe(200, 300, 200, 500) if err != nil { t.Fatal(err) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index cc79dcb5..96a59d43 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -117,23 +117,14 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } if _, ok := data["index"]; !ok { - data["index"] = []int{0} // index not specified + data["index"] = 0 // index not specified } + index, _ := data["index"].(int) - index, ok := data["index"].([]int) - if !ok || len(index) == 0 { - index = []int{0} - } - - _, ok = data["scope"] - if !ok { + if _, ok := data["scope"]; !ok { data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified } - - scope, ok := data["scope"].([]int) - if !ok || len(scope) != 4 { - scope = []int{0, 0, math.MaxInt64, math.MaxInt64} - } + scope, _ := data["scope"].([]int) ocrResults, err := s.getOCRResult(imageBuf) if err != nil { @@ -172,7 +163,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // match exactly, and not specify index, return the first one - if index[0] == 0 { + if index == 0 { return rect, nil } } @@ -183,7 +174,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // get index - idx := index[0] + idx := index if idx > 0 { // NOTICE: index start from 1 idx = idx - 1 @@ -212,19 +203,15 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... option(data) } - _, ok := data["scope"] - if !ok { + if _, ok := data["scope"]; !ok { data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified } - - scope, ok := data["scope"].([]int) - if !ok || len(scope) != 4 { - scope = []int{0, 0, math.MaxInt64, math.MaxInt64} - } + scope, _ := data["scope"].([]int) var success bool var rect image.Rectangle - var ocrTexts []string + ocrTexts := map[string]bool{} + for _, text := range texts { var found bool for _, ocrResult := range ocrResults { @@ -240,8 +227,8 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... }, } - if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { - ocrTexts = append(ocrTexts, ocrResult.Text) + if rect.Min.X >= scope[0] && rect.Max.X <= scope[2] && rect.Min.Y >= scope[1] && rect.Max.Y <= scope[3] { + ocrTexts[ocrResult.Text] = true // not contains text if !strings.Contains(ocrResult.Text, text) { diff --git a/hrp/pkg/uixt/opencv_off.go b/hrp/pkg/uixt/opencv_off.go index db1865ae..ade2c4cd 100644 --- a/hrp/pkg/uixt/opencv_off.go +++ b/hrp/pkg/uixt/opencv_off.go @@ -17,7 +17,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { log.Fatal().Msg("opencv is not supported") return } diff --git a/hrp/pkg/uixt/opencv_on.go b/hrp/pkg/uixt/opencv_on.go index d4acda45..e8b3d407 100644 --- a/hrp/pkg/uixt/opencv_on.go +++ b/hrp/pkg/uixt/opencv_on.go @@ -111,7 +111,7 @@ func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, return } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, index ...int) (x, y, width, height float64, err error) { +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { var bufSource, bufSearch *bytes.Buffer if bufSearch, err = getBufFromDisk(imagePath); err != nil { return 0, 0, 0, 0, err diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 8f2ba9f9..41a11d2a 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -12,7 +12,7 @@ func assertRelative(p float64) bool { } // SwipeRelative swipe from relative position [fromX, fromY] to relative position [toX, toY] -func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, identifier ...string) error { +func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, options ...DataOption) error { width := dExt.windowSize.Width height := dExt.windowSize.Height @@ -27,44 +27,37 @@ func (dExt *DriverExt) SwipeRelative(fromX, fromY, toX, toY float64, identifier toX = float64(width) * toX toY = float64(height) * toY - if len(identifier) > 0 && identifier[0] != "" { - option := WithCustomOption("log", map[string]interface{}{ - "enable": true, - "data": identifier[0], - }) - return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, option) - } - return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY) + return dExt.Driver.SwipeFloat(fromX, fromY, toX, toY, options...) } -func (dExt *DriverExt) SwipeTo(direction string, identifier ...string) (err error) { +func (dExt *DriverExt) SwipeTo(direction string, options ...DataOption) (err error) { switch direction { case "up": - return dExt.SwipeUp(identifier...) + return dExt.SwipeUp(options...) case "down": - return dExt.SwipeDown(identifier...) + return dExt.SwipeDown(options...) case "left": - return dExt.SwipeLeft(identifier...) + return dExt.SwipeLeft(options...) case "right": - return dExt.SwipeRight(identifier...) + return dExt.SwipeRight(options...) } return fmt.Errorf("unexpected direction: %s", direction) } -func (dExt *DriverExt) SwipeUp(identifier ...string) (err error) { - return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, identifier...) +func (dExt *DriverExt) SwipeUp(options ...DataOption) (err error) { + return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.1, options...) } -func (dExt *DriverExt) SwipeDown(identifier ...string) (err error) { - return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, identifier...) +func (dExt *DriverExt) SwipeDown(options ...DataOption) (err error) { + return dExt.SwipeRelative(0.5, 0.5, 0.5, 0.9, options...) } -func (dExt *DriverExt) SwipeLeft(identifier ...string) (err error) { - return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, identifier...) +func (dExt *DriverExt) SwipeLeft(options ...DataOption) (err error) { + return dExt.SwipeRelative(0.5, 0.5, 0.1, 0.5, options...) } -func (dExt *DriverExt) SwipeRight(identifier ...string) (err error) { - return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, identifier...) +func (dExt *DriverExt) SwipeRight(options ...DataOption) (err error) { + return dExt.SwipeRelative(0.5, 0.5, 0.9, 0.5, options...) } // FindCondition indicates the condition to find a UI element diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index d46c34aa..dfe0e487 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -4,19 +4,12 @@ import ( "fmt" ) -func (dExt *DriverExt) TapAbsXY(x, y float64, identifier string) error { +func (dExt *DriverExt) TapAbsXY(x, y float64, options ...DataOption) error { // tap on absolute coordinate [x, y] - if len(identifier) > 0 { - option := WithCustomOption("log", map[string]interface{}{ - "enable": true, - "data": identifier, - }) - return dExt.Driver.TapFloat(x, y, option) - } - return dExt.Driver.TapFloat(x, y) + return dExt.Driver.TapFloat(x, y, options...) } -func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { +func (dExt *DriverExt) TapXY(x, y float64, options ...DataOption) error { // tap on [x, y] percent of window size if x > 1 || y > 1 { return fmt.Errorf("x, y percentage should be < 1, got x=%v, y=%v", x, y) @@ -25,7 +18,7 @@ func (dExt *DriverExt) TapXY(x, y float64, identifier string) error { x = x * float64(dExt.windowSize.Width) y = y * float64(dExt.windowSize.Height) - return dExt.TapAbsXY(x, y, identifier) + return dExt.TapAbsXY(x, y, options...) } func (dExt *DriverExt) GetTextXY(ocrText string, options ...DataOption) (point PointF, err error) { @@ -58,8 +51,8 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin return points, nil } -func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, err error) { - x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, index...) +func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) { + x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...) if err != nil { return PointF{}, err } @@ -71,50 +64,69 @@ func (dExt *DriverExt) GetImageXY(imagePath string, index ...int) (point PointF, return point, nil } -func (dExt *DriverExt) TapByOCR(ocrText string, identifier string, ignoreNotFoundError bool, options ...DataOption) error { +func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { + data := map[string]interface{}{} + for _, option := range options { + option(data) + } point, err := dExt.GetTextXY(ocrText, options...) if err != nil { - if ignoreNotFoundError { - return nil + if d, ok := data["ignoreNotFoundError"]; ok { + if b, ok := d.(bool); b && ok { + return nil + } } return err } - return dExt.TapAbsXY(point.X, point.Y, identifier) + return dExt.TapAbsXY(point.X, point.Y, options...) } -func (dExt *DriverExt) TapByCV(imagePath string, identifier string, ignoreNotFoundError bool, index ...int) error { - point, err := dExt.GetImageXY(imagePath, index...) +func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { + data := map[string]interface{}{} + for _, option := range options { + option(data) + } + point, err := dExt.GetImageXY(imagePath, options...) if err != nil { - if ignoreNotFoundError { - return nil + if d, ok := data["ignoreNotFoundError"]; ok { + if b, ok := d.(bool); b && ok { + return nil + } } return err } - return dExt.TapAbsXY(point.X, point.Y, identifier) + return dExt.TapAbsXY(point.X, point.Y, options...) } -func (dExt *DriverExt) Tap(param string, identifier string, ignoreNotFoundError bool, index ...int) error { - return dExt.TapOffset(param, 0.5, 0.5, identifier, ignoreNotFoundError, index...) +func (dExt *DriverExt) Tap(param string, options ...DataOption) error { + return dExt.TapOffset(param, 0.5, 0.5, options...) } -func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, identifier string, ignoreNotFoundError bool, index ...int) (err error) { +func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options ...DataOption) (err error) { // click on element, find by name attribute ele, err := dExt.FindUIElement(param) if err == nil { return ele.Click() } - x, y, width, height, err := dExt.FindUIRectInUIKit(param, index...) + data := map[string]interface{}{} + for _, option := range options { + option(data) + } + + x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { - if ignoreNotFoundError { - return nil + if d, ok := data["ignoreNotFoundError"]; ok { + if b, ok := d.(bool); b && ok { + return nil + } } return err } - return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, identifier) + return dExt.TapAbsXY(x+width*xOffset, y+height*yOffset, options...) } func (dExt *DriverExt) DoubleTapXY(x, y float64) error { From 0e248882b1d529c92f6d4ddddd4cdc98c251d588 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Mon, 17 Oct 2022 13:45:00 +0800 Subject: [PATCH 4/6] change: update DataOptions --- hrp/pkg/uixt/android_driver.go | 63 ++++++++-------------- hrp/pkg/uixt/android_elment.go | 15 ++---- hrp/pkg/uixt/drag.go | 2 +- hrp/pkg/uixt/ext.go | 36 ++++++------- hrp/pkg/uixt/interface.go | 99 +++++++++++++++++++++++++--------- hrp/pkg/uixt/ios_driver.go | 41 ++++++-------- hrp/pkg/uixt/ios_element.go | 13 ++--- hrp/pkg/uixt/ios_test.go | 2 +- hrp/pkg/uixt/ocr_vedem.go | 37 +++---------- hrp/pkg/uixt/tap.go | 35 ++++-------- 10 files changed, 159 insertions(+), 184 deletions(-) diff --git a/hrp/pkg/uixt/android_driver.go b/hrp/pkg/uixt/android_driver.go index f9f065ce..f4ed98f9 100644 --- a/hrp/pkg/uixt/android_driver.go +++ b/hrp/pkg/uixt/android_driver.go @@ -491,12 +491,11 @@ func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "x": x, "y": y, } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - _, err = ud.httpPOST(data, "/session", ud.sessionId, "appium/tap") + _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "appium/tap") return } @@ -551,16 +550,11 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp "endY": toY, } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["steps"]; !ok { - data["steps"] = 12 // default steps - } - - return ud._drag(data) + return ud._drag(d.Data) } func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...DataOption) (err error) { @@ -572,16 +566,11 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D "endY": endY, } - // append options in post data for extra uiautomator configurations - // e.g. use WithPressDurationOption to set pressForDuration - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["steps"]; !ok { - data["steps"] = 12 // default steps - } - _, err = ud.httpPOST(data, "/session", ud.sessionId, "touch/perform") + _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "touch/perform") return } @@ -590,7 +579,7 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D // per step. So for a 100 steps, the swipe will take about 1/2 second to complete. // `steps` is the number of move steps sent to the system func (ud *uiaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { - options = append(options, WithStepsOption(12)) + options = append(options, WithDataSteps(12)) return ud.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } @@ -670,16 +659,11 @@ func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) { data := map[string]interface{}{ "text": text, } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["isReplace"]; !ok { - data["isReplace"] = true // default true - } - - _, err = ud.httpPOST(data, "/session", ud.sessionId, "keys") + _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "keys") return } @@ -687,17 +671,16 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) { data := map[string]interface{}{ "view": text, } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) var element WebElement - if valuetext, ok := data["textview"]; ok { + if valuetext, ok := d.Data["textview"]; ok { element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()}) - } else if valueid, ok := data["id"]; ok { + } else if valueid, ok := d.Data["id"]; ok { element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)}) - } else if valuedesc, ok := data["description"]; ok { + } else if valuedesc, ok := d.Data["description"]; ok { element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().Description(fmt.Sprintf("%v", valuedesc)).String()}) } else { element, err = ud.FindElement(BySelector{ClassName: ElementType{EditText: true}}) diff --git a/hrp/pkg/uixt/android_elment.go b/hrp/pkg/uixt/android_elment.go index e8fa67b1..60538955 100644 --- a/hrp/pkg/uixt/android_elment.go +++ b/hrp/pkg/uixt/android_elment.go @@ -29,16 +29,11 @@ func (ue uiaElement) SendKeys(text string, options ...DataOption) (err error) { "text": text, } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["isReplace"]; !ok { - data["isReplace"] = true // default true - } - - _, err = ue.parent.httpPOST(data, "/session", ue.parent.sessionId, "/element", ue.id, "/value") + _, err = ue.parent.httpPOST(d.Data, "/session", ue.parent.sessionId, "/element", ue.id, "/value") return } @@ -113,7 +108,7 @@ func (ue uiaElement) Swipe(fromX, fromY, toX, toY int) error { func (ue uiaElement) SwipeFloat(fromX, fromY, toX, toY float64) error { options := []DataOption{ - WithStepsOption(12), + WithDataSteps(12), WithCustomOption("elementId", ue.id), } return ue.parent._swipe(fromX, fromY, toX, toY, options...) diff --git a/hrp/pkg/uixt/drag.go b/hrp/pkg/uixt/drag.go index c049d71b..31d33b1d 100644 --- a/hrp/pkg/uixt/drag.go +++ b/hrp/pkg/uixt/drag.go @@ -26,5 +26,5 @@ func (dExt *DriverExt) DragOffsetFloat(pathname string, toX, toY, xOffset, yOffs fromY := y + height*yOffset return dExt.Driver.DragFloat(fromX, fromY, toX, toY, - WithPressDurationOption(pressForDuration[0])) + WithDataPressDuration(pressForDuration[0])) } diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 516eda63..f78279d4 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -383,9 +383,9 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - identifierOption := WithIdentifierOption(action.Identifier) - indexOption := WithIndexOption(action.Index) - scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + identifierOption := WithDataIdentifier(action.Identifier) + indexOption := WithDataIndex(action.Index) + scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findApp := func(d *DriverExt) error { @@ -423,9 +423,9 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - identifierOption := WithIdentifierOption(action.Identifier) - indexOption := WithIndexOption(action.Index) - scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + identifierOption := WithDataIdentifier(action.Identifier) + indexOption := WithDataIndex(action.Index) + scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findText := func(d *DriverExt) error { @@ -464,7 +464,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) var point PointF findText := func(d *DriverExt) error { @@ -482,7 +482,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } foundTextAction := func(d *DriverExt) error { // tap text - return d.TapAbsXY(point.X, point.Y, WithIdentifierOption(action.Identifier)) + return d.TapAbsXY(point.X, point.Y, WithDataIdentifier(action.Identifier)) } // default to retry 10 times @@ -520,7 +520,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - return dExt.TapXY(x, y, WithIdentifierOption(action.Identifier)) + return dExt.TapXY(x, y, WithDataIdentifier(action.Identifier)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapXY, action.Params) case ACTION_TapAbsXY: @@ -531,12 +531,12 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } x, _ := location[0].(float64) y, _ := location[1].(float64) - return dExt.TapAbsXY(x, y, WithIdentifierOption(action.Identifier)) + return dExt.TapAbsXY(x, y, WithDataIdentifier(action.Identifier)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapAbsXY, action.Params) case ACTION_Tap: if param, ok := action.Params.(string); ok { - return dExt.Tap(param, WithIdentifierOption(action.Identifier), WithIgnoreNotFoundErrorOption(true), WithIndexOption(action.Index)) + return dExt.Tap(param, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) } return fmt.Errorf("invalid %s params: %v", ACTION_Tap, action.Params) case ACTION_TapByOCR: @@ -545,16 +545,16 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.Scope = []float64{0, 0, 1, 1} } - indexOption := WithIndexOption(action.Index) - scopeOption := WithScopeOption(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) - identifierOption := WithIdentifierOption(action.Identifier) - IgnoreNotFoundErrorOption := WithIgnoreNotFoundErrorOption(action.IgnoreNotFoundError) + indexOption := WithDataIndex(action.Index) + scopeOption := WithDataScope(dExt.GetAbsScope(action.Scope[0], action.Scope[1], action.Scope[2], action.Scope[3])) + identifierOption := WithDataIdentifier(action.Identifier) + IgnoreNotFoundErrorOption := WithDataIgnoreNotFoundError(action.IgnoreNotFoundError) return dExt.TapByOCR(ocrText, identifierOption, IgnoreNotFoundErrorOption, indexOption, scopeOption) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByOCR, action.Params) case ACTION_TapByCV: if imagePath, ok := action.Params.(string); ok { - return dExt.TapByCV(imagePath, WithIdentifierOption(action.Identifier), WithIgnoreNotFoundErrorOption(true), WithIndexOption(action.Index)) + return dExt.TapByCV(imagePath, WithDataIdentifier(action.Identifier), WithDataIgnoreNotFoundError(true), WithDataIndex(action.Index)) } return fmt.Errorf("invalid %s params: %v", ACTION_TapByCV, action.Params) case ACTION_DoubleTapXY: @@ -574,7 +574,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } return fmt.Errorf("invalid %s params: %v", ACTION_DoubleTap, action.Params) case ACTION_Swipe: - identifierOption := WithIdentifierOption(action.Identifier) + identifierOption := WithDataIdentifier(action.Identifier) if positions, ok := action.Params.([]interface{}); ok { // relative fromX, fromY, toX, toY of window size: [0.5, 0.9, 0.5, 0.1] if len(positions) != 4 { @@ -606,7 +606,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { options = append(options, WithCustomOption("description", action.Description)) } if action.Identifier != "" { - options = append(options, WithIdentifierOption(action.Identifier)) + options = append(options, WithDataIdentifier(action.Identifier)) } return dExt.Driver.Input(param, options...) case CtlSleep: diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 1bf107ec..ce07f28a 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -3,6 +3,7 @@ package uixt import ( "bytes" "fmt" + "math" "reflect" "strconv" "strings" @@ -773,62 +774,110 @@ type Rect struct { Size } -type DataOption func(data map[string]interface{}) +type DataOption func(data *DataOptions) func WithCustomOption(key string, value interface{}) DataOption { - return func(data map[string]interface{}) { - data[key] = value + return func(data *DataOptions) { + data.Data[key] = value } } -func WithPressDurationOption(duraion float64) DataOption { - return func(data map[string]interface{}) { - data["duration"] = duraion +func WithDataPressDuration(duration float64) DataOption { + return func(data *DataOptions) { + data.Data["duration"] = duration } } -func WithStepsOption(steps int) DataOption { - return func(data map[string]interface{}) { - data["steps"] = steps +func WithDataSteps(steps int) DataOption { + return func(data *DataOptions) { + data.Data["steps"] = steps } } -func WithFrequencyOption(frequency int) DataOption { - return func(data map[string]interface{}) { - data["frequency"] = frequency +func WithDataFrequency(frequency int) DataOption { + return func(data *DataOptions) { + data.Data["frequency"] = frequency } } -func WithIndexOption(index int) DataOption { - return func(data map[string]interface{}) { - data["index"] = index +func WithDataIndex(index int) DataOption { + return func(data *DataOptions) { + data.Index = index } } -func WithScopeOption(x1, x2, y1, y2 int) DataOption { - return func(data map[string]interface{}) { - data["scope"] = []int{x1, x2, y1, y2} +func WithDataScope(x1, x2, y1, y2 int) DataOption { + return func(data *DataOptions) { + data.Scope = []int{x1, x2, y1, y2} } } -func WithIdentifierOption(identifier string) DataOption { +func WithDataIdentifier(identifier string) DataOption { if identifier == "" { - return func(data map[string]interface{}) {} + return func(data *DataOptions) {} } - return func(data map[string]interface{}) { - data["log"] = map[string]interface{}{ + return func(data *DataOptions) { + data.Data["log"] = map[string]interface{}{ "enable": true, "data": identifier, } } } -func WithIgnoreNotFoundErrorOption(ignoreError bool) DataOption { - return func(data map[string]interface{}) { - data["ignoreNotFoundError"] = ignoreError +func WithDataIgnoreNotFoundError(ignoreError bool) DataOption { + return func(data *DataOptions) { + data.IgnoreNotFoundError = ignoreError } } +type DataOptions struct { + Data map[string]interface{} // configurations used by ios/android driver + Scope []int // used by ocr to get text position in the scope + Index int // index of the target element, should start from 1 + IgnoreNotFoundError bool // ignore error if target element not found +} + +func NewData(options ...DataOption) *DataOptions { + data := &DataOptions{ + Data: map[string]interface{}{}, + } + for _, option := range options { + option(data) + } + + if len(data.Scope) == 0 { + data.Scope = []int{0, 0, math.MaxInt64, math.MaxInt64} // default scope + } + + if _, ok := data.Data["steps"]; !ok { + data.Data["steps"] = 12 // default steps + } + + if _, ok := data.Data["duration"]; !ok { + data.Data["duration"] = 1.0 // default duration + } + + if _, ok := data.Data["frequency"]; !ok { + data.Data["frequency"] = 60 // default frequency + } + + if _, ok := data.Data["isReplace"]; !ok { + data.Data["isReplace"] = true // default true + } + + return data +} + +func (d *DataOptions) MergeData(data map[string]interface{}) { + for key, value := range data { + d.Data[key] = value + } +} + +func (d *DataOptions) AddData(key string, value interface{}) { + d.Data[key] = value +} + // current implemeted device: IOSDevice, AndroidDevice type Device interface { UUID() string diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index e0a97c6f..e9593fbf 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -379,12 +379,11 @@ func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "x": x, "y": y, } - // append options in post data for extra WDA configurations - // e.g. add identifier in tap event logs - for _, option := range options { - option(data) - } - _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/tap/0") + // new data options in post data for extra WDA configurations + d := NewData(options...) + d.MergeData(data) + + _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/tap/0") return } @@ -433,26 +432,21 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp "toY": toY, } - // append options in post data for extra WDA configurations - // e.g. use WithPressDurationOption to set pressForDuration - for _, option := range options { - option(data) - } + // new data options in post data for extra WDA configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["duration"]; !ok { - data["duration"] = 1.0 // default duration - } - _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } func (wd *wdaDriver) Swipe(fromX, fromY, toX, toY int, options ...DataOption) error { - options = append(options, WithPressDurationOption(0)) + options = append(options, WithDataPressDuration(0)) return wd.SwipeFloat(float64(fromX), float64(fromY), float64(toX), float64(toY), options...) } func (wd *wdaDriver) SwipeFloat(fromX, fromY, toX, toY float64, options ...DataOption) error { - options = append(options, WithPressDurationOption(0)) + options = append(options, WithDataPressDuration(0)) return wd.DragFloat(fromX, fromY, toX, toY, options...) } @@ -514,16 +508,11 @@ func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) { // [[FBRoute POST:@"/wda/keys"] respondWithTarget:self action:@selector(handleKeys:)] data := map[string]interface{}{"value": strings.Split(text, "")} - // append options in post data for extra WDA configurations - // e.g. use WithFrequency to set frequency of typing - for _, option := range options { - option(data) - } + // new data options in post data for extra WDA configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["frequency"]; !ok { - data["frequency"] = 60 // default frequency - } - _, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/keys") + _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/keys") return } diff --git a/hrp/pkg/uixt/ios_element.go b/hrp/pkg/uixt/ios_element.go index 74006479..e4ca6300 100644 --- a/hrp/pkg/uixt/ios_element.go +++ b/hrp/pkg/uixt/ios_element.go @@ -30,16 +30,11 @@ func (we wdaElement) SendKeys(text string, options ...DataOption) (err error) { data := map[string]interface{}{ "value": strings.Split(text, ""), } - // append options in post data for extra uiautomator configurations - for _, option := range options { - option(data) - } + // new data options in post data for extra uiautomator configurations + d := NewData(options...) + d.MergeData(data) - if _, ok := data["frequency"]; !ok { - data["frequency"] = 60 - } - - _, err = we.parent.httpPOST(data, "/session", we.parent.sessionId, "/element", we.id, "/value") + _, err = we.parent.httpPOST(d.Data, "/session", we.parent.sessionId, "/element", we.id, "/value") return } diff --git a/hrp/pkg/uixt/ios_test.go b/hrp/pkg/uixt/ios_test.go index a635e305..07377317 100644 --- a/hrp/pkg/uixt/ios_test.go +++ b/hrp/pkg/uixt/ios_test.go @@ -443,7 +443,7 @@ func Test_remoteWD_TouchAndHold(t *testing.T) { func Test_remoteWD_Drag(t *testing.T) { setup(t) - // err := driver.Drag(200, 300, 200, 500, WithPressDurationOption(0.5)) + // err := driver.Drag(200, 300, 200, 500, WithDataPressDuration(0.5)) err := driver.Swipe(200, 300, 200, 500) if err != nil { t.Fatal(err) diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 96a59d43..cca5ea7d 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -5,7 +5,6 @@ import ( "fmt" "image" "io/ioutil" - "math" "mime/multipart" "net/http" "os" @@ -111,20 +110,7 @@ func getLogID(header http.Header) string { } func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) { - data := map[string]interface{}{} - for _, option := range options { - option(data) - } - - if _, ok := data["index"]; !ok { - data["index"] = 0 // index not specified - } - index, _ := data["index"].(int) - - if _, ok := data["scope"]; !ok { - data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified - } - scope, _ := data["scope"].([]int) + data := NewData(options...) ocrResults, err := s.getOCRResult(imageBuf) if err != nil { @@ -146,7 +132,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data Y: int(ocrResult.Points[2].Y), }, } - if rect.Min.X > scope[0] && rect.Max.X < scope[2] && rect.Min.Y > scope[1] && rect.Max.Y < scope[3] { + if rect.Min.X > data.Scope[0] && rect.Max.X < data.Scope[2] && rect.Min.Y > data.Scope[1] && rect.Max.Y < data.Scope[3] { ocrTexts = append(ocrTexts, ocrResult.Text) // not contains text @@ -163,7 +149,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // match exactly, and not specify index, return the first one - if index == 0 { + if data.Index == 0 { return rect, nil } } @@ -174,7 +160,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // get index - idx := index + idx := data.Index if idx > 0 { // NOTICE: index start from 1 idx = idx - 1 @@ -198,20 +184,11 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... return } - data := map[string]interface{}{} - for _, option := range options { - option(data) - } - - if _, ok := data["scope"]; !ok { - data["scope"] = []int{0, 0, math.MaxInt64, math.MaxInt64} // scope not specified - } - scope, _ := data["scope"].([]int) + data := NewData(options...) + ocrTexts := map[string]bool{} var success bool var rect image.Rectangle - ocrTexts := map[string]bool{} - for _, text := range texts { var found bool for _, ocrResult := range ocrResults { @@ -227,7 +204,7 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... }, } - if rect.Min.X >= scope[0] && rect.Max.X <= scope[2] && rect.Min.Y >= scope[1] && rect.Max.Y <= scope[3] { + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { ocrTexts[ocrResult.Text] = true // not contains text diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index dfe0e487..d7a642a2 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -65,16 +65,12 @@ func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (poin } func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { - data := map[string]interface{}{} - for _, option := range options { - option(data) - } + data := NewData(options...) + point, err := dExt.GetTextXY(ocrText, options...) if err != nil { - if d, ok := data["ignoreNotFoundError"]; ok { - if b, ok := d.(bool); b && ok { - return nil - } + if data.IgnoreNotFoundError { + return nil } return err } @@ -83,16 +79,12 @@ func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { } func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { - data := map[string]interface{}{} - for _, option := range options { - option(data) - } + data := NewData(options...) + point, err := dExt.GetImageXY(imagePath, options...) if err != nil { - if d, ok := data["ignoreNotFoundError"]; ok { - if b, ok := d.(bool); b && ok { - return nil - } + if data.IgnoreNotFoundError { + return nil } return err } @@ -111,17 +103,12 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options return ele.Click() } - data := map[string]interface{}{} - for _, option := range options { - option(data) - } + data := NewData(options...) x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { - if d, ok := data["ignoreNotFoundError"]; ok { - if b, ok := d.(bool); b && ok { - return nil - } + if data.IgnoreNotFoundError { + return nil } return err } From 317ec4fde80df5daf72ac1de6fe3ed2e79eed488 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Mon, 17 Oct 2022 15:28:46 +0800 Subject: [PATCH 5/6] change: update NewData function --- hrp/pkg/uixt/android_driver.go | 15 +++++---------- hrp/pkg/uixt/android_elment.go | 3 +-- hrp/pkg/uixt/interface.go | 14 ++------------ hrp/pkg/uixt/ios_driver.go | 9 +++------ hrp/pkg/uixt/ios_element.go | 3 +-- hrp/pkg/uixt/ocr_vedem.go | 4 ++-- hrp/pkg/uixt/tap.go | 6 +++--- 7 files changed, 17 insertions(+), 37 deletions(-) diff --git a/hrp/pkg/uixt/android_driver.go b/hrp/pkg/uixt/android_driver.go index f4ed98f9..3f1bdea0 100644 --- a/hrp/pkg/uixt/android_driver.go +++ b/hrp/pkg/uixt/android_driver.go @@ -492,8 +492,7 @@ func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "y": y, } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "appium/tap") return @@ -551,8 +550,7 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) return ud._drag(d.Data) } @@ -567,8 +565,7 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "touch/perform") return @@ -660,8 +657,7 @@ func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) { "text": text, } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "keys") return @@ -672,8 +668,7 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) { "view": text, } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) var element WebElement if valuetext, ok := d.Data["textview"]; ok { diff --git a/hrp/pkg/uixt/android_elment.go b/hrp/pkg/uixt/android_elment.go index 60538955..9d45cd39 100644 --- a/hrp/pkg/uixt/android_elment.go +++ b/hrp/pkg/uixt/android_elment.go @@ -30,8 +30,7 @@ func (ue uiaElement) SendKeys(text string, options ...DataOption) (err error) { } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = ue.parent.httpPOST(d.Data, "/session", ue.parent.sessionId, "/element", ue.id, "/value") return diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index ce07f28a..f922ee00 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -837,9 +837,9 @@ type DataOptions struct { IgnoreNotFoundError bool // ignore error if target element not found } -func NewData(options ...DataOption) *DataOptions { +func NewData(d map[string]interface{}, options ...DataOption) *DataOptions { data := &DataOptions{ - Data: map[string]interface{}{}, + Data: d, } for _, option := range options { option(data) @@ -868,16 +868,6 @@ func NewData(options ...DataOption) *DataOptions { return data } -func (d *DataOptions) MergeData(data map[string]interface{}) { - for key, value := range data { - d.Data[key] = value - } -} - -func (d *DataOptions) AddData(key string, value interface{}) { - d.Data[key] = value -} - // current implemeted device: IOSDevice, AndroidDevice type Device interface { UUID() string diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index e9593fbf..9db911e8 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -380,8 +380,7 @@ func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "y": y, } // new data options in post data for extra WDA configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/tap/0") return @@ -433,8 +432,7 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra WDA configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/dragfromtoforduration") return @@ -509,8 +507,7 @@ func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) { data := map[string]interface{}{"value": strings.Split(text, "")} // new data options in post data for extra WDA configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/keys") return diff --git a/hrp/pkg/uixt/ios_element.go b/hrp/pkg/uixt/ios_element.go index e4ca6300..9e2208b5 100644 --- a/hrp/pkg/uixt/ios_element.go +++ b/hrp/pkg/uixt/ios_element.go @@ -31,8 +31,7 @@ func (we wdaElement) SendKeys(text string, options ...DataOption) (err error) { "value": strings.Split(text, ""), } // new data options in post data for extra uiautomator configurations - d := NewData(options...) - d.MergeData(data) + d := NewData(data, options...) _, err = we.parent.httpPOST(d.Data, "/session", we.parent.sessionId, "/element", we.id, "/value") return diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index cca5ea7d..f1b58366 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -110,7 +110,7 @@ func getLogID(header http.Header) string { } func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) { - data := NewData(options...) + data := NewData(map[string]interface{}{}, options...) ocrResults, err := s.getOCRResult(imageBuf) if err != nil { @@ -184,7 +184,7 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... return } - data := NewData(options...) + data := NewData(map[string]interface{}{}, options...) ocrTexts := map[string]bool{} var success bool diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index d7a642a2..1629f137 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -65,7 +65,7 @@ func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (poin } func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { - data := NewData(options...) + data := NewData(map[string]interface{}{}, options...) point, err := dExt.GetTextXY(ocrText, options...) if err != nil { @@ -79,7 +79,7 @@ func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { } func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { - data := NewData(options...) + data := NewData(map[string]interface{}{}, options...) point, err := dExt.GetImageXY(imagePath, options...) if err != nil { @@ -103,7 +103,7 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options return ele.Click() } - data := NewData(options...) + data := NewData(map[string]interface{}{}, options...) x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { From 9b0d1a4ed0e72c89d5a7a9afb3a49bc658799a1f Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Mon, 17 Oct 2022 16:23:18 +0800 Subject: [PATCH 6/6] feat: add wait time between swipe and ocr --- hrp/pkg/uixt/ext.go | 17 ++++++++++++----- hrp/pkg/uixt/swipe.go | 5 ++++- hrp/step.go | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index f78279d4..39438cb1 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -66,6 +66,7 @@ type MobileAction struct { Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty"` // used to identify the action in log MaxRetryTimes int `json:"max_retry_times,omitempty" yaml:"max_retry_times,omitempty"` // max retry times + WaitTime float64 `json:"wait_time,omitempty" yaml:"wait_time,omitempty"` // wait time between swipe and ocr, unit: second Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty"` // used by swipe to tap text or app Scope []float64 `json:"scope,omitempty" yaml:"scope,omitempty"` // used by ocr to get text position in the scope Index int `json:"index,omitempty" yaml:"index,omitempty"` // index of the target element, should start from 1 @@ -90,6 +91,12 @@ func WithIndex(index int) ActionOption { } } +func WithWaitTime(sec float64) ActionOption { + return func(o *MobileAction) { + o.WaitTime = sec + } +} + // WithDirection inputs direction (up, down, left, right) func WithDirection(direction string) ActionOption { return func(o *MobileAction) { @@ -413,7 +420,7 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { action.MaxRetryTimes = 5 } // swipe next screen until app found - return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes) + return dExt.SwipeUntil("left", findApp, foundAppAction, action.MaxRetryTimes, action.WaitTime) } return fmt.Errorf("invalid %s params, should be app name(string), got %v", ACTION_SwipeToTapApp, action.Params) @@ -444,10 +451,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes) + return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes, action.WaitTime) } // swipe until found - return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes) + return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes, action.WaitTime) } return fmt.Errorf("invalid %s params, should be app text(string), got %v", ACTION_SwipeToTapText, action.Params) @@ -491,10 +498,10 @@ func (dExt *DriverExt) DoAction(action MobileAction) error { } if action.Direction != nil { - return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes) + return dExt.SwipeUntil(action.Direction, findText, foundTextAction, action.MaxRetryTimes, action.WaitTime) } // swipe until found - return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes) + return dExt.SwipeUntil("up", findText, foundTextAction, action.MaxRetryTimes, action.WaitTime) } return fmt.Errorf("invalid %s params, should be app text([]string), got %v", ACTION_SwipeToTapText, action.Params) diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 41a11d2a..b52bd770 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -2,6 +2,7 @@ package uixt import ( "fmt" + "time" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/rs/zerolog/log" @@ -66,7 +67,7 @@ type FindCondition func(driver *DriverExt) error // FoundAction indicates the action to do after a UI element is found type FoundAction func(driver *DriverExt) error -func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int) error { +func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition, action FoundAction, maxTimes int, waitTime float64) error { for i := 0; i < maxTimes; i++ { if err := condition(dExt); err == nil { // do action after found @@ -89,6 +90,8 @@ func (dExt *DriverExt) SwipeUntil(direction interface{}, condition FindCondition log.Error().Err(err).Msgf("swipe (%v, %v) to (%v, %v) failed", sx, sy, ex, ey) } } + // wait for swipe action to completed and content to load completely + time.Sleep(time.Duration(1000*waitTime) * time.Millisecond) } return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes) } diff --git a/hrp/step.go b/hrp/step.go index 1f3cb5ee..b721faea 100644 --- a/hrp/step.go +++ b/hrp/step.go @@ -23,6 +23,7 @@ const ( var ( WithIdentifier = uixt.WithIdentifier WithMaxRetryTimes = uixt.WithMaxRetryTimes + WithWaitTime = uixt.WithWaitTime WithIndex = uixt.WithIndex WithTimeout = uixt.WithTimeout WithIgnoreNotFoundError = uixt.WithIgnoreNotFoundError