From b9633638fa2625be6f6d8e6548a94d17eec87ff3 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 23 Nov 2022 23:22:51 +0800 Subject: [PATCH] refactor: NewDataOptions and NewData --- hrp/pkg/uixt/android_driver.go | 24 ++++++------- hrp/pkg/uixt/android_elment.go | 4 +-- hrp/pkg/uixt/demo/main_test.go | 28 ++++++++++++++++ hrp/pkg/uixt/ext.go | 5 +++ hrp/pkg/uixt/interface.go | 39 +++++++++++++--------- hrp/pkg/uixt/ios_driver.go | 16 ++++----- hrp/pkg/uixt/ios_element.go | 4 +-- hrp/pkg/uixt/ocr_vedem.go | 61 +++++++++++++++++++--------------- hrp/pkg/uixt/swipe.go | 12 +++---- hrp/pkg/uixt/tap.go | 12 +++---- 10 files changed, 127 insertions(+), 78 deletions(-) diff --git a/hrp/pkg/uixt/android_driver.go b/hrp/pkg/uixt/android_driver.go index 5ebf69dd..3bbc37c4 100644 --- a/hrp/pkg/uixt/android_driver.go +++ b/hrp/pkg/uixt/android_driver.go @@ -508,9 +508,9 @@ func (ud *uiaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "y": y, } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "appium/tap") + _, err = ud.httpPOST(newData, "/session", ud.sessionId, "appium/tap") return } @@ -566,9 +566,9 @@ func (ud *uiaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - return ud._drag(d.Data) + return ud._drag(newData) } func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...DataOption) (err error) { @@ -581,9 +581,9 @@ func (ud *uiaDriver) _swipe(startX, startY, endX, endY interface{}, options ...D } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "touch/perform") + _, err = ud.httpPOST(newData, "/session", ud.sessionId, "touch/perform") return } @@ -672,9 +672,9 @@ func (ud *uiaDriver) SendKeys(text string, options ...DataOption) (err error) { "text": text, } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = ud.httpPOST(d.Data, "/session", ud.sessionId, "keys") + _, err = ud.httpPOST(newData, "/session", ud.sessionId, "keys") return } @@ -683,14 +683,14 @@ func (ud *uiaDriver) Input(text string, options ...DataOption) (err error) { "view": text, } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) var element WebElement - if valuetext, ok := d.Data["textview"]; ok { + if valuetext, ok := newData["textview"]; ok { element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().TextContains(fmt.Sprintf("%v", valuetext)).String()}) - } else if valueid, ok := d.Data["id"]; ok { + } else if valueid, ok := newData["id"]; ok { element, err = ud.FindElement(BySelector{ResourceIdID: fmt.Sprintf("%v", valueid)}) - } else if valuedesc, ok := d.Data["description"]; ok { + } else if valuedesc, ok := newData["description"]; ok { element, err = ud.FindElement(BySelector{UiAutomator: NewUiSelectorHelper().Description(fmt.Sprintf("%v", valuedesc)).String()}) } else { element, err = ud.FindElement(BySelector{ClassName: ElementType{EditText: true}}) diff --git a/hrp/pkg/uixt/android_elment.go b/hrp/pkg/uixt/android_elment.go index 9d45cd39..760f52ab 100644 --- a/hrp/pkg/uixt/android_elment.go +++ b/hrp/pkg/uixt/android_elment.go @@ -30,9 +30,9 @@ func (ue uiaElement) SendKeys(text string, options ...DataOption) (err error) { } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = ue.parent.httpPOST(d.Data, "/session", ue.parent.sessionId, "/element", ue.id, "/value") + _, err = ue.parent.httpPOST(newData, "/session", ue.parent.sessionId, "/element", ue.id, "/value") return } diff --git a/hrp/pkg/uixt/demo/main_test.go b/hrp/pkg/uixt/demo/main_test.go index 1ff036ce..de169814 100644 --- a/hrp/pkg/uixt/demo/main_test.go +++ b/hrp/pkg/uixt/demo/main_test.go @@ -3,6 +3,7 @@ package demo import ( + "fmt" "testing" "time" @@ -44,3 +45,30 @@ func TestIOSDemo(t *testing.T) { } } } + +func TestIOSGetOCRTexts(t *testing.T) { + device, err := uixt.NewIOSDevice( + uixt.WithWDAPort(8700), uixt.WithWDAMjpegPort(8800), + uixt.WithResetHomeOnStartup(false), // not reset home on startup + ) + if err != nil { + t.Fatal(err) + } + + capabilities := uixt.NewCapabilities() + capabilities.WithDefaultAlertAction(uixt.AlertActionAccept) // or uixt.AlertActionDismiss + driverExt, err := device.NewDriver(capabilities) + if err != nil { + t.Fatal(err) + } + + for { + texts, err := driverExt.GetTextsByOCR() + if err != nil { + t.Fatal(err) + } + + fmt.Println(texts) + time.Sleep(3 * time.Second) + } +} diff --git a/hrp/pkg/uixt/ext.go b/hrp/pkg/uixt/ext.go index 531f95b8..88f6ab50 100644 --- a/hrp/pkg/uixt/ext.go +++ b/hrp/pkg/uixt/ext.go @@ -205,6 +205,7 @@ type DriverExt struct { frame *bytes.Buffer doneMjpegStream chan bool scale float64 + ocrService OCRService // used to get text from image StartTime time.Time // used to associate screenshots name ScreenShots []string // save screenshots path perfStop chan struct{} // stop performance monitor @@ -227,6 +228,10 @@ func extend(driver WebDriver) (dExt *DriverExt, err error) { return nil, err } + if dExt.ocrService, err = newVEDEMOCRService(); err != nil { + return nil, err + } + return dExt, nil } diff --git a/hrp/pkg/uixt/interface.go b/hrp/pkg/uixt/interface.go index 77ae84cb..5dfba32a 100644 --- a/hrp/pkg/uixt/interface.go +++ b/hrp/pkg/uixt/interface.go @@ -860,13 +860,8 @@ func WithDataWaitTime(sec float64) DataOption { } } -func NewData(data map[string]interface{}, options ...DataOption) *DataOptions { - if data == nil { - data = make(map[string]interface{}) - } - dataOptions := &DataOptions{ - Data: data, - } +func NewDataOptions(options ...DataOption) *DataOptions { + dataOptions := &DataOptions{} for _, option := range options { option(dataOptions) } @@ -874,7 +869,18 @@ func NewData(data map[string]interface{}, options ...DataOption) *DataOptions { if len(dataOptions.Scope) == 0 { dataOptions.Scope = []int{0, 0, math.MaxInt64, math.MaxInt64} // default scope } + return dataOptions +} +func NewData(data map[string]interface{}, options ...DataOption) map[string]interface{} { + dataOptions := NewDataOptions(options...) + + // merge with data options + for k, v := range dataOptions.Data { + data[k] = v + } + + // handle point offset if len(dataOptions.Offset) == 2 { if x, ok := data["x"]; ok { xf, _ := builtin.Interface2Float64(x) @@ -886,23 +892,24 @@ func NewData(data map[string]interface{}, options ...DataOption) *DataOptions { } } - if _, ok := dataOptions.Data["steps"]; !ok { - dataOptions.Data["steps"] = 12 // default steps + // add default options + if _, ok := data["steps"]; !ok { + data["steps"] = 12 // default steps } - if _, ok := dataOptions.Data["duration"]; !ok { - dataOptions.Data["duration"] = 0 // default duration + if _, ok := data["duration"]; !ok { + data["duration"] = 0 // default duration } - if _, ok := dataOptions.Data["frequency"]; !ok { - dataOptions.Data["frequency"] = 60 // default frequency + if _, ok := data["frequency"]; !ok { + data["frequency"] = 60 // default frequency } - if _, ok := dataOptions.Data["isReplace"]; !ok { - dataOptions.Data["isReplace"] = true // default true + if _, ok := data["isReplace"]; !ok { + data["isReplace"] = true // default true } - return dataOptions + return data } // current implemeted device: IOSDevice, AndroidDevice diff --git a/hrp/pkg/uixt/ios_driver.go b/hrp/pkg/uixt/ios_driver.go index a14c08c8..03f3e213 100644 --- a/hrp/pkg/uixt/ios_driver.go +++ b/hrp/pkg/uixt/ios_driver.go @@ -381,9 +381,9 @@ func (wd *wdaDriver) TapFloat(x, y float64, options ...DataOption) (err error) { "y": y, } // new data options in post data for extra WDA configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/tap/0") + _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/tap/0") return } @@ -433,9 +433,9 @@ func (wd *wdaDriver) DragFloat(fromX, fromY, toX, toY float64, options ...DataOp } // new data options in post data for extra WDA configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } @@ -506,9 +506,9 @@ func (wd *wdaDriver) SendKeys(text string, options ...DataOption) (err error) { data := map[string]interface{}{"value": strings.Split(text, "")} // new data options in post data for extra WDA configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/keys") + _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/keys") return } @@ -541,9 +541,9 @@ func (wd *wdaDriver) PressBack(options ...DataOption) (err error) { } // new data options in post data for extra WDA configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = wd.httpPOST(d.Data, "/session", wd.sessionId, "/wda/dragfromtoforduration") + _, err = wd.httpPOST(newData, "/session", wd.sessionId, "/wda/dragfromtoforduration") return } diff --git a/hrp/pkg/uixt/ios_element.go b/hrp/pkg/uixt/ios_element.go index 9e2208b5..02f54aac 100644 --- a/hrp/pkg/uixt/ios_element.go +++ b/hrp/pkg/uixt/ios_element.go @@ -31,9 +31,9 @@ func (we wdaElement) SendKeys(text string, options ...DataOption) (err error) { "value": strings.Split(text, ""), } // new data options in post data for extra uiautomator configurations - d := NewData(data, options...) + newData := NewData(data, options...) - _, err = we.parent.httpPOST(d.Data, "/session", we.parent.sessionId, "/element", we.id, "/value") + _, err = we.parent.httpPOST(newData, "/session", we.parent.sessionId, "/element", we.id, "/value") return } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index 542fd711..06a6dec1 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -154,15 +154,14 @@ type OCRText struct { type OCRTexts []OCRText -func (t OCRTexts) Texts() []string { - var texts []string +func (t OCRTexts) Texts() (texts []string) { for _, text := range t { texts = append(texts, text.Text) } return texts } -func (s *veDEMOCRService) GetAllTexts(imageBuf []byte) ( +func (s *veDEMOCRService) GetTexts(imageBuf []byte, options ...DataOption) ( ocrTexts OCRTexts, err error) { ocrResults, err := s.getOCRResult(imageBuf) @@ -194,21 +193,21 @@ func (s *veDEMOCRService) GetAllTexts(imageBuf []byte) ( func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...DataOption) ( rect image.Rectangle, err error) { - ocrTexts, err := s.GetAllTexts(imageBuf) + ocrTexts, err := s.GetTexts(imageBuf) if err != nil { - log.Error().Err(err).Msg("GetAllTexts failed") + log.Error().Err(err).Msg("GetTexts failed") return } - data := NewData(map[string]interface{}{}, options...) + dataOptions := NewDataOptions(options...) var rects []image.Rectangle for _, ocrText := range ocrTexts { rect = ocrText.Rect // check if text in scope - if rect.Min.X < data.Scope[0] || rect.Max.X > data.Scope[2] || - rect.Min.Y < data.Scope[1] || rect.Max.Y > data.Scope[3] { + if rect.Min.X < dataOptions.Scope[0] || rect.Max.X > dataOptions.Scope[2] || + rect.Min.Y < dataOptions.Scope[1] || rect.Max.Y > dataOptions.Scope[3] { // not in scope continue } @@ -226,7 +225,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // match exactly, and not specify index, return the first one - if data.Index == 0 { + if dataOptions.Index == 0 { return rect, nil } } @@ -237,7 +236,7 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data } // get index - idx := data.Index + idx := dataOptions.Index if idx > 0 { // NOTICE: index start from 1 idx = idx - 1 @@ -257,13 +256,13 @@ func (s *veDEMOCRService) FindText(text string, imageBuf []byte, options ...Data func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ...DataOption) ( rects []image.Rectangle, err error) { - ocrTexts, err := s.GetAllTexts(imageBuf) + ocrTexts, err := s.GetTexts(imageBuf) if err != nil { - log.Error().Err(err).Msg("GetAllTexts failed") + log.Error().Err(err).Msg("GetTexts failed") return } - data := NewData(map[string]interface{}{}, options...) + dataOptions := NewDataOptions(options...) var success bool for _, text := range texts { @@ -272,8 +271,8 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... rect := ocrText.Rect // check if text in scope - if rect.Min.X < data.Scope[0] || rect.Max.X > data.Scope[2] || - rect.Min.Y < data.Scope[1] || rect.Max.Y > data.Scope[3] { + if rect.Min.X < dataOptions.Scope[0] || rect.Max.X > dataOptions.Scope[2] || + rect.Min.Y < dataOptions.Scope[1] || rect.Max.Y > dataOptions.Scope[3] { // not in scope continue } @@ -302,7 +301,25 @@ func (s *veDEMOCRService) FindTexts(texts []string, imageBuf []byte, options ... } type OCRService interface { - FindText(text string, imageBuf []byte, index ...int) (rect image.Rectangle, err error) + GetTexts(imageBuf []byte, options ...DataOption) (ocrTexts OCRTexts, err error) + FindText(text string, imageBuf []byte, options ...DataOption) (rect image.Rectangle, err error) + FindTexts(texts []string, imageBuf []byte, options ...DataOption) (rects []image.Rectangle, err error) +} + +func (dExt *DriverExt) GetTextsByOCR() (texts OCRTexts, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.takeScreenShot(); err != nil { + err = fmt.Errorf("takeScreenShot error: %v", err) + return + } + + ocrTexts, err := dExt.ocrService.GetTexts(bufSource.Bytes()) + if err != nil { + log.Error().Err(err).Msg("GetTexts failed") + return + } + + return ocrTexts, nil } func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, y, width, height float64, err error) { @@ -312,11 +329,7 @@ func (dExt *DriverExt) FindTextByOCR(ocrText string, options ...DataOption) (x, return } - service, err := newVEDEMOCRService() - if err != nil { - return - } - rect, err := service.FindText(ocrText, bufSource.Bytes(), options...) + rect, err := dExt.ocrService.FindText(ocrText, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindText failed: %s", err.Error()) return @@ -335,11 +348,7 @@ func (dExt *DriverExt) FindTextsByOCR(ocrTexts []string, options ...DataOption) return } - service, err := newVEDEMOCRService() - if err != nil { - return - } - rects, err := service.FindTexts(ocrTexts, bufSource.Bytes(), options...) + rects, err := dExt.ocrService.FindTexts(ocrTexts, bufSource.Bytes(), options...) if err != nil { log.Warn().Msgf("FindTexts failed: %s", err.Error()) return diff --git a/hrp/pkg/uixt/swipe.go b/hrp/pkg/uixt/swipe.go index 4c4e865f..b1f8f6d0 100644 --- a/hrp/pkg/uixt/swipe.go +++ b/hrp/pkg/uixt/swipe.go @@ -69,9 +69,9 @@ type Action func(driver *DriverExt) error // findCondition indicates the condition to find a UI element // foundAction indicates the action to do after a UI element is found func (dExt *DriverExt) SwipeUntil(direction interface{}, findCondition Action, foundAction Action, options ...DataOption) error { - d := NewData(nil, options...) - maxRetryTimes := d.MaxRetryTimes - interval := d.Interval + dataOptions := NewDataOptions(options...) + maxRetryTimes := dataOptions.MaxRetryTimes + interval := dataOptions.Interval for i := 0; i < maxRetryTimes; i++ { if err := findCondition(dExt); err == nil { @@ -103,9 +103,9 @@ func (dExt *DriverExt) SwipeUntil(direction interface{}, findCondition Action, f } func (dExt *DriverExt) LoopUntil(findAction, findCondition, foundAction Action, options ...DataOption) error { - d := NewData(nil, options...) - maxRetryTimes := d.MaxRetryTimes - interval := d.Interval + dataOptions := NewDataOptions(options...) + maxRetryTimes := dataOptions.MaxRetryTimes + interval := dataOptions.Interval for i := 0; i < maxRetryTimes; i++ { if err := findCondition(dExt); err == nil { diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 1629f137..3d4b584a 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -65,11 +65,11 @@ func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (poin } func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { - data := NewData(map[string]interface{}{}, options...) + dataOptions := NewDataOptions(options...) point, err := dExt.GetTextXY(ocrText, options...) if err != nil { - if data.IgnoreNotFoundError { + if dataOptions.IgnoreNotFoundError { return nil } return err @@ -79,11 +79,11 @@ func (dExt *DriverExt) TapByOCR(ocrText string, options ...DataOption) error { } func (dExt *DriverExt) TapByCV(imagePath string, options ...DataOption) error { - data := NewData(map[string]interface{}{}, options...) + dataOptions := NewDataOptions(options...) point, err := dExt.GetImageXY(imagePath, options...) if err != nil { - if data.IgnoreNotFoundError { + if dataOptions.IgnoreNotFoundError { return nil } return err @@ -103,11 +103,11 @@ func (dExt *DriverExt) TapOffset(param string, xOffset, yOffset float64, options return ele.Click() } - data := NewData(map[string]interface{}{}, options...) + dataOptions := NewDataOptions(options...) x, y, width, height, err := dExt.FindUIRectInUIKit(param, options...) if err != nil { - if data.IgnoreNotFoundError { + if dataOptions.IgnoreNotFoundError { return nil } return err