From 1ecea74214c3664578d4c3a84946d9e4c7c60913 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 26 Sep 2022 14:47:19 +0800 Subject: [PATCH] fix: screenshot --- hrp/internal/uixt/android_device.go | 12 ++++++------ hrp/internal/uixt/ext.go | 10 +++++++--- hrp/internal/uixt/ios_device.go | 24 ++++++++++++------------ hrp/internal/uixt/ios_driver.go | 4 ++-- hrp/internal/uixt/ocr_on.go | 11 +++++++---- hrp/internal/uixt/ocr_test.go | 2 +- hrp/internal/uixt/swipe.go | 3 --- 7 files changed, 35 insertions(+), 31 deletions(-) diff --git a/hrp/internal/uixt/android_device.go b/hrp/internal/uixt/android_device.go index ae394be4..f00880ad 100644 --- a/hrp/internal/uixt/android_device.go +++ b/hrp/internal/uixt/android_device.go @@ -101,7 +101,7 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er } device.SerialNumber = dev.Serial() - device.Device = dev + device.d = dev return device, nil } @@ -109,7 +109,7 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er } type AndroidDevice struct { - gadb.Device + d gadb.Device SerialNumber string `json:"serial,omitempty" yaml:"serial,omitempty"` IP string `json:"ip,omitempty" yaml:"ip,omitempty"` Port int `json:"port,omitempty" yaml:"port,omitempty"` @@ -136,17 +136,17 @@ func (dev *AndroidDevice) NewUSBDriver(capabilities Capabilities) (driver *uiaDr if localPort, err = getFreePort(); err != nil { return nil, err } - if err = dev.Forward(localPort, UIA2ServerPort); err != nil { + if err = dev.d.Forward(localPort, UIA2ServerPort); err != nil { return nil, err } rawURL := fmt.Sprintf("http://%s%d:6790/wd/hub", forwardToPrefix, localPort) driver, err = NewUIADriver(capabilities, rawURL) if err != nil { - _ = dev.ForwardKill(localPort) + _ = dev.d.ForwardKill(localPort) return nil, err } - driver.adbDevice = dev.Device + driver.adbDevice = dev.d driver.localPort = localPort conn, err := net.Dial("tcp", fmt.Sprintf(":%d", localPort)) @@ -165,7 +165,7 @@ func (dev *AndroidDevice) NewHTTPDriver(capabilities Capabilities) (driver *uiaD if driver, err = NewUIADriver(capabilities, rawURL); err != nil { return nil, err } - driver.adbDevice = dev.Device + driver.adbDevice = dev.d return driver, nil } diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go index 5eb7d0d1..1b44922b 100644 --- a/hrp/internal/uixt/ext.go +++ b/hrp/internal/uixt/ext.go @@ -152,10 +152,10 @@ func (dExt *DriverExt) takeScreenShot() (raw *bytes.Buffer, err error) { return dExt.frame, nil } if raw, err = dExt.Driver.Screenshot(); err != nil { - log.Error().Err(err).Msgf("screenshot failed: %v", err) + log.Error().Err(err).Msg("takeScreenShot failed") return nil, err } - return + return raw, nil } // saveScreenShot saves image file to $CWD/screenshots/ folder @@ -203,7 +203,11 @@ func (dExt *DriverExt) ScreenShot(fileName string) (string, error) { return "", errors.Wrap(err, "screenshot by WDA failed") } - return dExt.saveScreenShot(raw, fileName) + path, err := dExt.saveScreenShot(raw, fileName) + if err != nil { + return "", errors.Wrap(err, "save screenshot failed") + } + return path, nil } // isPathExists returns true if path exists, whether path is file or dir diff --git a/hrp/internal/uixt/ios_device.go b/hrp/internal/uixt/ios_device.go index 9a243ce3..d71abd5a 100644 --- a/hrp/internal/uixt/ios_device.go +++ b/hrp/internal/uixt/ios_device.go @@ -64,7 +64,7 @@ func InitWDAClient(device *IOSDevice) (*DriverExt, error) { // aviod getting stuck when some super app is activate such as douyin or wexin log.Info().Msg("switch to iOS springboard") bundleID := "com.apple.springboard" - _, err = iosDevice.AppLaunch(bundleID) + _, err = iosDevice.d.AppLaunch(bundleID) if err != nil { return nil, errors.Wrap(err, "launch springboard failed") } @@ -153,7 +153,7 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) { } device.UDID = dev.Properties().SerialNumber - device.Device = dev + device.d = dev return device, nil } @@ -161,7 +161,7 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) { } type IOSDevice struct { - giDevice.Device + d giDevice.Device UDID string `json:"udid,omitempty" yaml:"udid,omitempty"` Port int `json:"port,omitempty" yaml:"port,omitempty"` MjpegPort int `json:"mjpeg_port,omitempty" yaml:"mjpeg_port,omitempty"` @@ -203,12 +203,12 @@ func (dev *IOSDevice) NewHTTPDriver(capabilities Capabilities) (driver WebDriver func (dev *IOSDevice) NewUSBDriver(capabilities Capabilities) (driver WebDriver, err error) { wd := new(wdaDriver) - if wd.defaultConn, err = dev.NewConnect(dev.Port, 0); err != nil { + if wd.defaultConn, err = dev.d.NewConnect(dev.Port, 0); err != nil { return nil, fmt.Errorf("create connection: %w", err) } wd.client = convertToHTTPClient(wd.defaultConn.RawConn()) - if wd.mjpegUSBConn, err = dev.NewConnect(dev.MjpegPort, 0); err != nil { + if wd.mjpegUSBConn, err = dev.d.NewConnect(dev.MjpegPort, 0); err != nil { return nil, fmt.Errorf("create connection MJPEG: %w", err) } wd.mjpegClient = convertToHTTPClient(wd.mjpegUSBConn.RawConn()) @@ -377,7 +377,7 @@ func (r rawResponse) checkErr() (err error) { func (r rawResponse) valueConvertToString() (s string, err error) { reply := new(struct{ Value string }) if err = json.Unmarshal(r, reply); err != nil { - return "", err + return "", errors.Wrapf(err, "json.Unmarshal failed, rawResponse: %s", string(r)) } s = reply.Value return @@ -411,13 +411,13 @@ func (r rawResponse) valueConvertToJsonRawMessage() (raw builtinJSON.RawMessage, } func (r rawResponse) valueDecodeAsBase64() (raw *bytes.Buffer, err error) { - var str string - if str, err = r.valueConvertToString(); err != nil { - return nil, err + str, err := r.valueConvertToString() + if err != nil { + return nil, errors.Wrap(err, "failed to convert value to string") } - var decodeString []byte - if decodeString, err = base64.StdEncoding.DecodeString(str); err != nil { - return nil, err + decodeString, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return nil, errors.Wrap(err, "failed to decode base64 string") } raw = bytes.NewBuffer(decodeString) return diff --git a/hrp/internal/uixt/ios_driver.go b/hrp/internal/uixt/ios_driver.go index e99e33bc..9e5eda75 100644 --- a/hrp/internal/uixt/ios_driver.go +++ b/hrp/internal/uixt/ios_driver.go @@ -699,11 +699,11 @@ func (wd *wdaDriver) Screenshot() (raw *bytes.Buffer, err error) { // [[FBRoute GET:@"/screenshot"].withoutSession respondWithTarget:self action:@selector(handleGetScreenshot:)] var rawResp rawResponse if rawResp, err = wd.httpGET("/session", wd.sessionId, "/screenshot"); err != nil { - return nil, err + return nil, errors.Wrap(err, "get WDA screenshot data failed") } if raw, err = rawResp.valueDecodeAsBase64(); err != nil { - return nil, err + return nil, errors.Wrap(err, "decode WDA screenshot data failed") } return } diff --git a/hrp/internal/uixt/ocr_on.go b/hrp/internal/uixt/ocr_on.go index 5809020b..282780de 100644 --- a/hrp/internal/uixt/ocr_on.go +++ b/hrp/internal/uixt/ocr_on.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "github.com/rs/zerolog/log" + "github.com/httprunner/httprunner/v4/hrp/internal/json" ) @@ -21,8 +23,8 @@ var client = &http.Client{ } type OCRResult struct { - Text string `json:"text"` - Points []Point `json:"points"` + Text string `json:"text"` + Points []PointF `json:"points"` } type ResponseOCR struct { @@ -134,14 +136,15 @@ type OCRService interface { func (dExt *DriverExt) FindTextByOCR(ocrText string) (x, y, width, height float64, err error) { var bufSource *bytes.Buffer if bufSource, err = dExt.takeScreenShot(); err != nil { - err = fmt.Errorf("screenshot error: %v", err) + err = fmt.Errorf("takeScreenShot error: %v", err) return } service := &veDEMOCRService{} rect, err := service.FindText(ocrText, bufSource.Bytes()) if err != nil { - err = fmt.Errorf("find text failed: %v", err) + log.Warn().Err(err).Msg("FindText failed") + err = fmt.Errorf("FindText failed: %v", err) return } diff --git a/hrp/internal/uixt/ocr_test.go b/hrp/internal/uixt/ocr_test.go index 922c9e0b..59b19160 100644 --- a/hrp/internal/uixt/ocr_test.go +++ b/hrp/internal/uixt/ocr_test.go @@ -7,7 +7,7 @@ import ( ) func TestDriverExtOCR(t *testing.T) { - driverExt, err := InitWDAClient() + driverExt, err := InitWDAClient(nil) checkErr(t, err) x, y, width, height, err := driverExt.FindTextByOCR("抖音") diff --git a/hrp/internal/uixt/swipe.go b/hrp/internal/uixt/swipe.go index 7973b53a..c329263d 100644 --- a/hrp/internal/uixt/swipe.go +++ b/hrp/internal/uixt/swipe.go @@ -2,7 +2,6 @@ package uixt import ( "fmt" - "time" "github.com/rs/zerolog/log" ) @@ -82,8 +81,6 @@ func (dExt *DriverExt) SwipeUntil(direction string, condition FindCondition, act if err := dExt.SwipeTo(direction); err != nil { log.Error().Err(err).Msgf("swipe %s failed", direction) } - // wait for swipe done - time.Sleep(500 * time.Millisecond) } return fmt.Errorf("swipe %s %d times, match condition failed", direction, maxTimes) }