diff --git a/hrp/internal/env/env.go b/hrp/internal/env/env.go index adaa9401..f3ac0345 100644 --- a/hrp/internal/env/env.go +++ b/hrp/internal/env/env.go @@ -22,6 +22,9 @@ var ( VEDEM_SD_URL = os.Getenv("VEDEM_SD_URL") VEDEM_SD_AK = os.Getenv("VEDEM_SD_AK") VEDEM_SD_SK = os.Getenv("VEDEM_SD_SK") + VEDEM_UI_URL = os.Getenv("VEDEM_UI_URL") + VEDEM_UI_AK = os.Getenv("VEDEM_UI_AK") + VEDEM_UI_SK = os.Getenv("VEDEM_UI_SK") DISABLE_GA = os.Getenv("DISABLE_GA") DISABLE_SENTRY = os.Getenv("DISABLE_SENTRY") PYPI_INDEX_URL = os.Getenv("PYPI_INDEX_URL") diff --git a/hrp/pkg/uixt/algorithm.go b/hrp/pkg/uixt/algorithm.go index 49da027d..21807bc3 100644 --- a/hrp/pkg/uixt/algorithm.go +++ b/hrp/pkg/uixt/algorithm.go @@ -192,6 +192,25 @@ func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOpt return } +func (dExt *DriverExt) FindDetectUIRectInUIKit(uiName string, options ...DataOption) (x, y, width, height float64, err error) { + var bufSource *bytes.Buffer + if bufSource, err = dExt.TakeScreenShotAfterAction(); err != nil { + return 0, 0, 0, 0, err + } + + service, err := newVEDEMUIService() + if err != nil { + return 0, 0, 0, 0, err + } + var rect image.Rectangle + rect, err = service.FindUI(uiName, bufSource.Bytes()) + if err != nil { + return 0, 0, 0, 0, err + } + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + type CVService interface { FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) } diff --git a/hrp/pkg/uixt/ocr_vedem.go b/hrp/pkg/uixt/ocr_vedem.go index ee392b17..5176a422 100644 --- a/hrp/pkg/uixt/ocr_vedem.go +++ b/hrp/pkg/uixt/ocr_vedem.go @@ -85,8 +85,10 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err fmt.Sprintf("construct request error: %v", err)) } - token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, bodyBuf.Bytes()) + signToken := "UNSIGNED-PAYLOAD" + token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, []byte(signToken)) req.Header.Add("Agw-Auth", token) + req.Header.Add("Agw-Auth-Content", signToken) req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) var resp *http.Response @@ -165,8 +167,8 @@ func (t OCRTexts) Texts() (texts []string) { } func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption) ( - ocrTexts OCRTexts, err error) { - + ocrTexts OCRTexts, err error, +) { ocrResults, err := s.getOCRResult(imageBuf) if err != nil { log.Error().Err(err).Msg("getOCRResult failed") @@ -204,8 +206,8 @@ func (s *veDEMOCRService) GetTexts(imageBuf *bytes.Buffer, options ...DataOption } func (s *veDEMOCRService) FindText(text string, imageBuf *bytes.Buffer, options ...DataOption) ( - rect image.Rectangle, err error) { - + rect image.Rectangle, err error, +) { ocrTexts, err := s.GetTexts(imageBuf, options...) if err != nil { log.Error().Err(err).Msg("GetTexts failed") @@ -260,8 +262,8 @@ func (s *veDEMOCRService) FindText(text string, imageBuf *bytes.Buffer, options } func (s *veDEMOCRService) FindTexts(texts []string, imageBuf *bytes.Buffer, options ...DataOption) ( - rects []image.Rectangle, err error) { - + rects []image.Rectangle, err error, +) { ocrTexts, err := s.GetTexts(imageBuf, options...) if err != nil { log.Error().Err(err).Msg("GetTexts failed") diff --git a/hrp/pkg/uixt/tap.go b/hrp/pkg/uixt/tap.go index 56b48989..5898ba7f 100644 --- a/hrp/pkg/uixt/tap.go +++ b/hrp/pkg/uixt/tap.go @@ -63,12 +63,18 @@ func (dExt *DriverExt) GetTextXYs(ocrText []string, options ...DataOption) (poin return points, nil } -func (dExt *DriverExt) GetImageXY(imagePath string, options ...DataOption) (point PointF, err error) { +func (dExt *DriverExt) GetImageXY(imageParam string, options ...DataOption) (point PointF, err error) { // close popup if necessary if dExt.ClosePopup { dExt.ClosePopupHandler() } - x, y, width, height, err := dExt.FindImageRectInUIKit(imagePath, options...) + var x, y, width, height float64 + switch imageParam { + case ShoppingBag, DyHouse: + x, y, width, height, err = dExt.FindDetectUIRectInUIKit(imageParam, options...) + default: + x, y, width, height, err = dExt.FindImageRectInUIKit(imageParam, options...) + } if err != nil { return PointF{}, err } @@ -147,7 +153,6 @@ func (dExt *DriverExt) DoubleTap(param string) (err error) { } func (dExt *DriverExt) DoubleTapOffset(param string, xOffset, yOffset float64) (err error) { - // close popup if necessary if dExt.ClosePopup { dExt.ClosePopupHandler() diff --git a/hrp/pkg/uixt/ui_vedem.go b/hrp/pkg/uixt/ui_vedem.go new file mode 100644 index 00000000..b35df3e7 --- /dev/null +++ b/hrp/pkg/uixt/ui_vedem.go @@ -0,0 +1,200 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "io/ioutil" + "mime/multipart" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" + "github.com/httprunner/httprunner/v4/hrp/internal/env" + "github.com/httprunner/httprunner/v4/hrp/internal/json" +) + +const ( + ShoppingBag = "shoppingbag" + DyHouse = "dyhouse" +) + +type UIResult map[string][]Box + +type UIResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result UIResult `json:"result"` +} + +type veDEMUIService struct{} + +func newVEDEMUIService() (*veDEMUIService, error) { + if err := checkUIEnv(); err != nil { + return nil, err + } + return &veDEMUIService{}, nil +} + +func checkUIEnv() error { + if env.VEDEM_UI_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_UI_URL missed") + } + if env.VEDEM_UI_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_UI_AK missed") + } + if env.VEDEM_UI_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_UI_SK missed") + } + return nil +} + +func (s *veDEMUIService) getUIResult(uiType string, sourceImage []byte) (UIResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("types", uiType) + + formWriter, err := bodyWriter.CreateFormFile("image", "screenshot.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + size, err := formWriter.Write(sourceImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + err = bodyWriter.Close() + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + } + + req, err := http.NewRequest("POST", env.VEDEM_UI_URL, bodyBuf) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + } + + signToken := "UNSIGNED-PAYLOAD" + token := builtin.Sign("auth-v2", env.VEDEM_OCR_AK, env.VEDEM_OCR_SK, []byte(signToken)) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Agw-Auth-Content", signToken) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + req.Header.Add("x-tt-env", "boe_0529162338") + + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + resp, err = client.Do(req) + var logID string + if resp != nil { + logID = getLogID(resp.Header) + } + if err == nil && resp.StatusCode == http.StatusOK { + log.Debug(). + Str("X-TT-LOGID", logID). + Int("imageBufSize", size). + Msg("request UI service success") + break + } + log.Error().Err(err). + Str("X-TT-LOGID", logID). + Int("imageBufSize", size). + Msgf("request UI service failed, retry %d", i) + time.Sleep(1 * time.Second) + } + if resp == nil { + return nil, code.CVServiceConnectionError + } + + defer resp.Body.Close() + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + } + + var uiResult UIResponse + err = json.Unmarshal(results, &uiResult) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("json unmarshal response body error: %v", err)) + } + + return uiResult.Result, nil +} + +func (s *veDEMUIService) FindUI(uiType string, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := NewDataOptions(options...) + + uiResultMap, err := s.getUIResult(uiType, byteSource) + if err != nil { + log.Error().Err(err).Msg("getUIResult failed") + return + } + + uiResult, ok := uiResultMap[uiType] + if !ok { + err = fmt.Errorf("UI type %v not detected", uiResult) + log.Error().Err(err).Msg("getUIResult failed") + return + } + + var rects []image.Rectangle + for _, box := range uiResult { + rect = image.Rectangle{ + // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(box.Point.X), + Y: int(box.Point.Y), + }, + Max: image.Point{ + X: int(box.Point.X + box.Width), + Y: int(box.Point.Y + box.Height), + }, + } + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { + rects = append(rects, rect) + + // match exactly, and not specify index, return the first one + if data.Index == 0 { + return rect, nil + } + } + } + + if len(rects) == 0 { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image not found")) + } + + // get index + idx := data.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image found, index %d out of range", idx)) + } + + return rects[idx], nil +} diff --git a/hrp/pkg/uixt/ui_vedem_test.go b/hrp/pkg/uixt/ui_vedem_test.go new file mode 100644 index 00000000..f373ed80 --- /dev/null +++ b/hrp/pkg/uixt/ui_vedem_test.go @@ -0,0 +1,32 @@ +package uixt + +import ( + "fmt" + "io/ioutil" + "testing" +) + +func checkUI(uiName string, source []byte) error { + service, err := newVEDEMUIService() + if err != nil { + return err + } + uiResults, err := service.getUIResult(uiName, source) + if err != nil { + return err + } + fmt.Println(uiResults) + return nil +} + +func TestUIWithLocalFile(t *testing.T) { + sourcePath := "/Users/bytedance/Desktop/lifeservice.png" + file, err := ioutil.ReadFile(sourcePath) + if err != nil { + t.Fatal(err) + } + + if err := checkUI("dyhouse", file); err != nil { + t.Fatal(err) + } +}