From f0cafa9a9b2a3172dfb745dac13fd94cf0f6a667 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Tue, 6 Dec 2022 14:40:35 +0800 Subject: [PATCH] feat: icon matching of mobile ui automation --- hrp/pkg/uixt/im_vedem.go | 68 ++++++++++++++--------------------- hrp/pkg/uixt/im_vedem_test.go | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 41 deletions(-) create mode 100644 hrp/pkg/uixt/im_vedem_test.go diff --git a/hrp/pkg/uixt/im_vedem.go b/hrp/pkg/uixt/im_vedem.go index 49e232de..99d187e8 100644 --- a/hrp/pkg/uixt/im_vedem.go +++ b/hrp/pkg/uixt/im_vedem.go @@ -2,10 +2,11 @@ package uixt import ( "bytes" + "encoding/base64" "fmt" "image" + "io" "io/ioutil" - "mime/multipart" "net/http" "time" @@ -17,9 +18,15 @@ import ( "github.com/rs/zerolog/log" ) +type Box struct { + Point PointF `json:"point"` + Width float64 `json:"width"` + Height float64 `json:"height"` +} + type IMResult struct { - Image string `json:"image"` - Points []PointF `json:"points"` + Box Box `json:"box"` + Distance float64 `json:"distance"` } type IMResponse struct { @@ -50,49 +57,32 @@ func checkIMEnv() error { return nil } +func convertBase64(imgByte []byte) (baseImg string) { + return base64.StdEncoding.EncodeToString(imgByte) +} + func (s *veDEMIMService) getIMResult(searchImage []byte, sourceImage []byte) ([]IMResult, error) { - bodyBuf := &bytes.Buffer{} - bodyWriter := multipart.NewWriter(bodyBuf) - bodyWriter.WriteField("withDet", "true") - // bodyWriter.WriteField("timestampOnly", "true") - - formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png") - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("create form file error: %v", err)) - } - size, err := formWriter.Write(searchImage) - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("write form error: %v", err)) + data := map[string]interface{}{ + "sourceImage": convertBase64(sourceImage), + "targetImages": []string{convertBase64(searchImage)}, } - formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png") + // post json + dataBytes, err := json.Marshal(data) if err != nil { return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("create form file error: %v", err)) - } - _, err = formWriter.Write(sourceImage) - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("write form error: %v", err)) + fmt.Sprintf("json marshal error: %v", err)) } - err = bodyWriter.Close() - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("close body writer error: %v", err)) - } - - req, err := http.NewRequest("POST", env.VEDEM_IM_URL, bodyBuf) + req, err := http.NewRequest("POST", env.VEDEM_IM_URL, io.NopCloser(bytes.NewReader(dataBytes))) if err != nil { return nil, errors.Wrap(code.CVRequestError, fmt.Sprintf("construct request error: %v", err)) } - token := builtin.Sign("auth-v2", env.VEDEM_IM_AK, env.VEDEM_IM_SK, bodyBuf.Bytes()) + token := builtin.Sign("auth-v2", env.VEDEM_IM_AK, env.VEDEM_IM_SK, dataBytes) req.Header.Add("Agw-Auth", token) - req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + req.Header.Add("Content-Type", "application/json; charset=utf-8") var resp *http.Response // retry 3 times @@ -108,7 +98,6 @@ func (s *veDEMIMService) getIMResult(searchImage []byte, sourceImage []byte) ([] } log.Error().Err(err). Str("logID", logID). - Int("imageBufSize", size). Msgf("request CV service failed, retry %d", i) time.Sleep(1 * time.Second) } @@ -150,22 +139,19 @@ func (s *veDEMIMService) FindImage(byteSearch []byte, byteSource []byte, options } var rects []image.Rectangle - var cvImages []string for _, cvResult := range cvResults { rect = image.Rectangle{ // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 Min: image.Point{ - X: int(cvResult.Points[0].X), - Y: int(cvResult.Points[0].Y), + X: int(cvResult.Box.Point.X), + Y: int(cvResult.Box.Point.Y), }, Max: image.Point{ - X: int(cvResult.Points[2].X), - Y: int(cvResult.Points[2].Y), + X: int(cvResult.Box.Point.X + cvResult.Box.Width), + Y: int(cvResult.Box.Point.Y + cvResult.Box.Height), }, } if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { - cvImages = append(cvImages, cvResult.Image) - rects = append(rects, rect) // match exactly, and not specify index, return the first one diff --git a/hrp/pkg/uixt/im_vedem_test.go b/hrp/pkg/uixt/im_vedem_test.go new file mode 100644 index 00000000..35c3b9c4 --- /dev/null +++ b/hrp/pkg/uixt/im_vedem_test.go @@ -0,0 +1,62 @@ +package uixt + +import ( + "fmt" + "io/ioutil" + "os" + "testing" +) + +func checkIM(search []byte, source []byte) error { + service, err := newVEDEMIMService() + if err != nil { + return err + } + sdResults, err := service.getIMResult(search, source) + if err != nil { + return err + } + fmt.Println(sdResults) + return nil +} + +func TestIMWithScreenshot(t *testing.T) { + device, _ := NewIOSDevice(WithWDAPort(8700), WithWDAMjpegPort(8800)) + driver, err := device.NewUSBDriver(nil) + if err != nil { + t.Fatal(err) + } + + raw, err := driver.Screenshot() + if err != nil { + t.Fatal(err) + } + + imagePath := "~/Downloads/1669385239_validate_1669385367.png" + search, err := os.ReadFile(imagePath) + if err != nil { + t.Fatal(err) + } + + if err := checkIM(search, raw.Bytes()); err != nil { + t.Fatal(err) + } +} + +func TestIMWithLocalFile(t *testing.T) { + imagePath := "/Users/bytedance/Downloads/20221202-223440.png" + search, err := ioutil.ReadFile(imagePath) + if err != nil { + t.Fatal(err) + } + + sourcePath := "/Users/bytedance/Downloads/20221202-223432.jpeg" + file, err := ioutil.ReadFile(sourcePath) + if err != nil { + t.Fatal(err) + } + + if err := checkIM(search, file); err != nil { + t.Fatal(err) + } +}