From c467b55c72064161d1019e0547a798fa562194a2 Mon Sep 17 00:00:00 2001 From: "xucong.053" Date: Thu, 24 Nov 2022 14:49:25 +0800 Subject: [PATCH] feat: support closing app popup window by cv service --- hrp/internal/env/env.go | 3 + hrp/pkg/uixt/cp_vedem.go | 186 ++++++++++++++++++++ hrp/pkg/uixt/{cv_vedem.go => im_vedem.go} | 172 +------------------ hrp/pkg/uixt/sd_vedem.go | 199 ++++++++++++++++++++++ 4 files changed, 390 insertions(+), 170 deletions(-) create mode 100644 hrp/pkg/uixt/cp_vedem.go rename hrp/pkg/uixt/{cv_vedem.go => im_vedem.go} (54%) create mode 100644 hrp/pkg/uixt/sd_vedem.go diff --git a/hrp/internal/env/env.go b/hrp/internal/env/env.go index dda2d1e9..7967cd8c 100644 --- a/hrp/internal/env/env.go +++ b/hrp/internal/env/env.go @@ -19,6 +19,9 @@ var ( VEDEM_CP_URL = os.Getenv("VEDEM_CP_URL") VEDEM_CP_AK = os.Getenv("VEDEM_CP_AK") VEDEM_CP_SK = os.Getenv("VEDEM_CP_SK") + VEDEM_SD_URL = os.Getenv("VEDEM_SD_URL") + VEDEM_SD_AK = os.Getenv("VEDEM_SD_AK") + VEDEM_SD_SK = os.Getenv("VEDEM_SD_SK") DISABLE_GA = os.Getenv("DISABLE_GA") DISABLE_SENTRY = os.Getenv("DISABLE_SENTRY") PYPI_INDEX_URL = os.Getenv("PYPI_INDEX_URL") diff --git a/hrp/pkg/uixt/cp_vedem.go b/hrp/pkg/uixt/cp_vedem.go new file mode 100644 index 00000000..73438b55 --- /dev/null +++ b/hrp/pkg/uixt/cp_vedem.go @@ -0,0 +1,186 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "io/ioutil" + "mime/multipart" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" + "github.com/httprunner/httprunner/v4/hrp/internal/env" + "github.com/httprunner/httprunner/v4/hrp/internal/json" +) + +type CPResult struct { + Point PointF `json:"point"` + Width float64 `json:"width"` + Height float64 `json:"height"` +} + +type CPResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result []CPResult `json:"result"` +} + +type veDEMCPService struct{} + +func newVEDEMCPService() (*veDEMCPService, error) { + if err := checkCPEnv(); err != nil { + return nil, err + } + return &veDEMCPService{}, nil +} + +func checkCPEnv() error { + if env.VEDEM_CP_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_URL missed") + } + if env.VEDEM_CP_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_AK missed") + } + if env.VEDEM_CP_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_SK missed") + } + return nil +} + +func (s *veDEMCPService) getCPResult(sourceImage []byte) ([]CPResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("withDet", "true") + // bodyWriter.WriteField("timestampOnly", "true") + + formWriter, err := bodyWriter.CreateFormFile("image", "image.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + _, err = formWriter.Write(sourceImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + err = bodyWriter.Close() + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + } + + req, err := http.NewRequest("POST", env.VEDEM_CP_URL, bodyBuf) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + } + + token := builtin.Sign("auth-v2", env.VEDEM_CP_AK, env.VEDEM_CP_SK, bodyBuf.Bytes()) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + resp, err = client.Do(req) + if err == nil { + break + } + + var logID string + if resp != nil { + logID = getLogID(resp.Header) + } + log.Error().Err(err). + Str("logID", logID). + Msgf("request CV service failed, retry %d", i) + time.Sleep(1 * time.Second) + } + if resp == nil { + return nil, code.CVServiceConnectionError + } + + defer resp.Body.Close() + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + } + + var cpResult CPResponse + err = json.Unmarshal(results, &cpResult) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("json unmarshal response body error: %v", err)) + } + + return cpResult.Result, nil +} + +func (s *veDEMCPService) FindPopupCloseButton(byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := NewData(map[string]interface{}{}, options...) + + cpResults, err := s.getCPResult(byteSource) + if err != nil { + log.Error().Err(err).Msg("getCPResult failed") + return + } + + var rects []image.Rectangle + for _, cpResult := range cpResults { + rect = image.Rectangle{ + // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(cpResult.Point.X), + Y: int(cpResult.Point.Y), + }, + Max: image.Point{ + X: int(cpResult.Point.X + cpResult.Width), + Y: int(cpResult.Point.Y + cpResult.Height), + }, + } + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { + rects = append(rects, rect) + + // match exactly, and not specify index, return the first one + if data.Index == 0 { + return rect, nil + } + } + } + + if len(rects) == 0 { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("popup close button not found")) + } + + // get index + idx := data.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("popup close button found, index %d out of range", idx)) + } + + return rects[idx], nil +} diff --git a/hrp/pkg/uixt/cv_vedem.go b/hrp/pkg/uixt/im_vedem.go similarity index 54% rename from hrp/pkg/uixt/cv_vedem.go rename to hrp/pkg/uixt/im_vedem.go index 7d0c70ca..49e232de 100644 --- a/hrp/pkg/uixt/cv_vedem.go +++ b/hrp/pkg/uixt/im_vedem.go @@ -9,13 +9,12 @@ import ( "net/http" "time" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/code" "github.com/httprunner/httprunner/v4/hrp/internal/env" "github.com/httprunner/httprunner/v4/hrp/internal/json" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" ) type IMResult struct { @@ -198,170 +197,3 @@ func (s *veDEMIMService) FindImage(byteSearch []byte, byteSource []byte, options return rects[idx], nil } - -type CPResult struct { - Point PointF `json:"point"` - Width float64 `json:"width"` - Height float64 `json:"height"` -} - -type CPResponse struct { - Code int `json:"code"` - Message string `json:"message"` - Result []CPResult `json:"result"` -} - -type veDEMCPService struct{} - -func newVEDEMCPService() (*veDEMCPService, error) { - if err := checkCPEnv(); err != nil { - return nil, err - } - return &veDEMCPService{}, nil -} - -func checkCPEnv() error { - if env.VEDEM_CP_URL == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_URL missed") - } - if env.VEDEM_CP_AK == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_AK missed") - } - if env.VEDEM_CP_SK == "" { - return errors.Wrap(code.CVEnvMissedError, "VEDEM_CP_SK missed") - } - return nil -} - -func (s *veDEMCPService) getCPResult(sourceImage []byte) ([]CPResult, error) { - bodyBuf := &bytes.Buffer{} - bodyWriter := multipart.NewWriter(bodyBuf) - bodyWriter.WriteField("withDet", "true") - // bodyWriter.WriteField("timestampOnly", "true") - - formWriter, err := bodyWriter.CreateFormFile("image", "image.png") - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("create form file error: %v", err)) - } - _, err = formWriter.Write(sourceImage) - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("write form error: %v", err)) - } - - err = bodyWriter.Close() - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("close body writer error: %v", err)) - } - - req, err := http.NewRequest("POST", env.VEDEM_CP_URL, bodyBuf) - if err != nil { - return nil, errors.Wrap(code.CVRequestError, - fmt.Sprintf("construct request error: %v", err)) - } - - token := builtin.Sign("auth-v2", env.VEDEM_CP_AK, env.VEDEM_CP_SK, bodyBuf.Bytes()) - req.Header.Add("Agw-Auth", token) - req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) - - var resp *http.Response - // retry 3 times - for i := 1; i <= 3; i++ { - resp, err = client.Do(req) - if err == nil { - break - } - - var logID string - if resp != nil { - logID = getLogID(resp.Header) - } - log.Error().Err(err). - Str("logID", logID). - Msgf("request CV service failed, retry %d", i) - time.Sleep(1 * time.Second) - } - if resp == nil { - return nil, code.CVServiceConnectionError - } - - defer resp.Body.Close() - - results, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(code.CVResponseError, - fmt.Sprintf("read response body error: %v", err)) - } - - if resp.StatusCode != http.StatusOK { - return nil, errors.Wrap(code.CVResponseError, - fmt.Sprintf("unexpected response status code: %d, results: %v", - resp.StatusCode, string(results))) - } - - var cpResult CPResponse - err = json.Unmarshal(results, &cpResult) - if err != nil { - return nil, errors.Wrap(code.CVResponseError, - fmt.Sprintf("json unmarshal response body error: %v", err)) - } - - return cpResult.Result, nil -} - -func (s *veDEMCPService) FindPopupCloseButton(byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { - data := NewData(map[string]interface{}{}, options...) - - cpResults, err := s.getCPResult(byteSource) - if err != nil { - log.Error().Err(err).Msg("getCPResult failed") - return - } - - var rects []image.Rectangle - for _, cpResult := range cpResults { - rect = image.Rectangle{ - // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 - Min: image.Point{ - X: int(cpResult.Point.X), - Y: int(cpResult.Point.Y), - }, - Max: image.Point{ - X: int(cpResult.Point.X + cpResult.Width), - Y: int(cpResult.Point.Y + cpResult.Height), - }, - } - if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { - rects = append(rects, rect) - - // match exactly, and not specify index, return the first one - if data.Index == 0 { - return rect, nil - } - } - } - - if len(rects) == 0 { - return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, - fmt.Sprintf("popup close button not found")) - } - - // get index - idx := data.Index - if idx > 0 { - // NOTICE: index start from 1 - idx = idx - 1 - } else if idx < 0 { - idx = len(rects) + idx - } - - // index out of range - if idx >= len(rects) { - return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, - fmt.Sprintf("popup close button found, index %d out of range", idx)) - } - - return rects[idx], nil -} diff --git a/hrp/pkg/uixt/sd_vedem.go b/hrp/pkg/uixt/sd_vedem.go new file mode 100644 index 00000000..45884073 --- /dev/null +++ b/hrp/pkg/uixt/sd_vedem.go @@ -0,0 +1,199 @@ +package uixt + +import ( + "bytes" + "fmt" + "image" + "io/ioutil" + "mime/multipart" + "net/http" + "time" + + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/code" + "github.com/httprunner/httprunner/v4/hrp/internal/env" + "github.com/httprunner/httprunner/v4/hrp/internal/json" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +type SDResult struct { + Image string `json:"image"` + Points []PointF `json:"points"` +} + +type SDResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result []SDResult `json:"result"` +} + +type veDEMSDService struct{} + +func newVEDEMSDService() (*veDEMSDService, error) { + if err := checkSDEnv(); err != nil { + return nil, err + } + return &veDEMSDService{}, nil +} + +func checkSDEnv() error { + if env.VEDEM_SD_URL == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_SD_URL missed") + } + if env.VEDEM_SD_AK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_SD_AK missed") + } + if env.VEDEM_SD_SK == "" { + return errors.Wrap(code.CVEnvMissedError, "VEDEM_SD_SK missed") + } + return nil +} + +func (s *veDEMSDService) getSDResult(searchImage []byte, sourceImage []byte) ([]SDResult, error) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + bodyWriter.WriteField("withDet", "true") + // bodyWriter.WriteField("timestampOnly", "true") + + formWriter, err := bodyWriter.CreateFormFile("searchImage", "searchImage.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + size, err := formWriter.Write(searchImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + formWriter, err = bodyWriter.CreateFormFile("sourceImage", "sourceImage.png") + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("create form file error: %v", err)) + } + _, err = formWriter.Write(sourceImage) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("write form error: %v", err)) + } + + err = bodyWriter.Close() + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("close body writer error: %v", err)) + } + + req, err := http.NewRequest("POST", env.VEDEM_SD_URL, bodyBuf) + if err != nil { + return nil, errors.Wrap(code.CVRequestError, + fmt.Sprintf("construct request error: %v", err)) + } + + token := builtin.Sign("auth-v2", env.VEDEM_SD_AK, env.VEDEM_SD_SK, bodyBuf.Bytes()) + req.Header.Add("Agw-Auth", token) + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + + var resp *http.Response + // retry 3 times + for i := 1; i <= 3; i++ { + resp, err = client.Do(req) + if err == nil { + break + } + + var logID string + if resp != nil { + logID = getLogID(resp.Header) + } + log.Error().Err(err). + Str("logID", logID). + Int("imageBufSize", size). + Msgf("request CV service failed, retry %d", i) + time.Sleep(1 * time.Second) + } + if resp == nil { + return nil, code.CVServiceConnectionError + } + + defer resp.Body.Close() + + results, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("read response body error: %v", err)) + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("unexpected response status code: %d, results: %v", + resp.StatusCode, string(results))) + } + + var cvResult SDResponse + err = json.Unmarshal(results, &cvResult) + if err != nil { + return nil, errors.Wrap(code.CVResponseError, + fmt.Sprintf("json unmarshal response body error: %v", err)) + } + + return cvResult.Result, nil +} + +func (s *veDEMSDService) FindImage(byteSearch []byte, byteSource []byte, options ...DataOption) (rect image.Rectangle, err error) { + data := NewData(map[string]interface{}{}, options...) + + cvResults, err := s.getSDResult(byteSearch, byteSource) + if err != nil { + log.Error().Err(err).Msg("getSDResult failed") + return + } + + var rects []image.Rectangle + var cvImages []string + for _, cvResult := range cvResults { + rect = image.Rectangle{ + // cvResult.Points 顺序：左上 -> 右上 -> 右下 -> 左下 + Min: image.Point{ + X: int(cvResult.Points[0].X), + Y: int(cvResult.Points[0].Y), + }, + Max: image.Point{ + X: int(cvResult.Points[2].X), + Y: int(cvResult.Points[2].Y), + }, + } + if rect.Min.X >= data.Scope[0] && rect.Max.X <= data.Scope[2] && rect.Min.Y >= data.Scope[1] && rect.Max.Y <= data.Scope[3] { + cvImages = append(cvImages, cvResult.Image) + + rects = append(rects, rect) + + // match exactly, and not specify index, return the first one + if data.Index == 0 { + return rect, nil + } + } + } + + if len(rects) == 0 { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image not found")) + } + + // get index + idx := data.Index + if idx > 0 { + // NOTICE: index start from 1 + idx = idx - 1 + } else if idx < 0 { + idx = len(rects) + idx + } + + // index out of range + if idx >= len(rects) { + return image.Rectangle{}, errors.Wrap(code.CVImageNotFoundError, + fmt.Sprintf("image found, index %d out of range", idx)) + } + + return rects[idx], nil +}