From a742b575fab04c15941e11f1b266d0eb12ade448 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Wed, 21 Sep 2022 15:37:10 +0800 Subject: [PATCH] change: make opencv as optional build tags --- hrp/internal/uixt/default.go | 29 ++++++ hrp/internal/uixt/ext.go | 138 ++-------------------------- hrp/internal/uixt/gesture.go | 3 +- hrp/internal/uixt/gesture_test.go | 3 +- hrp/internal/uixt/opencv.go | 146 ++++++++++++++++++++++++++++++ scripts/build.sh | 1 + 6 files changed, 187 insertions(+), 133 deletions(-) create mode 100644 hrp/internal/uixt/default.go create mode 100644 hrp/internal/uixt/opencv.go diff --git a/hrp/internal/uixt/default.go b/hrp/internal/uixt/default.go new file mode 100644 index 00000000..7506f686 --- /dev/null +++ b/hrp/internal/uixt/default.go @@ -0,0 +1,29 @@ +//go:build !opencv + +package uixt + +import ( + "image" + + "github.com/rs/zerolog/log" +) + +func (dExt *DriverExt) extendOpenCV(threshold float64, matchMode ...TemplateMatchMode) (err error) { + log.Fatal().Msg("opencv is not supported") + return +} + +func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { + log.Fatal().Msg("opencv is not supported") + return +} + +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { + log.Fatal().Msg("opencv is not supported") + return +} + +func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { + log.Fatal().Msg("opencv is not supported") + return +} diff --git a/hrp/internal/uixt/ext.go b/hrp/internal/uixt/ext.go index 82d9c394..02462183 100644 --- a/hrp/internal/uixt/ext.go +++ b/hrp/internal/uixt/ext.go @@ -6,7 +6,6 @@ import ( "image" "image/jpeg" "image/png" - "io/ioutil" "mime" "mime/multipart" "net/http" @@ -15,7 +14,6 @@ import ( "strings" "github.com/electricbubble/gwda" - cvHelper "github.com/electricbubble/opencv-helper" "github.com/pkg/errors" "github.com/rs/zerolog/log" ) @@ -23,40 +21,16 @@ import ( // TemplateMatchMode is the type of the template matching operation. type TemplateMatchMode int -const ( - // TmSqdiff maps to TM_SQDIFF - TmSqdiff TemplateMatchMode = iota - // TmSqdiffNormed maps to TM_SQDIFF_NORMED - TmSqdiffNormed - // TmCcorr maps to TM_CCORR - TmCcorr - // TmCcorrNormed maps to TM_CCORR_NORMED - TmCcorrNormed - // TmCcoeff maps to TM_CCOEFF - TmCcoeff - // TmCcoeffNormed maps to TM_CCOEFF_NORMED - TmCcoeffNormed -) - -type DebugMode int - -const ( - // DmOff no output - DmOff DebugMode = iota - // DmEachMatch output matched and mismatched values - DmEachMatch - // DmNotMatch output only values that do not match - DmNotMatch -) - type DriverExt struct { gwda.WebDriver windowSize gwda.Size - scale float64 - MatchMode TemplateMatchMode - Threshold float64 frame *bytes.Buffer doneMjpegStream chan bool + + // OpenCV + scale float64 + matchMode TemplateMatchMode + threshold float64 } // Extend 获得扩展后的 Driver， @@ -68,45 +42,14 @@ func Extend(driver gwda.WebDriver, threshold float64, matchMode ...TemplateMatch dExt = &DriverExt{WebDriver: driver} dExt.doneMjpegStream = make(chan bool, 1) - if dExt.scale, err = dExt.Scale(); err != nil { - return &DriverExt{}, err - } - // get device window size dExt.windowSize, err = dExt.WebDriver.WindowSize() if err != nil { return nil, errors.Wrap(err, "failed to get windows size") } - if len(matchMode) == 0 { - matchMode = []TemplateMatchMode{TmCcoeffNormed} - } - dExt.MatchMode = matchMode[0] - cvHelper.Debug(cvHelper.DebugMode(DmOff)) - dExt.Threshold = threshold - return dExt, nil -} - -func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.WebDriver = dExt.WebDriver - newExt.scale = dExt.scale - newExt.MatchMode = dExt.MatchMode - newExt.Threshold = threshold - return -} - -func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.WebDriver = dExt.WebDriver - newExt.scale = dExt.scale - newExt.MatchMode = matchMode - newExt.Threshold = dExt.Threshold - return -} - -func (dExt *DriverExt) Debug(dm DebugMode) { - cvHelper.Debug(cvHelper.DebugMode(dm)) + err = dExt.extendOpenCV(threshold, matchMode...) + return dExt, err } func (dExt *DriverExt) ConnectMjpegStream(httpClient *http.Client) (err error) { @@ -225,46 +168,6 @@ func (dExt *DriverExt) ScreenShot(fileName string) (string, error) { return dExt.saveScreenShot(raw, fileName) } -// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { -// pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename()) -// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { -// return image.Rectangle{}, err -// } -// -// if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil { -// return image.Rectangle{}, err -// } -// return -// } - -func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(search); err != nil { - return nil, err - } - if bufSource, err = dExt.takeScreenShot(); err != nil { - return nil, err - } - - if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil { - return nil, err - } - return -} - -func getBufFromDisk(name string) (*bytes.Buffer, error) { - var f *os.File - var err error - if f, err = os.Open(name); err != nil { - return nil, err - } - var all []byte - if all, err = ioutil.ReadAll(f); err != nil { - return nil, err - } - return bytes.NewBuffer(all), nil -} - // isPathExists returns true if path exists, whether path is file or dir func isPathExists(path string) bool { if _, err := os.Stat(path); os.IsNotExist(err) { @@ -299,33 +202,6 @@ func (dExt *DriverExt) FindUIRectInUIKit(search string) (x, y, width, height flo return dExt.FindImageRectInUIKit(search) } -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(imagePath); err != nil { - return 0, 0, 0, 0, err - } - if bufSource, err = dExt.takeScreenShot(); err != nil { - return 0, 0, 0, 0, err - } - - var rect image.Rectangle - if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.Threshold), cvHelper.TemplateMatchMode(dExt.MatchMode)); err != nil { - return 0, 0, 0, 0, err - } - - // if rect, err = dExt.findImgRect(search); err != nil { - // return 0, 0, 0, 0, err - // } - x, y, width, height = dExt.MappingToRectInUIKit(rect) - return -} - -func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { - x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale - width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale - return -} - func (dExt *DriverExt) PerformTouchActions(touchActions *gwda.TouchActions) error { return dExt.PerformAppiumTouchActions(touchActions) } diff --git a/hrp/internal/uixt/gesture.go b/hrp/internal/uixt/gesture.go index 75462167..2491bb27 100644 --- a/hrp/internal/uixt/gesture.go +++ b/hrp/internal/uixt/gesture.go @@ -1,8 +1,9 @@ +//go:build opencv + package uixt import ( "image" - "sort" "github.com/electricbubble/gwda" ) diff --git a/hrp/internal/uixt/gesture_test.go b/hrp/internal/uixt/gesture_test.go index 075023bd..5e4ce57b 100644 --- a/hrp/internal/uixt/gesture_test.go +++ b/hrp/internal/uixt/gesture_test.go @@ -1,7 +1,8 @@ +//go:build opencv + package uixt import ( - "strconv" "strings" "testing" ) diff --git a/hrp/internal/uixt/opencv.go b/hrp/internal/uixt/opencv.go new file mode 100644 index 00000000..504e6eef --- /dev/null +++ b/hrp/internal/uixt/opencv.go @@ -0,0 +1,146 @@ +//go:build opencv + +package uixt + +import ( + "bytes" + "image" + "io/ioutil" + "os" + + cvHelper "github.com/electricbubble/opencv-helper" +) + +const ( + // TmSqdiff maps to TM_SQDIFF + TmSqdiff TemplateMatchMode = iota + // TmSqdiffNormed maps to TM_SQDIFF_NORMED + TmSqdiffNormed + // TmCcorr maps to TM_CCORR + TmCcorr + // TmCcorrNormed maps to TM_CCORR_NORMED + TmCcorrNormed + // TmCcoeff maps to TM_CCOEFF + TmCcoeff + // TmCcoeffNormed maps to TM_CCOEFF_NORMED + TmCcoeffNormed +) + +type DebugMode int + +const ( + // DmOff no output + DmOff DebugMode = iota + // DmEachMatch output matched and mismatched values + DmEachMatch + // DmNotMatch output only values that do not match + DmNotMatch +) + +// Extend 获得扩展后的 Driver， +// 并指定匹配阀值， +// 获取当前设备的 Scale， +// 默认匹配模式为 TmCcoeffNormed， +// 默认关闭 OpenCV 匹配值计算后的输出 +func (dExt *DriverExt) extendOpenCV(threshold float64, matchMode ...TemplateMatchMode) (err error) { + if dExt.scale, err = dExt.Scale(); err != nil { + return err + } + + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{TmCcoeffNormed} + } + dExt.matchMode = matchMode[0] + cvHelper.Debug(cvHelper.DebugMode(DmOff)) + dExt.threshold = threshold + return nil +} + +func (dExt *DriverExt) Debug(dm DebugMode) { + cvHelper.Debug(cvHelper.DebugMode(dm)) +} + +func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.WebDriver = dExt.WebDriver + newExt.scale = dExt.scale + newExt.matchMode = dExt.matchMode + newExt.threshold = threshold + return +} + +func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.WebDriver = dExt.WebDriver + newExt.scale = dExt.scale + newExt.matchMode = matchMode + newExt.threshold = dExt.threshold + return +} + +// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { +// pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename()) +// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { +// return image.Rectangle{}, err +// } +// +// if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil { +// return image.Rectangle{}, err +// } +// return +// } + +func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(search); err != nil { + return nil, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return nil, err + } + + if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil { + return nil, err + } + return +} + +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string) (x, y, width, height float64, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(imagePath); err != nil { + return 0, 0, 0, 0, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return 0, 0, 0, 0, err + } + + var rect image.Rectangle + if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil { + return 0, 0, 0, 0, err + } + + // if rect, err = dExt.findImgRect(search); err != nil { + // return 0, 0, 0, 0, err + // } + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + +func getBufFromDisk(name string) (*bytes.Buffer, error) { + var f *os.File + var err error + if f, err = os.Open(name); err != nil { + return nil, err + } + var all []byte + if all, err = ioutil.ReadAll(f); err != nil { + return nil, err + } + return bytes.NewBuffer(all), nil +} + +func (dExt *DriverExt) MappingToRectInUIKit(rect image.Rectangle) (x, y, width, height float64) { + x, y = float64(rect.Min.X)/dExt.scale, float64(rect.Min.Y)/dExt.scale + width, height = float64(rect.Dx())/dExt.scale, float64(rect.Dy())/dExt.scale + return +} diff --git a/scripts/build.sh b/scripts/build.sh index 8043ceb0..7a75d8ed 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -15,6 +15,7 @@ mkdir -p "output" bin_path="output/hrp" # build +# optional build tags: opencv go build -ldflags '-s -w' -o "$bin_path" hrp/cmd/cli/main.go # check output and version