From b205e99ffbdc08d2bf9cade4c915076299e8380e Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 23 Oct 2022 23:19:13 +0800 Subject: [PATCH] refactor: merge opencv-helper --- go.mod | 3 +- go.sum | 3 - hrp/pkg/uixt/opencv.go | 477 ++++++++++++++++++++++++++++++++++++ hrp/pkg/uixt/opencv_on.go | 146 ----------- hrp/pkg/uixt/opencv_test.go | 259 ++++++++++++++++++++ 5 files changed, 737 insertions(+), 151 deletions(-) create mode 100644 hrp/pkg/uixt/opencv.go delete mode 100644 hrp/pkg/uixt/opencv_on.go create mode 100644 hrp/pkg/uixt/opencv_test.go diff --git a/go.mod b/go.mod index 838ee3c0..fc8e3a47 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.18 require ( github.com/andybalholm/brotli v1.0.4 github.com/denisbrodbeck/machineid v1.0.1 - github.com/electricbubble/opencv-helper v0.0.3 github.com/fatih/color v1.13.0 github.com/getsentry/sentry-go v0.13.0 github.com/go-openapi/spec v0.20.7 @@ -29,6 +28,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.0 + gocv.io/x/gocv v0.31.0 golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 google.golang.org/grpc v1.49.0 @@ -73,7 +73,6 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.5.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - gocv.io/x/gocv v0.31.0 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect diff --git a/go.sum b/go.sum index c1652ddb..b941d7b5 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/electricbubble/opencv-helper v0.0.3 h1:p0sHTUPPPm8GqzVUtYH+wQbJoguzotUXVRAS7Ibk7nI= -github.com/electricbubble/opencv-helper v0.0.3/go.mod h1:VHB21p5xsIjXUsUleWSaKGJosRsRAO7cuJoZKf7uCcc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -422,7 +420,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -gocv.io/x/gocv v0.27.0/go.mod h1:n4LnYjykU6y9gn48yZf4eLCdtuSb77XxSkW6g0wGf/A= gocv.io/x/gocv v0.31.0 h1:BHDtK8v+YPvoSPQTTiZB2fM/7BLg6511JqkruY2z6LQ= gocv.io/x/gocv v0.31.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/hrp/pkg/uixt/opencv.go b/hrp/pkg/uixt/opencv.go new file mode 100644 index 00000000..b8f7fe8d --- /dev/null +++ b/hrp/pkg/uixt/opencv.go @@ -0,0 +1,477 @@ +//go:build opencv + +package uixt + +import ( + "bytes" + "fmt" + "image" + "image/color" + "io/ioutil" + "log" + "math" + "os" + + "github.com/pkg/errors" + "gocv.io/x/gocv" +) + +const ( + // TmCcoeffNormed maps to TM_CCOEFF_NORMED + TmCcoeffNormed TemplateMatchMode = iota + // TmSqdiff maps to TM_SQDIFF + TmSqdiff + // TmSqdiffNormed maps to TM_SQDIFF_NORMED + TmSqdiffNormed + // TmCcorr maps to TM_CCORR + TmCcorr + // TmCcorrNormed maps to TM_CCORR_NORMED + TmCcorrNormed + // TmCcoeff maps to TM_CCOEFF + TmCcoeff +) + +type DebugMode int + +const ( + // DmOff no output + DmOff DebugMode = iota + // DmEachMatch output matched and mismatched values + DmEachMatch + // DmNotMatch output only values that do not match + DmNotMatch +) + +// Extend 获得扩展后的 Driver， +// 并指定匹配阀值， +// 获取当前设备的 Scale， +// 默认匹配模式为 TmCcoeffNormed， +// 默认关闭 OpenCV 匹配值计算后的输出 +func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) { + dExt, err = extend(driver) + if err != nil { + return nil, err + } + + for _, option := range options { + option(&dExt.CVArgs) + } + + if dExt.threshold == 0 { + dExt.threshold = 0.95 // default threshold + } + if dExt.matchMode == 0 { + dExt.matchMode = TmCcoeffNormed // default match mode + } + Debug(DebugMode(DmOff)) + return +} + +func (dExt *DriverExt) Debug(dm DebugMode) { + Debug(DebugMode(dm)) +} + +func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.Driver = dExt.Driver + newExt.scale = dExt.scale + newExt.matchMode = dExt.matchMode + newExt.threshold = threshold + return +} + +func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { + newExt = new(DriverExt) + newExt.Driver = dExt.Driver + newExt.scale = dExt.scale + newExt.matchMode = matchMode + newExt.threshold = dExt.threshold + return +} + +// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { +// pathSource := filepath.Join(sExt.pathname, GenFilename()) +// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { +// return image.Rectangle{}, err +// } +// +// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), TemplateMatchMode(sExt.MatchMode)); err != nil { +// return image.Rectangle{}, err +// } +// return +// } + +func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(search); err != nil { + return nil, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return nil, err + } + + if rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil { + return nil, err + } + return +} + +func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { + var bufSource, bufSearch *bytes.Buffer + if bufSearch, err = getBufFromDisk(imagePath); err != nil { + return 0, 0, 0, 0, err + } + if bufSource, err = dExt.takeScreenShot(); err != nil { + return 0, 0, 0, 0, err + } + + var rect image.Rectangle + if rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil { + return 0, 0, 0, 0, err + } + + // if rect, err = dExt.findImgRect(search); err != nil { + // return 0, 0, 0, 0, err + // } + x, y, width, height = dExt.MappingToRectInUIKit(rect) + return +} + +func getBufFromDisk(name string) (*bytes.Buffer, error) { + var f *os.File + var err error + if f, err = os.Open(name); err != nil { + return nil, err + } + var all []byte + if all, err = ioutil.ReadAll(f); err != nil { + return nil, err + } + return bytes.NewBuffer(all), nil +} + +var debug = DmOff + +const dmOutputMsg = `[DEBUG] The current value is '%.4f', the expected value is '%.4f'` + +func Debug(dm DebugMode) { + debug = dm +} + +const DefaultMatchMode = TmCcoeffNormed + +var fillColor = color.RGBA{R: 255, G: 255, B: 255, A: 0} + +func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) { + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{DefaultMatchMode} + } + var matImage, matTpl gocv.Mat + if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { + return image.Point{}, err + } + defer func() { + _ = matImage.Close() + _ = matTpl.Close() + }() + return getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) +} + +func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) { + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{DefaultMatchMode} + } + var matImage, matTpl gocv.Mat + if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { + return image.Point{}, err + } + defer func() { + _ = matImage.Close() + _ = matTpl.Close() + }() + + return getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) +} + +func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) { + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{DefaultMatchMode} + } + var matImage, matTpl gocv.Mat + if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { + return nil, err + } + defer func() { + _ = matImage.Close() + _ = matTpl.Close() + }() + + var loc image.Point + if loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]); err != nil { + return nil, err + } + widthTpl := matTpl.Cols() + heightTpl := matTpl.Rows() + + locs = make([]image.Point, 0, 9) + locs = append(locs, loc) + + gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor) + + loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) + for ; err == nil; loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) { + locs = append(locs, loc) + gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor) + } + + return locs, nil +} + +func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) { + if len(matchMode) == 0 { + matchMode = []TemplateMatchMode{DefaultMatchMode} + } + var matImage, matTpl gocv.Mat + if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { + return nil, err + } + defer func() { + _ = matImage.Close() + _ = matTpl.Close() + }() + + var loc image.Point + if loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]); err != nil { + return nil, err + } + widthTpl := matTpl.Cols() + heightTpl := matTpl.Rows() + + locs = make([]image.Point, 0, 9) + locs = append(locs, loc) + + gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor) + + loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) + for ; err == nil; loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) { + locs = append(locs, loc) + gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor) + } + + return locs, nil +} + +// getPts 根据图片坐标和宽高，获取填充区域 +func getPts(loc image.Point, width, height int) [][]image.Point { + return [][]image.Point{ + { + image.Pt(loc.X, loc.Y), + image.Pt(loc.X, loc.Y+height), + image.Pt(loc.X+width, loc.Y+height), + image.Pt(loc.X+width, loc.Y), + }, + } +} + +func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) { + var matTpl gocv.Mat + if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { + return image.Rectangle{}, err + } + defer func() { + _ = matTpl.Close() + }() + + var loc image.Point + if loc, err = FindImageLocationFromDisk(source, search, threshold, matchMode...); err != nil { + return image.Rectangle{}, err + } + rect = image.Rect(loc.X, loc.Y, loc.X+matTpl.Cols(), loc.Y+matTpl.Rows()) + return +} + +func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) { + var matTpl gocv.Mat + if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil { + return nil, err + } + defer func() { + _ = matTpl.Close() + }() + + var locs []image.Point + if locs, err = FindAllImageLocationsFromDisk(source, search, threshold, matchMode...); err != nil { + return nil, err + } + + rects = make([]image.Rectangle, 0, len(locs)) + for i := range locs { + r := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows()) + rects = append(rects, r) + } + return +} + +func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) { + var matTpl gocv.Mat + if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { + return image.Rectangle{}, err + } + defer func() { + _ = matTpl.Close() + }() + + var loc image.Point + if loc, err = FindImageLocationFromRaw(source, search, threshold, matchMode...); err != nil { + return image.Rectangle{}, err + } + rect = image.Rect(loc.X, loc.Y, loc.X+matTpl.Cols(), loc.Y+matTpl.Rows()) + return +} + +func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) { + var matTpl gocv.Mat + if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil { + return nil, err + } + defer func() { + _ = matTpl.Close() + }() + + var locs []image.Point + if locs, err = FindAllImageLocationsFromRaw(source, search, threshold, matchMode...); err != nil { + return nil, err + } + + rects = make([]image.Rectangle, 0, len(locs)) + for i := range locs { + r := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows()) + rects = append(rects, r) + } + return +} + +// getMatsFromDisk 从指定路径获取原图和目标图的 `gocv.Mat` +func getMatsFromDisk(nameImage, nameTpl string, flags gocv.IMReadFlag) (matImage, matTpl gocv.Mat, err error) { + matImage = gocv.IMRead(nameImage, flags) + if matImage.Empty() { + return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %s", nameImage) + } + matTpl = gocv.IMRead(nameTpl, flags) + if matTpl.Empty() { + return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %s", nameTpl) + } + return +} + +// func getBufsFromDisk(nameImage, nameTpl string) (bufImage, bufTpl *bytes.Buffer, err error) { +// getBuf := func(_name string) (*bytes.Buffer, error) { +// var f *os.File +// var e error +// if f, e = os.Open(_name); e != nil { +// return nil, e +// } +// var all []byte +// if all, e = ioutil.ReadAll(f); e != nil { +// return nil, e +// } +// return bytes.NewBuffer(all), nil +// } +// if bufImage, err = getBuf(nameImage); err != nil { +// return nil, nil, err +// } +// if bufTpl, err = getBuf(nameTpl); err != nil { +// return nil, nil, err +// } +// return +// } + +// getMatsFromRaw 获取原图和目标图的 `gocv.Mat` +func getMatsFromRaw(bufImage, bufTpl *bytes.Buffer, flags gocv.IMReadFlag) (matImage, matTpl gocv.Mat, err error) { + if matImage, err = gocv.IMDecode(bufImage.Bytes(), flags); err != nil { + return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %w", err) + } + if matImage.Empty() { + return gocv.Mat{}, gocv.Mat{}, errors.New("invalid read [source]") + } + if matTpl, err = gocv.IMDecode(bufTpl.Bytes(), flags); err != nil { + return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %w", err) + } + if matTpl.Empty() { + return gocv.Mat{}, gocv.Mat{}, errors.New("invalid read [template]") + } + return +} + +// getMatchingLocation 获取匹配的图片位置 +func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode TemplateMatchMode) (loc image.Point, err error) { + if threshold > 1 { + threshold = 1.0 + } + // TM_SQDIFF：该方法使用平方差进行匹配，最好匹配为 0。值越大匹配结果越差。 + // TM_SQDIFF_NORMED：该方法使用归一化的平方差进行匹配，最佳匹配也在结果为0处。 + // TmCcoeff 将模版对其均值的相对值与图像对其均值的相关值进行匹配,1表示完美匹配,-1表示糟糕的匹配,0表示没有任何相关性(随机序列)。 + minVal, maxVal, minLoc, maxLoc := getMatchingResult(matImage, matTpl, matchMode) + + // fmt.Println(matchMode[0], "\t", minVal, maxVal, "\t", minLoc, maxLoc) + // fmt.Printf("%s\t %.10f \t %.10f \t %v \t %v \n", matchMode[0], minVal, maxVal, minLoc, maxLoc) + + var val float32 + val, loc = getValLoc(minVal, maxVal, minLoc, maxLoc, matchMode) + + if debug == DmEachMatch { + log.Println(fmt.Sprintf(dmOutputMsg, val, threshold)) + } + + if val >= threshold { + return loc, nil + } else { + if debug == DmNotMatch { + log.Println(fmt.Sprintf(dmOutputMsg, val, threshold)) + } + return image.Point{}, errors.New("no such target search image") + } +} + +// getMatchingResult 匹配图片并返回匹配值和位置 +func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) { + matResult, tmpMask := gocv.NewMat(), gocv.NewMat() + defer func() { + _ = matResult.Close() + _ = tmpMask.Close() + }() + gocv.MatchTemplate(matImage, matTpl, &matResult, gocv.TemplateMatchMode(matchMode), tmpMask) + minVal, maxVal, minLoc, maxLoc = gocv.MinMaxLoc(matResult) + return +} + +// getValLoc 根据不同的匹配模式返回匹配值和位置 +func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode TemplateMatchMode) (val float32, loc image.Point) { + val, loc = maxVal, maxLoc + + switch matchMode { + case TmSqdiff, TmSqdiffNormed: + // 平方差，最佳匹配为 0 + val = minVal + // minVal = 8 + // for val >= 1 { + // val -= 1 + // } + if val >= 1 { + val = float32(math.Mod(float64(val), 1)) + } + val = 1 - val + loc = minLoc + case TmCcoeff: + // TmCcoeff 将模版对其均值的相对值与图像对其均值的相关值进行匹配,1表示完美匹配,-1表示糟糕的匹配,0表示没有任何相关性(随机序列)。 + // maxVal = 5064792.5000000000 + _, frac := math.Modf(float64(val)) + val = float32(frac) + case TmCcorr: + // maxVal = 50553512.0000000000 + _, frac := math.Modf(float64(val)) + val = float32(frac) + } + // fmt.Println("匹配度", val) + return +} diff --git a/hrp/pkg/uixt/opencv_on.go b/hrp/pkg/uixt/opencv_on.go deleted file mode 100644 index e8b3d407..00000000 --- a/hrp/pkg/uixt/opencv_on.go +++ /dev/null @@ -1,146 +0,0 @@ -//go:build opencv - -package uixt - -import ( - "bytes" - "image" - "io/ioutil" - "os" - - cvHelper "github.com/electricbubble/opencv-helper" -) - -const ( - // TmCcoeffNormed maps to TM_CCOEFF_NORMED - TmCcoeffNormed TemplateMatchMode = iota - // TmSqdiff maps to TM_SQDIFF - TmSqdiff - // TmSqdiffNormed maps to TM_SQDIFF_NORMED - TmSqdiffNormed - // TmCcorr maps to TM_CCORR - TmCcorr - // TmCcorrNormed maps to TM_CCORR_NORMED - TmCcorrNormed - // TmCcoeff maps to TM_CCOEFF - TmCcoeff -) - -type DebugMode int - -const ( - // DmOff no output - DmOff DebugMode = iota - // DmEachMatch output matched and mismatched values - DmEachMatch - // DmNotMatch output only values that do not match - DmNotMatch -) - -// Extend 获得扩展后的 Driver， -// 并指定匹配阀值， -// 获取当前设备的 Scale， -// 默认匹配模式为 TmCcoeffNormed， -// 默认关闭 OpenCV 匹配值计算后的输出 -func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) { - dExt, err = extend(driver) - if err != nil { - return nil, err - } - - for _, option := range options { - option(&dExt.CVArgs) - } - - if dExt.threshold == 0 { - dExt.threshold = 0.95 // default threshold - } - if dExt.matchMode == 0 { - dExt.matchMode = TmCcoeffNormed // default match mode - } - cvHelper.Debug(cvHelper.DebugMode(DmOff)) - return -} - -func (dExt *DriverExt) Debug(dm DebugMode) { - cvHelper.Debug(cvHelper.DebugMode(dm)) -} - -func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.Driver = dExt.Driver - newExt.scale = dExt.scale - newExt.matchMode = dExt.matchMode - newExt.threshold = threshold - return -} - -func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) { - newExt = new(DriverExt) - newExt.Driver = dExt.Driver - newExt.scale = dExt.scale - newExt.matchMode = matchMode - newExt.threshold = dExt.threshold - return -} - -// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) { -// pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename()) -// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil { -// return image.Rectangle{}, err -// } -// -// if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil { -// return image.Rectangle{}, err -// } -// return -// } - -func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(search); err != nil { - return nil, err - } - if bufSource, err = dExt.takeScreenShot(); err != nil { - return nil, err - } - - if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil { - return nil, err - } - return -} - -func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) { - var bufSource, bufSearch *bytes.Buffer - if bufSearch, err = getBufFromDisk(imagePath); err != nil { - return 0, 0, 0, 0, err - } - if bufSource, err = dExt.takeScreenShot(); err != nil { - return 0, 0, 0, 0, err - } - - var rect image.Rectangle - if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil { - return 0, 0, 0, 0, err - } - - // if rect, err = dExt.findImgRect(search); err != nil { - // return 0, 0, 0, 0, err - // } - x, y, width, height = dExt.MappingToRectInUIKit(rect) - return -} - -func getBufFromDisk(name string) (*bytes.Buffer, error) { - var f *os.File - var err error - if f, err = os.Open(name); err != nil { - return nil, err - } - var all []byte - if all, err = ioutil.ReadAll(f); err != nil { - return nil, err - } - return bytes.NewBuffer(all), nil -} diff --git a/hrp/pkg/uixt/opencv_test.go b/hrp/pkg/uixt/opencv_test.go new file mode 100644 index 00000000..1d2cccf8 --- /dev/null +++ b/hrp/pkg/uixt/opencv_test.go @@ -0,0 +1,259 @@ +//go:build opencv + +package uixt + +import ( + "bytes" + "image" + "image/color" + "io/ioutil" + "sort" + "strconv" + "testing" + + "gocv.io/x/gocv" +) + +func TestFindImageLocation(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + // pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg" + // pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + fileSource, err := ioutil.ReadFile(pathSource) + checkErr(t, err) + fileSearch, err := ioutil.ReadFile(pathSearch) + checkErr(t, err) + bufferSource := bytes.NewBuffer(fileSource) + bufferSearch := bytes.NewBuffer(fileSearch) + _, _ = bufferSource, bufferSearch + + var imgLoc image.Point + // imgLoc, err := FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95) + imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmSqdiff) + checkErr(t, err) + t.Log(imgLoc) + + imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmSqdiffNormed) + checkErr(t, err) + t.Log(imgLoc) + + // imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, gocv.TmCcoeff) + // checkErr(t, err) + // t.Log(imgLoc) + + imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmCcoeffNormed) + checkErr(t, err) + t.Log(imgLoc) + + // imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, gocv.TmCcorr) + // checkErr(t, err) + // t.Log(imgLoc) + + imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmCcorrNormed) + checkErr(t, err) + t.Log(imgLoc) + + return + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor) + rect := image.Rect(imgLoc.X, imgLoc.Y, imgLoc.X+matTpl.Cols(), imgLoc.Y+matTpl.Rows()) + gocv.Rectangle(&matBig, rect, blue, 3) + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +} + +func TestFindImageRectFromDisk(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSource = "/Users/hero/Documents/temp/2020-05/opencv/loop1.png" + // pathSource = "/Users/hero/Documents/temp/2020-05/opencv/loop2.png" + + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + imgRect, err := FindImageRectFromDisk(pathSource, pathSearch, 0.95, TmCcorrNormed) + checkErr(t, err) + t.Log(imgRect) + + // return + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + gocv.Rectangle(&matBig, imgRect, blue, 3) + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +} + +func TestFindAllImageLocationsFromDisk(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg" + pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + locs, err := FindAllImageLocationsFromDisk(pathSource, pathSearch, 0.95) + checkErr(t, err) + t.Log(locs) + + sort.Slice(locs, func(i, j int) bool { + if locs[i].Y < locs[j].Y { + return true + } else if locs[i].Y == locs[j].Y { + if locs[i].X < locs[j].X { + return true + } + } + return false + }) + + // return + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor) + for i := range locs { + rect := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows()) + gocv.Rectangle(&matBig, rect, blue, 3) + gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), locs[i], gocv.FontHersheySimplex, 2, blue, 3) + } + + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +} + +func TestFindAllImageRectsFromDisk(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg" + pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + rects, err := FindAllImageRectsFromDisk(pathSource, pathSearch, 0.95) + checkErr(t, err) + t.Log(rects) + + // return + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + for i := range rects { + gocv.Rectangle(&matBig, rects[i], blue, 3) + gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), rects[i].Min, gocv.FontHersheySimplex, 2, blue, 3) + } + + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +} + +func TestFindImageLocationFromRaw(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + // pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg" + // pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + fileSource, err := ioutil.ReadFile(pathSource) + checkErr(t, err) + fileSearch, err := ioutil.ReadFile(pathSearch) + checkErr(t, err) + bufferSource := bytes.NewBuffer(fileSource) + bufferSearch := bytes.NewBuffer(fileSearch) + _, _ = bufferSource, bufferSearch + + var imgLoc image.Point + // imgLoc, err := FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95) + imgLoc, err = FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95, TmSqdiff) + checkErr(t, err) + t.Log(imgLoc) + + t.Log(FindImageRectFromRaw(bufferSource, bufferSearch, 0.95, TmSqdiff)) + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor) + rect := image.Rect(imgLoc.X, imgLoc.Y, imgLoc.X+matTpl.Cols(), imgLoc.Y+matTpl.Rows()) + gocv.Rectangle(&matBig, rect, blue, 3) + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +} + +func TestFindAllImageRectsFromRaw(t *testing.T) { + pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png" + pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png" + + pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg" + pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png" + + fileSource, err := ioutil.ReadFile(pathSource) + checkErr(t, err) + fileSearch, err := ioutil.ReadFile(pathSearch) + checkErr(t, err) + bufferSource := bytes.NewBuffer(fileSource) + bufferSearch := bytes.NewBuffer(fileSearch) + _, _ = bufferSource, bufferSearch + + rects, err := FindAllImageRectsFromRaw(bufferSource, bufferSearch, 0.95) + checkErr(t, err) + t.Log(rects) + + sort.Slice(rects, func(i, j int) bool { + if rects[i].Min.Y < rects[j].Min.Y { + return true + } else if rects[i].Min.Y == rects[j].Min.Y { + if rects[i].Min.X < rects[j].Min.X { + return true + } + } + return false + }) + + // return + + window := gocv.NewWindow("Find Image") + defer window.Close() + blue := color.RGBA{R: 0, G: 0, B: 255, A: 0} + matBig := gocv.IMRead(pathSource, gocv.IMReadColor) + for i := range rects { + gocv.Rectangle(&matBig, rects[i], blue, 3) + gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), rects[i].Min, gocv.FontHersheySimplex, 2, blue, 3) + } + + for { + window.IMShow(matBig) + if window.WaitKey(1) >= 0 { + break + } + } +}