GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-05-13 06:29:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: merge opencv-helper

Browse Source
This commit is contained in:
debugtalk
2022-10-23 23:19:13 +08:00
parent 863ffdf798
commit b205e99ffb
5 changed files with 737 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@@ -5,7 +5,6 @@ go 1.18
require (
github.com/andybalholm/brotli v1.0.4
github.com/denisbrodbeck/machineid v1.0.1
github.com/electricbubble/opencv-helper v0.0.3
github.com/fatih/color v1.13.0
github.com/getsentry/sentry-go v0.13.0
github.com/go-openapi/spec v0.20.7
@@ -29,6 +28,7 @@ require (
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.8.0
gocv.io/x/gocv v0.31.0
golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
google.golang.org/grpc v1.49.0
@@ -73,7 +73,6 @@ require (
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.5.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
gocv.io/x/gocv v0.31.0 // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect

3
go.sum
View File

@@ -98,8 +98,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/electricbubble/opencv-helper v0.0.3 h1:p0sHTUPPPm8GqzVUtYH+wQbJoguzotUXVRAS7Ibk7nI=
github.com/electricbubble/opencv-helper v0.0.3/go.mod h1:VHB21p5xsIjXUsUleWSaKGJosRsRAO7cuJoZKf7uCcc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -422,7 +420,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
gocv.io/x/gocv v0.27.0/go.mod h1:n4LnYjykU6y9gn48yZf4eLCdtuSb77XxSkW6g0wGf/A=
gocv.io/x/gocv v0.31.0 h1:BHDtK8v+YPvoSPQTTiZB2fM/7BLg6511JqkruY2z6LQ=
gocv.io/x/gocv v0.31.0/go.mod h1:oc6FvfYqfBp99p+yOEzs9tbYF9gOrAQSeL/dyIPefJU=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

477
hrp/pkg/uixt/opencv.go Normal file
View File

@@ -0,0 +1,477 @@
//go:build opencv
package uixt
import (
"bytes"
"fmt"
"image"
"image/color"
"io/ioutil"
"log"
"math"
"os"
"github.com/pkg/errors"
"gocv.io/x/gocv"
)
const (
// TmCcoeffNormed maps to TM_CCOEFF_NORMED
TmCcoeffNormed TemplateMatchMode = iota
// TmSqdiff maps to TM_SQDIFF
TmSqdiff
// TmSqdiffNormed maps to TM_SQDIFF_NORMED
TmSqdiffNormed
// TmCcorr maps to TM_CCORR
TmCcorr
// TmCcorrNormed maps to TM_CCORR_NORMED
TmCcorrNormed
// TmCcoeff maps to TM_CCOEFF
TmCcoeff
)
type DebugMode int
const (
// DmOff no output
DmOff DebugMode = iota
// DmEachMatch output matched and mismatched values
DmEachMatch
// DmNotMatch output only values that do not match
DmNotMatch
)
// Extend 获得扩展后的 Driver，
// 并指定匹配阀值，
// 获取当前设备的 Scale，
// 默认匹配模式为 TmCcoeffNormed，
// 默认关闭 OpenCV 匹配值计算后的输出
func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) {
dExt, err = extend(driver)
if err != nil {
return nil, err
}
for _, option := range options {
option(&dExt.CVArgs)
}
if dExt.threshold == 0 {
dExt.threshold = 0.95 // default threshold
}
if dExt.matchMode == 0 {
dExt.matchMode = TmCcoeffNormed // default match mode
}
Debug(DebugMode(DmOff))
return
}
func (dExt *DriverExt) Debug(dm DebugMode) {
Debug(DebugMode(dm))
}
func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = dExt.matchMode
newExt.threshold = threshold
return
}
func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = matchMode
newExt.threshold = dExt.threshold
return
}
// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
// pathSource := filepath.Join(sExt.pathname, GenFilename())
// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
// return image.Rectangle{}, err
// }
//
// if rect, err = FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), TemplateMatchMode(sExt.MatchMode)); err != nil {
// return image.Rectangle{}, err
// }
// return
// }
func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(search); err != nil {
return nil, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return nil, err
}
if rects, err = FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil {
return nil, err
}
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return 0, 0, 0, 0, err
}
var rect image.Rectangle
if rect, err = FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), TemplateMatchMode(dExt.matchMode)); err != nil {
return 0, 0, 0, 0, err
}
// if rect, err = dExt.findImgRect(search); err != nil {
// return 0, 0, 0, 0, err
// }
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func getBufFromDisk(name string) (*bytes.Buffer, error) {
var f *os.File
var err error
if f, err = os.Open(name); err != nil {
return nil, err
}
var all []byte
if all, err = ioutil.ReadAll(f); err != nil {
return nil, err
}
return bytes.NewBuffer(all), nil
}
var debug = DmOff
const dmOutputMsg = `[DEBUG] The current value is '%.4f', the expected value is '%.4f'`
func Debug(dm DebugMode) {
debug = dm
}
const DefaultMatchMode = TmCcoeffNormed
var fillColor = color.RGBA{R: 255, G: 255, B: 255, A: 0}
func FindImageLocationFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return image.Point{}, err
}
defer func() {
_ = matImage.Close()
_ = matTpl.Close()
}()
return getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
}
func FindImageLocationFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (loc image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return image.Point{}, err
}
defer func() {
_ = matImage.Close()
_ = matTpl.Close()
}()
return getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
}
func FindAllImageLocationsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
}
defer func() {
_ = matImage.Close()
_ = matTpl.Close()
}()
var loc image.Point
if loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]); err != nil {
return nil, err
}
widthTpl := matTpl.Cols()
heightTpl := matTpl.Rows()
locs = make([]image.Point, 0, 9)
locs = append(locs, loc)
gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor)
loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
for ; err == nil; loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) {
locs = append(locs, loc)
gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor)
}
return locs, nil
}
func FindAllImageLocationsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (locs []image.Point, err error) {
if len(matchMode) == 0 {
matchMode = []TemplateMatchMode{DefaultMatchMode}
}
var matImage, matTpl gocv.Mat
if matImage, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
}
defer func() {
_ = matImage.Close()
_ = matTpl.Close()
}()
var loc image.Point
if loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]); err != nil {
return nil, err
}
widthTpl := matTpl.Cols()
heightTpl := matTpl.Rows()
locs = make([]image.Point, 0, 9)
locs = append(locs, loc)
gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor)
loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0])
for ; err == nil; loc, err = getMatchingLocation(matImage, matTpl, threshold, matchMode[0]) {
locs = append(locs, loc)
gocv.FillPoly(&matImage, gocv.NewPointsVectorFromPoints(getPts(loc, widthTpl, heightTpl)), fillColor)
}
return locs, nil
}
// getPts 根据图片坐标和宽高，获取填充区域
func getPts(loc image.Point, width, height int) [][]image.Point {
return [][]image.Point{
{
image.Pt(loc.X, loc.Y),
image.Pt(loc.X, loc.Y+height),
image.Pt(loc.X+width, loc.Y+height),
image.Pt(loc.X+width, loc.Y),
},
}
}
func FindImageRectFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return image.Rectangle{}, err
}
defer func() {
_ = matTpl.Close()
}()
var loc image.Point
if loc, err = FindImageLocationFromDisk(source, search, threshold, matchMode...); err != nil {
return image.Rectangle{}, err
}
rect = image.Rect(loc.X, loc.Y, loc.X+matTpl.Cols(), loc.Y+matTpl.Rows())
return
}
func FindAllImageRectsFromDisk(source, search string, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromDisk(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
}
defer func() {
_ = matTpl.Close()
}()
var locs []image.Point
if locs, err = FindAllImageLocationsFromDisk(source, search, threshold, matchMode...); err != nil {
return nil, err
}
rects = make([]image.Rectangle, 0, len(locs))
for i := range locs {
r := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows())
rects = append(rects, r)
}
return
}
func FindImageRectFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rect image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return image.Rectangle{}, err
}
defer func() {
_ = matTpl.Close()
}()
var loc image.Point
if loc, err = FindImageLocationFromRaw(source, search, threshold, matchMode...); err != nil {
return image.Rectangle{}, err
}
rect = image.Rect(loc.X, loc.Y, loc.X+matTpl.Cols(), loc.Y+matTpl.Rows())
return
}
func FindAllImageRectsFromRaw(source, search *bytes.Buffer, threshold float32, matchMode ...TemplateMatchMode) (rects []image.Rectangle, err error) {
var matTpl gocv.Mat
if _, matTpl, err = getMatsFromRaw(source, search, gocv.IMReadGrayScale); err != nil {
return nil, err
}
defer func() {
_ = matTpl.Close()
}()
var locs []image.Point
if locs, err = FindAllImageLocationsFromRaw(source, search, threshold, matchMode...); err != nil {
return nil, err
}
rects = make([]image.Rectangle, 0, len(locs))
for i := range locs {
r := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows())
rects = append(rects, r)
}
return
}
// getMatsFromDisk 从指定路径获取原图和目标图的 `gocv.Mat`
func getMatsFromDisk(nameImage, nameTpl string, flags gocv.IMReadFlag) (matImage, matTpl gocv.Mat, err error) {
matImage = gocv.IMRead(nameImage, flags)
if matImage.Empty() {
return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %s", nameImage)
}
matTpl = gocv.IMRead(nameTpl, flags)
if matTpl.Empty() {
return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %s", nameTpl)
}
return
}
// func getBufsFromDisk(nameImage, nameTpl string) (bufImage, bufTpl *bytes.Buffer, err error) {
// getBuf := func(_name string) (*bytes.Buffer, error) {
// var f *os.File
// var e error
// if f, e = os.Open(_name); e != nil {
// return nil, e
// }
// var all []byte
// if all, e = ioutil.ReadAll(f); e != nil {
// return nil, e
// }
// return bytes.NewBuffer(all), nil
// }
// if bufImage, err = getBuf(nameImage); err != nil {
// return nil, nil, err
// }
// if bufTpl, err = getBuf(nameTpl); err != nil {
// return nil, nil, err
// }
// return
// }
// getMatsFromRaw 获取原图和目标图的 `gocv.Mat`
func getMatsFromRaw(bufImage, bufTpl *bytes.Buffer, flags gocv.IMReadFlag) (matImage, matTpl gocv.Mat, err error) {
if matImage, err = gocv.IMDecode(bufImage.Bytes(), flags); err != nil {
return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %w", err)
}
if matImage.Empty() {
return gocv.Mat{}, gocv.Mat{}, errors.New("invalid read [source]")
}
if matTpl, err = gocv.IMDecode(bufTpl.Bytes(), flags); err != nil {
return gocv.Mat{}, gocv.Mat{}, fmt.Errorf("invalid read %w", err)
}
if matTpl.Empty() {
return gocv.Mat{}, gocv.Mat{}, errors.New("invalid read [template]")
}
return
}
// getMatchingLocation 获取匹配的图片位置
func getMatchingLocation(matImage gocv.Mat, matTpl gocv.Mat, threshold float32, matchMode TemplateMatchMode) (loc image.Point, err error) {
if threshold > 1 {
threshold = 1.0
}
// TM_SQDIFF：该方法使用平方差进行匹配，最好匹配为 0。值越大匹配结果越差。
// TM_SQDIFF_NORMED：该方法使用归一化的平方差进行匹配，最佳匹配也在结果为0处。
// TmCcoeff 将模版对其均值的相对值与图像对其均值的相关值进行匹配,1表示完美匹配,-1表示糟糕的匹配,0表示没有任何相关性(随机序列)。
minVal, maxVal, minLoc, maxLoc := getMatchingResult(matImage, matTpl, matchMode)
// fmt.Println(matchMode[0], "\t", minVal, maxVal, "\t", minLoc, maxLoc)
// fmt.Printf("%s\t %.10f \t %.10f \t %v \t %v \n", matchMode[0], minVal, maxVal, minLoc, maxLoc)
var val float32
val, loc = getValLoc(minVal, maxVal, minLoc, maxLoc, matchMode)
if debug == DmEachMatch {
log.Println(fmt.Sprintf(dmOutputMsg, val, threshold))
}
if val >= threshold {
return loc, nil
} else {
if debug == DmNotMatch {
log.Println(fmt.Sprintf(dmOutputMsg, val, threshold))
}
return image.Point{}, errors.New("no such target search image")
}
}
// getMatchingResult 匹配图片并返回匹配值和位置
func getMatchingResult(matImage gocv.Mat, matTpl gocv.Mat, matchMode TemplateMatchMode) (minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point) {
matResult, tmpMask := gocv.NewMat(), gocv.NewMat()
defer func() {
_ = matResult.Close()
_ = tmpMask.Close()
}()
gocv.MatchTemplate(matImage, matTpl, &matResult, gocv.TemplateMatchMode(matchMode), tmpMask)
minVal, maxVal, minLoc, maxLoc = gocv.MinMaxLoc(matResult)
return
}
// getValLoc 根据不同的匹配模式返回匹配值和位置
func getValLoc(minVal float32, maxVal float32, minLoc image.Point, maxLoc image.Point, matchMode TemplateMatchMode) (val float32, loc image.Point) {
val, loc = maxVal, maxLoc
switch matchMode {
case TmSqdiff, TmSqdiffNormed:
// 平方差，最佳匹配为 0
val = minVal
// minVal = 8
// for val >= 1 {
// val -= 1
// }
if val >= 1 {
val = float32(math.Mod(float64(val), 1))
}
val = 1 - val
loc = minLoc
case TmCcoeff:
// TmCcoeff 将模版对其均值的相对值与图像对其均值的相关值进行匹配,1表示完美匹配,-1表示糟糕的匹配,0表示没有任何相关性(随机序列)。
// maxVal = 5064792.5000000000
_, frac := math.Modf(float64(val))
val = float32(frac)
case TmCcorr:
// maxVal = 50553512.0000000000
_, frac := math.Modf(float64(val))
val = float32(frac)
}
// fmt.Println("匹配度", val)
return
}

146
hrp/pkg/uixt/opencv_on.go
View File

@@ -1,146 +0,0 @@
//go:build opencv
package uixt
import (
"bytes"
"image"
"io/ioutil"
"os"
cvHelper "github.com/electricbubble/opencv-helper"
)
const (
// TmCcoeffNormed maps to TM_CCOEFF_NORMED
TmCcoeffNormed TemplateMatchMode = iota
// TmSqdiff maps to TM_SQDIFF
TmSqdiff
// TmSqdiffNormed maps to TM_SQDIFF_NORMED
TmSqdiffNormed
// TmCcorr maps to TM_CCORR
TmCcorr
// TmCcorrNormed maps to TM_CCORR_NORMED
TmCcorrNormed
// TmCcoeff maps to TM_CCOEFF
TmCcoeff
)
type DebugMode int
const (
// DmOff no output
DmOff DebugMode = iota
// DmEachMatch output matched and mismatched values
DmEachMatch
// DmNotMatch output only values that do not match
DmNotMatch
)
// Extend 获得扩展后的 Driver，
// 并指定匹配阀值，
// 获取当前设备的 Scale，
// 默认匹配模式为 TmCcoeffNormed，
// 默认关闭 OpenCV 匹配值计算后的输出
func Extend(driver WebDriver, options ...CVOption) (dExt *DriverExt, err error) {
dExt, err = extend(driver)
if err != nil {
return nil, err
}
for _, option := range options {
option(&dExt.CVArgs)
}
if dExt.threshold == 0 {
dExt.threshold = 0.95 // default threshold
}
if dExt.matchMode == 0 {
dExt.matchMode = TmCcoeffNormed // default match mode
}
cvHelper.Debug(cvHelper.DebugMode(DmOff))
return
}
func (dExt *DriverExt) Debug(dm DebugMode) {
cvHelper.Debug(cvHelper.DebugMode(dm))
}
func (dExt *DriverExt) OnlyOnceThreshold(threshold float64) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = dExt.matchMode
newExt.threshold = threshold
return
}
func (dExt *DriverExt) OnlyOnceMatchMode(matchMode TemplateMatchMode) (newExt *DriverExt) {
newExt = new(DriverExt)
newExt.Driver = dExt.Driver
newExt.scale = dExt.scale
newExt.matchMode = matchMode
newExt.threshold = dExt.threshold
return
}
// func (sExt *DriverExt) findImgRect(search string) (rect image.Rectangle, err error) {
// pathSource := filepath.Join(sExt.pathname, cvHelper.GenFilename())
// if err = sExt.driver.ScreenshotToDisk(pathSource); err != nil {
// return image.Rectangle{}, err
// }
//
// if rect, err = cvHelper.FindImageRectFromDisk(pathSource, search, float32(sExt.Threshold), cvHelper.TemplateMatchMode(sExt.MatchMode)); err != nil {
// return image.Rectangle{}, err
// }
// return
// }
func (dExt *DriverExt) FindAllImageRect(search string) (rects []image.Rectangle, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(search); err != nil {
return nil, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return nil, err
}
if rects, err = cvHelper.FindAllImageRectsFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil {
return nil, err
}
return
}
func (dExt *DriverExt) FindImageRectInUIKit(imagePath string, options ...DataOption) (x, y, width, height float64, err error) {
var bufSource, bufSearch *bytes.Buffer
if bufSearch, err = getBufFromDisk(imagePath); err != nil {
return 0, 0, 0, 0, err
}
if bufSource, err = dExt.takeScreenShot(); err != nil {
return 0, 0, 0, 0, err
}
var rect image.Rectangle
if rect, err = cvHelper.FindImageRectFromRaw(bufSource, bufSearch, float32(dExt.threshold), cvHelper.TemplateMatchMode(dExt.matchMode)); err != nil {
return 0, 0, 0, 0, err
}
// if rect, err = dExt.findImgRect(search); err != nil {
// return 0, 0, 0, 0, err
// }
x, y, width, height = dExt.MappingToRectInUIKit(rect)
return
}
func getBufFromDisk(name string) (*bytes.Buffer, error) {
var f *os.File
var err error
if f, err = os.Open(name); err != nil {
return nil, err
}
var all []byte
if all, err = ioutil.ReadAll(f); err != nil {
return nil, err
}
return bytes.NewBuffer(all), nil
}

259
hrp/pkg/uixt/opencv_test.go Normal file
View File

@@ -0,0 +1,259 @@
//go:build opencv
package uixt
import (
"bytes"
"image"
"image/color"
"io/ioutil"
"sort"
"strconv"
"testing"
"gocv.io/x/gocv"
)
func TestFindImageLocation(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
// pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg"
// pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
fileSource, err := ioutil.ReadFile(pathSource)
checkErr(t, err)
fileSearch, err := ioutil.ReadFile(pathSearch)
checkErr(t, err)
bufferSource := bytes.NewBuffer(fileSource)
bufferSearch := bytes.NewBuffer(fileSearch)
_, _ = bufferSource, bufferSearch
var imgLoc image.Point
// imgLoc, err := FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95)
imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmSqdiff)
checkErr(t, err)
t.Log(imgLoc)
imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmSqdiffNormed)
checkErr(t, err)
t.Log(imgLoc)
// imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, gocv.TmCcoeff)
// checkErr(t, err)
// t.Log(imgLoc)
imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmCcoeffNormed)
checkErr(t, err)
t.Log(imgLoc)
// imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, gocv.TmCcorr)
// checkErr(t, err)
// t.Log(imgLoc)
imgLoc, err = FindImageLocationFromDisk(pathSource, pathSearch, 0.95, TmCcorrNormed)
checkErr(t, err)
t.Log(imgLoc)
return
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor)
rect := image.Rect(imgLoc.X, imgLoc.Y, imgLoc.X+matTpl.Cols(), imgLoc.Y+matTpl.Rows())
gocv.Rectangle(&matBig, rect, blue, 3)
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}
func TestFindImageRectFromDisk(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSource = "/Users/hero/Documents/temp/2020-05/opencv/loop1.png"
// pathSource = "/Users/hero/Documents/temp/2020-05/opencv/loop2.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
imgRect, err := FindImageRectFromDisk(pathSource, pathSearch, 0.95, TmCcorrNormed)
checkErr(t, err)
t.Log(imgRect)
// return
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
gocv.Rectangle(&matBig, imgRect, blue, 3)
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}
func TestFindAllImageLocationsFromDisk(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg"
pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
locs, err := FindAllImageLocationsFromDisk(pathSource, pathSearch, 0.95)
checkErr(t, err)
t.Log(locs)
sort.Slice(locs, func(i, j int) bool {
if locs[i].Y < locs[j].Y {
return true
} else if locs[i].Y == locs[j].Y {
if locs[i].X < locs[j].X {
return true
}
}
return false
})
// return
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor)
for i := range locs {
rect := image.Rect(locs[i].X, locs[i].Y, locs[i].X+matTpl.Cols(), locs[i].Y+matTpl.Rows())
gocv.Rectangle(&matBig, rect, blue, 3)
gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), locs[i], gocv.FontHersheySimplex, 2, blue, 3)
}
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}
func TestFindAllImageRectsFromDisk(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg"
pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
rects, err := FindAllImageRectsFromDisk(pathSource, pathSearch, 0.95)
checkErr(t, err)
t.Log(rects)
// return
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
for i := range rects {
gocv.Rectangle(&matBig, rects[i], blue, 3)
gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), rects[i].Min, gocv.FontHersheySimplex, 2, blue, 3)
}
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}
func TestFindImageLocationFromRaw(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
// pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg"
// pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
fileSource, err := ioutil.ReadFile(pathSource)
checkErr(t, err)
fileSearch, err := ioutil.ReadFile(pathSearch)
checkErr(t, err)
bufferSource := bytes.NewBuffer(fileSource)
bufferSearch := bytes.NewBuffer(fileSearch)
_, _ = bufferSource, bufferSearch
var imgLoc image.Point
// imgLoc, err := FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95)
imgLoc, err = FindImageLocationFromRaw(bufferSource, bufferSearch, 0.95, TmSqdiff)
checkErr(t, err)
t.Log(imgLoc)
t.Log(FindImageRectFromRaw(bufferSource, bufferSearch, 0.95, TmSqdiff))
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
matTpl := gocv.IMRead(pathSearch, gocv.IMReadColor)
rect := image.Rect(imgLoc.X, imgLoc.Y, imgLoc.X+matTpl.Cols(), imgLoc.Y+matTpl.Rows())
gocv.Rectangle(&matBig, rect, blue, 3)
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}
func TestFindAllImageRectsFromRaw(t *testing.T) {
pathSource := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste_2020-05-18_16-20-31.png"
pathSearch := "/Users/hero/Documents/temp/2020-05/opencv/Snipaste.png"
pathSource = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5291.jpg"
pathSearch = "/Users/hero/Documents/temp/2020-05/opencv/IMG_5.png"
fileSource, err := ioutil.ReadFile(pathSource)
checkErr(t, err)
fileSearch, err := ioutil.ReadFile(pathSearch)
checkErr(t, err)
bufferSource := bytes.NewBuffer(fileSource)
bufferSearch := bytes.NewBuffer(fileSearch)
_, _ = bufferSource, bufferSearch
rects, err := FindAllImageRectsFromRaw(bufferSource, bufferSearch, 0.95)
checkErr(t, err)
t.Log(rects)
sort.Slice(rects, func(i, j int) bool {
if rects[i].Min.Y < rects[j].Min.Y {
return true
} else if rects[i].Min.Y == rects[j].Min.Y {
if rects[i].Min.X < rects[j].Min.X {
return true
}
}
return false
})
// return
window := gocv.NewWindow("Find Image")
defer window.Close()
blue := color.RGBA{R: 0, G: 0, B: 255, A: 0}
matBig := gocv.IMRead(pathSource, gocv.IMReadColor)
for i := range rects {
gocv.Rectangle(&matBig, rects[i], blue, 3)
gocv.PutText(&matBig, strconv.FormatInt(int64(i), 10), rects[i].Min, gocv.FontHersheySimplex, 2, blue, 3)
}
for {
window.IMShow(matBig)
if window.WaitKey(1) >= 0 {
break
}
}
}