mirror of
https://github.com/httprunner/httprunner.git
synced 2026-05-07 05:32:43 +08:00
Merge branch 'feat-ai' into 'master'
refactor See merge request iesqa/httprunner!82
This commit is contained in:
@@ -4,7 +4,6 @@ import (
|
||||
"reflect"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -43,8 +42,8 @@ type TConfig struct {
|
||||
Path string `json:"path,omitempty" yaml:"path,omitempty"` // testcase file path
|
||||
PluginSetting *PluginConfig `json:"plugin,omitempty" yaml:"plugin,omitempty"` // plugin config
|
||||
IgnorePopup bool `json:"ignore_popup,omitempty" yaml:"ignore_popup,omitempty"`
|
||||
LLMService ai.LLMServiceType `json:"llm_service,omitempty" yaml:"llm_service,omitempty"`
|
||||
CVService ai.CVServiceType `json:"cv_service,omitempty" yaml:"cv_service,omitempty"`
|
||||
LLMService option.LLMServiceType `json:"llm_service,omitempty" yaml:"llm_service,omitempty"`
|
||||
CVService option.CVServiceType `json:"cv_service,omitempty" yaml:"cv_service,omitempty"`
|
||||
}
|
||||
|
||||
func (c *TConfig) Get() *TConfig {
|
||||
@@ -112,13 +111,13 @@ func (c *TConfig) SetWeight(weight int) *TConfig {
|
||||
}
|
||||
|
||||
// SetLLMService sets LLM service for current testcase.
|
||||
func (c *TConfig) SetLLMService(llmService ai.LLMServiceType) *TConfig {
|
||||
func (c *TConfig) SetLLMService(llmService option.LLMServiceType) *TConfig {
|
||||
c.LLMService = llmService
|
||||
return c
|
||||
}
|
||||
|
||||
// SetCVService sets CV service for current testcase.
|
||||
func (c *TConfig) SetCVService(cvService ai.CVServiceType) *TConfig {
|
||||
func (c *TConfig) SetCVService(cvService option.CVServiceType) *TConfig {
|
||||
c.CVService = cvService
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ func LoadCurlCase(path string) (*hrp.TestCaseDef, error) {
|
||||
}
|
||||
|
||||
func readFileLines(path string) ([]string, error) {
|
||||
file, err := os.OpenFile(path, os.O_RDONLY, 0o600)
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("path", path).Msg("open file failed")
|
||||
return nil, err
|
||||
|
||||
31
docs/uixt/ui_mark.md
Normal file
31
docs/uixt/ui_mark.md
Normal file
@@ -0,0 +1,31 @@
|
||||
### UI操作标注
|
||||
|
||||
针对 UI 操作的位置进行标注,帮助用户直观地了解操作发生的位置。
|
||||
|
||||
#### 功能说明
|
||||
|
||||
- 点击操作(tap):使用红色矩形框标注点击位置
|
||||
- 滑动操作(swipe):使用红色箭头标注滑动方向,从起始点指向结束点
|
||||
|
||||
#### 使用方法
|
||||
|
||||
只需在操作函数中添加 `WithMarkOperationEnabled(true)` 选项即可启用操作标注功能:
|
||||
|
||||
```go
|
||||
// 启用操作标注功能
|
||||
opts := []option.ActionOption{option.WithMarkOperationEnabled(true)}
|
||||
|
||||
// 执行点击操作,会自动用红色矩形标注点击位置
|
||||
err := driver.TapXY(0.5, 0.5, opts...)
|
||||
|
||||
// 执行滑动操作,会自动用红色箭头标注滑动方向
|
||||
err = driver.Swipe(0.2, 0.5, 0.8, 0.5, opts...)
|
||||
|
||||
// 可以同时使用其他选项
|
||||
opts = append(opts, option.WithScreenShotFileName("custom_name"))
|
||||
err = driver.TapXY(0.3, 0.7, opts...)
|
||||
```
|
||||
|
||||
#### 标注结果
|
||||
|
||||
标注后的图片会保存在截图目录中,文件名格式为:`{timestamp}_{tap|swipe}_marked.png`
|
||||
@@ -52,7 +52,7 @@ func launchAppDriver(pkgName string) (driverExt *uixt.XTDriver, err error) {
|
||||
|
||||
time.Sleep(15 * time.Second)
|
||||
|
||||
driverExt = uixt.NewXTDriver(driver)
|
||||
driverExt, _ = uixt.NewXTDriver(driver)
|
||||
|
||||
// 处理弹窗
|
||||
err = driverExt.ClosePopupsHandler()
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -53,8 +52,8 @@ func launchAppDriver(pkgName string) (driverExt *uixt.XTDriver, err error) {
|
||||
|
||||
time.Sleep(15 * time.Second)
|
||||
|
||||
driverExt = uixt.NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, _ = uixt.NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
|
||||
// 处理弹窗
|
||||
err = driverExt.ClosePopupsHandler()
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -39,8 +38,8 @@ var rootCmd = &cobra.Command{
|
||||
return errors.New("android or ios app bundldID is required")
|
||||
}
|
||||
|
||||
driverExt := uixt.NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM),
|
||||
driverExt, _ := uixt.NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM),
|
||||
)
|
||||
|
||||
wc := NewWorldCupLive(driverExt, matchName, bundleID, duration, interval)
|
||||
|
||||
@@ -48,7 +48,7 @@ func initIOSDriver(uuid string) *uixt.XTDriver {
|
||||
log.Fatal().Err(err).Msg("failed to init ios device")
|
||||
}
|
||||
driver, _ := device.NewDriver()
|
||||
driverExt := uixt.NewXTDriver(driver)
|
||||
driverExt, _ := uixt.NewXTDriver(driver)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func initAndroidDriver(uuid string) *uixt.XTDriver {
|
||||
log.Fatal().Err(err).Msg("failed to init android device")
|
||||
}
|
||||
driver, _ := device.NewDriver()
|
||||
driverExt := uixt.NewXTDriver(driver)
|
||||
driverExt, _ := uixt.NewXTDriver(driver)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
|
||||
4
go.mod
4
go.mod
@@ -9,7 +9,6 @@ require (
|
||||
github.com/andybalholm/brotli v1.0.4
|
||||
github.com/bytedance/sonic v1.13.2
|
||||
github.com/cloudwego/eino v0.3.26
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd
|
||||
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd
|
||||
@@ -42,6 +41,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bytedance/mockey v1.2.14 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
@@ -100,8 +100,6 @@ require (
|
||||
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/volcengine/volc-sdk-golang v1.0.23 // indirect
|
||||
github.com/volcengine/volcengine-go-sdk v1.0.185 // indirect
|
||||
github.com/yargevad/filepathx v1.0.0 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
|
||||
|
||||
67
go.sum
67
go.sum
@@ -1,11 +1,8 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
|
||||
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
|
||||
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
|
||||
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
|
||||
@@ -19,18 +16,14 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN
|
||||
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/eino v0.3.26 h1:FdJJTCdNrc9xPcYkLZiEyr7AA+WgyCKCbY+VNDXIaCE=
|
||||
github.com/cloudwego/eino v0.3.26/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6 h1:k17Z9VIRBL0/t7Ty1drGgY9tVOraM5xuO6gy7Qx7xus=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6/go.mod h1:13kQjYGLMgla6xTbejlpqhuk3i5BPlNv5S+1pmknlOo=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd h1:XEI7RezzV/cnOnhc1YeBJi6a0UoM41JTph4AZZR7+D8=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd/go.mod h1:8gMakAGQUR+IaWTSD0cpcD4U5FYq5puZ73/QjXqs1oU=
|
||||
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa h1:Jrmw8Q9g1WcE+x5t3o0TsEBM8RoMRURJI6P52I/ld74=
|
||||
@@ -53,8 +46,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0=
|
||||
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
|
||||
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
|
||||
@@ -106,27 +97,12 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
|
||||
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
@@ -135,7 +111,6 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18=
|
||||
@@ -186,7 +161,6 @@ github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQe
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -259,7 +233,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 h1:I4N3ZRnkZPbDN935Tg8QDf8fRpHp3bZ0U0/L42jBgNE=
|
||||
@@ -299,7 +272,6 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
@@ -316,10 +288,6 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
|
||||
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8=
|
||||
github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU=
|
||||
github.com/volcengine/volcengine-go-sdk v1.0.185 h1:MIH+YgdWZhO1fNg/vxLohl8ad7hlklaf46wpaTS1TN0=
|
||||
github.com/volcengine/volcengine-go-sdk v1.0.185/go.mod h1:gfEDc1s7SYaGoY+WH2dRrS3qiuDJMkwqyfXWCa7+7oA=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
|
||||
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
|
||||
@@ -338,20 +306,12 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -359,13 +319,10 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -396,10 +353,6 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
|
||||
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
|
||||
@@ -407,34 +360,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
|
||||
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
|
||||
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
@@ -453,8 +388,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5 h1:DOUDfNS+CFMM46k18FRF5k/0yz5NhZYMiUQxf4xglIU=
|
||||
gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
|
||||
howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
|
||||
@@ -6,9 +6,13 @@ import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/csv"
|
||||
builtinJSON "encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"math"
|
||||
"math/rand"
|
||||
"net"
|
||||
@@ -26,6 +30,7 @@ import (
|
||||
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
|
||||
func Dump2JSON(data interface{}, path string) error {
|
||||
@@ -484,3 +489,41 @@ func RunCommandWithCallback(cmdName string, args []string, callback LineCallback
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// LoadImage loads image file and returns base64 encoded string and image size
|
||||
func LoadImage(imagePath string) (base64Str string, size types.Size, err error) {
|
||||
// Read the image file
|
||||
imageFile, err := os.Open(imagePath)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
|
||||
}
|
||||
defer imageFile.Close()
|
||||
|
||||
// Decode the image to get its resolution
|
||||
imageData, format, err := image.Decode(imageFile)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
|
||||
}
|
||||
|
||||
// Get the resolution of the image
|
||||
width := imageData.Bounds().Dx()
|
||||
height := imageData.Bounds().Dy()
|
||||
size = types.Size{Width: width, Height: height}
|
||||
|
||||
// Convert image to base64
|
||||
buf := new(bytes.Buffer)
|
||||
if format == "jpeg" || format == "jpg" {
|
||||
if err := jpeg.Encode(buf, imageData, nil); err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
|
||||
}
|
||||
} else {
|
||||
// default use png
|
||||
if err := png.Encode(buf, imageData); err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
|
||||
}
|
||||
}
|
||||
base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
|
||||
base64.StdEncoding.EncodeToString(buf.Bytes()))
|
||||
|
||||
return base64Str, size, nil
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
v5.0.0-beta-2504292008
|
||||
v5.0.0-beta-2505071715
|
||||
|
||||
@@ -536,7 +536,7 @@ func (d *Device) List(remotePath string) (devFileInfos []DeviceFileInfo, err err
|
||||
}
|
||||
|
||||
func (d *Device) PushFile(localPath, remotePath string, modification ...time.Time) (err error) {
|
||||
localFile, err := os.OpenFile(localPath, os.O_RDONLY, 0o600)
|
||||
localFile, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -645,7 +645,7 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt
|
||||
}
|
||||
|
||||
func (d *Device) InstallAPK(apkPath string, args ...string) (string, error) {
|
||||
apkFile, err := os.OpenFile(apkPath, os.O_RDONLY, 0o600)
|
||||
apkFile, err := os.Open(apkPath)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, fmt.Sprintf("open apk file %s failed", apkPath))
|
||||
}
|
||||
|
||||
25
runner.go
25
runner.go
@@ -28,7 +28,7 @@ import (
|
||||
"github.com/httprunner/httprunner/v5/internal/sdk"
|
||||
"github.com/httprunner/httprunner/v5/internal/version"
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
// Run starts to run testcase with default configs.
|
||||
@@ -419,15 +419,15 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
|
||||
r.parametersIterator = parametersIterator
|
||||
|
||||
// ai options
|
||||
aiOpts := []ai.AIServiceOption{}
|
||||
aiOpts := []option.AIServiceOption{}
|
||||
if parsedConfig.LLMService != "" {
|
||||
aiOpts = append(aiOpts, ai.WithLLMService(parsedConfig.LLMService))
|
||||
aiOpts = append(aiOpts, option.WithLLMService(option.LLMServiceType(parsedConfig.LLMService)))
|
||||
}
|
||||
if parsedConfig.CVService == "" {
|
||||
// default to vedem
|
||||
parsedConfig.CVService = ai.CVServiceTypeVEDEM
|
||||
parsedConfig.CVService = option.CVServiceTypeVEDEM
|
||||
}
|
||||
aiOpts = append(aiOpts, ai.WithCVService(parsedConfig.CVService))
|
||||
aiOpts = append(aiOpts, option.WithCVService(parsedConfig.CVService))
|
||||
|
||||
// parse android devices config
|
||||
for _, androidDeviceOptions := range parsedConfig.Android {
|
||||
@@ -446,7 +446,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
|
||||
return nil, errors.Wrap(err, "init android driver failed")
|
||||
}
|
||||
|
||||
driverExt := uixt.NewXTDriver(driver, aiOpts...)
|
||||
driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "init android XTDriver failed")
|
||||
}
|
||||
r.uixtDrivers[androidDeviceOptions.SerialNumber] = driverExt
|
||||
}
|
||||
// parse iOS devices config
|
||||
@@ -466,7 +469,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
|
||||
return nil, errors.Wrap(err, "init ios driver failed")
|
||||
}
|
||||
|
||||
driverExt := uixt.NewXTDriver(driver, aiOpts...)
|
||||
driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "init ios XTDriver failed")
|
||||
}
|
||||
r.uixtDrivers[iosDeviceOptions.UDID] = driverExt
|
||||
}
|
||||
// parse harmony devices config
|
||||
@@ -486,7 +492,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
|
||||
return nil, errors.Wrap(err, "init harmony driver failed")
|
||||
}
|
||||
|
||||
driverExt := uixt.NewXTDriver(driver, aiOpts...)
|
||||
driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "init harmony XTDriver failed")
|
||||
}
|
||||
r.uixtDrivers[harmonyDeviceOptions.ConnectKey] = driverExt
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -33,8 +32,12 @@ func (r *Router) GetDriver(c *gin.Context) (driverExt *uixt.XTDriver, err error)
|
||||
return
|
||||
}
|
||||
|
||||
driverExt = uixt.NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err = uixt.NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
if err != nil {
|
||||
RenderErrorInitDriver(c, err)
|
||||
return
|
||||
}
|
||||
c.Set("driver", driverExt)
|
||||
return driverExt, nil
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func (s *Summary) GenHTMLReport() error {
|
||||
}
|
||||
|
||||
reportPath := filepath.Join(reportsDir, "report.html")
|
||||
file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0o600)
|
||||
file, err := os.Open(reportPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("open file failed")
|
||||
return err
|
||||
|
||||
169
uixt/ai/ai.go
169
uixt/ai/ai.go
@@ -3,55 +3,14 @@ package ai
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
)
|
||||
|
||||
func NewAIService(opts ...AIServiceOption) *AIServices {
|
||||
services := &AIServices{}
|
||||
for _, option := range opts {
|
||||
option(services)
|
||||
}
|
||||
return services
|
||||
}
|
||||
|
||||
type AIServices struct {
|
||||
ICVService
|
||||
ILLMService
|
||||
}
|
||||
|
||||
type AIServiceOption func(*AIServices)
|
||||
|
||||
type CVServiceType string
|
||||
|
||||
const (
|
||||
CVServiceTypeVEDEM CVServiceType = "vedem"
|
||||
CVServiceTypeOpenCV CVServiceType = "opencv"
|
||||
)
|
||||
|
||||
func WithCVService(service CVServiceType) AIServiceOption {
|
||||
return func(opts *AIServices) {
|
||||
if service == CVServiceTypeVEDEM {
|
||||
var err error
|
||||
opts.ICVService, err = NewVEDEMImageService()
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init vedem image service failed")
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type LLMServiceType string
|
||||
|
||||
const (
|
||||
LLMServiceTypeUITARS LLMServiceType = "ui-tars"
|
||||
LLMServiceTypeGPT4o LLMServiceType = "gpt-4o"
|
||||
LLMServiceTypeGPT4Vision LLMServiceType = "gpt-4-vision"
|
||||
LLMServiceTypeQwenVL LLMServiceType = "qwen-vl"
|
||||
LLMServiceTypeDeepSeekV3 LLMServiceType = "deepseek-v3"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// ILLMService 定义了 LLM 服务接口,包括规划和断言功能
|
||||
@@ -60,38 +19,29 @@ type ILLMService interface {
|
||||
Assert(opts *AssertOptions) (*AssertionResponse, error)
|
||||
}
|
||||
|
||||
func WithLLMService(modelType LLMServiceType) AIServiceOption {
|
||||
return func(opts *AIServices) {
|
||||
// init planner
|
||||
var planner IPlanner
|
||||
var err error
|
||||
switch modelType {
|
||||
case LLMServiceTypeGPT4o:
|
||||
// TODO: implement gpt-4o planner and asserter
|
||||
planner, err = NewPlanner(context.Background())
|
||||
case LLMServiceTypeUITARS:
|
||||
planner, err = NewUITarsPlanner(context.Background())
|
||||
}
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("init %s planner failed", modelType)
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
|
||||
// init asserter
|
||||
asserter, err := NewAsserter(context.Background(), modelType)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("init %s asserter failed", modelType)
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
|
||||
opts.ILLMService = &combinedLLMService{
|
||||
planner: planner,
|
||||
asserter: asserter,
|
||||
}
|
||||
func NewLLMService(modelType option.LLMServiceType) (ILLMService, error) {
|
||||
modelConfig, err := GetModelConfig(modelType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
asserter, err := NewAsserter(context.Background(), modelConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &combinedLLMService{
|
||||
planner: planner,
|
||||
asserter: asserter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// combinedLLMService 实现了 ILLMService 接口,组合了规划和断言功能
|
||||
// ⭐️支持采用不同的模型服务进行规划和断言
|
||||
type combinedLLMService struct {
|
||||
planner IPlanner // 提供规划功能
|
||||
asserter IAsserter // 提供断言功能
|
||||
@@ -106,3 +56,72 @@ func (c *combinedLLMService) Call(opts *PlanningOptions) (*PlanningResult, error
|
||||
func (c *combinedLLMService) Assert(opts *AssertOptions) (*AssertionResponse, error) {
|
||||
return c.asserter.Assert(opts)
|
||||
}
|
||||
|
||||
// LLM model config env variables
|
||||
const (
|
||||
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
|
||||
EnvOpenAIAPIKey = "OPENAI_API_KEY"
|
||||
EnvModelName = "LLM_MODEL_NAME"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type ModelConfig struct {
|
||||
*openai.ChatModelConfig
|
||||
ModelType option.LLMServiceType
|
||||
}
|
||||
|
||||
// GetModelConfig get OpenAI config
|
||||
func GetModelConfig(modelType option.LLMServiceType) (*ModelConfig, error) {
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return nil, errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
|
||||
openaiBaseURL := os.Getenv(EnvOpenAIBaseURL)
|
||||
if openaiBaseURL == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIBaseURL)
|
||||
}
|
||||
openaiAPIKey := os.Getenv(EnvOpenAIAPIKey)
|
||||
if openaiAPIKey == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIAPIKey)
|
||||
}
|
||||
modelName := os.Getenv(EnvModelName)
|
||||
if modelName == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvModelName)
|
||||
}
|
||||
|
||||
temperature := float32(0.01)
|
||||
modelConfig := &openai.ChatModelConfig{
|
||||
BaseURL: openaiBaseURL,
|
||||
APIKey: openaiAPIKey,
|
||||
Model: modelName,
|
||||
Timeout: defaultTimeout,
|
||||
Temperature: &temperature,
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return &ModelConfig{
|
||||
ChatModelConfig: modelConfig,
|
||||
ModelType: modelType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// maskAPIKey masks the API key
|
||||
func maskAPIKey(key string) string {
|
||||
if len(key) <= 8 {
|
||||
return "******"
|
||||
}
|
||||
|
||||
return key[:4] + "******" + key[len(key)-4:]
|
||||
}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/ark"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
EnvArkBaseURL = "ARK_BASE_URL"
|
||||
EnvArkAPIKey = "ARK_API_KEY"
|
||||
EnvArkModelID = "ARK_MODEL_ID"
|
||||
)
|
||||
|
||||
func GetArkModelConfig() (*ark.ChatModelConfig, error) {
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return nil, errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
|
||||
arkBaseURL := os.Getenv(EnvArkBaseURL)
|
||||
arkAPIKey := os.Getenv(EnvArkAPIKey)
|
||||
if arkAPIKey == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvArkAPIKey)
|
||||
}
|
||||
modelName := os.Getenv(EnvArkModelID)
|
||||
if modelName == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvArkModelID)
|
||||
}
|
||||
timeout := defaultTimeout
|
||||
|
||||
// https://www.volcengine.com/docs/82379/1494384?redirect=1
|
||||
temperature := float32(0.01) // [0, 2] 采样温度。控制了生成文本时对每个候选词的概率分布进行平滑的程度。
|
||||
// topP := float32(0.7) // [0, 1] 核采样概率阈值。模型会考虑概率质量在 top_p 内的 token 结果。
|
||||
// maxTokens := int(4096) // 模型可以生成的最大 token 数量。输入 token 和输出 token 的总长度还受模型的上下文长度限制。
|
||||
// frequencyPenalty := float32(0) // [-2, 2] 频率惩罚系数。如果值为正,会根据新 token 在文本中的出现频率对其进行惩罚,从而降低模型逐字重复的可能性。
|
||||
|
||||
modelConfig := &ark.ChatModelConfig{
|
||||
BaseURL: arkBaseURL,
|
||||
APIKey: arkAPIKey,
|
||||
Model: modelName,
|
||||
Timeout: &timeout,
|
||||
Temperature: &temperature,
|
||||
// TopP: &topP,
|
||||
// MaxTokens: &maxTokens,
|
||||
// FrequencyPenalty: &frequencyPenalty,
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return modelConfig, nil
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
openai2 "github.com/cloudwego/eino-ext/libs/acl/openai"
|
||||
"github.com/getkin/kin-openapi/openapi3gen"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
|
||||
EnvOpenAIAPIKey = "OPENAI_API_KEY"
|
||||
EnvModelName = "LLM_MODEL_NAME"
|
||||
)
|
||||
|
||||
// GetOpenAIModelConfig get OpenAI config
|
||||
func GetOpenAIModelConfig() (*openai.ChatModelConfig, error) {
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return nil, errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
|
||||
openaiBaseURL := os.Getenv(EnvOpenAIBaseURL)
|
||||
if openaiBaseURL == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIBaseURL)
|
||||
}
|
||||
openaiAPIKey := os.Getenv(EnvOpenAIAPIKey)
|
||||
if openaiAPIKey == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIAPIKey)
|
||||
}
|
||||
modelName := os.Getenv(EnvModelName)
|
||||
if modelName == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvModelName)
|
||||
}
|
||||
|
||||
type OutputFormat struct {
|
||||
Thought string `json:"thought"`
|
||||
Action string `json:"action"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modelConfig := &openai.ChatModelConfig{
|
||||
BaseURL: openaiBaseURL,
|
||||
APIKey: openaiAPIKey,
|
||||
Model: modelName,
|
||||
Timeout: defaultTimeout,
|
||||
// set structured response format
|
||||
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
|
||||
ResponseFormat: &openai2.ChatCompletionResponseFormat{
|
||||
Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
|
||||
JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
|
||||
Name: "thought_and_action",
|
||||
Description: "data that describes planning thought and action",
|
||||
Schema: outputFormatSchema.Value,
|
||||
Strict: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return modelConfig, nil
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOption(t *testing.T) {
|
||||
options := NewAIService(
|
||||
WithCVService(CVServiceTypeOpenCV),
|
||||
WithLLMService(LLMServiceTypeUITARS),
|
||||
)
|
||||
t.Log(options)
|
||||
}
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/ark"
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
openai2 "github.com/cloudwego/eino-ext/libs/acl/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/getkin/kin-openapi/openapi3gen"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
@@ -39,54 +41,57 @@ type AssertionResponse struct {
|
||||
// Asserter handles assertion using different AI models
|
||||
type Asserter struct {
|
||||
ctx context.Context
|
||||
modelConfig *ModelConfig
|
||||
model model.ToolCallingChatModel
|
||||
systemPrompt string
|
||||
history ConversationHistory
|
||||
modelType LLMServiceType
|
||||
}
|
||||
|
||||
// NewAsserter creates a new Asserter instance
|
||||
func NewAsserter(ctx context.Context, modelType LLMServiceType) (*Asserter, error) {
|
||||
func NewAsserter(ctx context.Context, modelConfig *ModelConfig) (*Asserter, error) {
|
||||
asserter := &Asserter{
|
||||
ctx: ctx,
|
||||
modelType: modelType,
|
||||
systemPrompt: getAssertionSystemPrompt(modelType),
|
||||
modelConfig: modelConfig,
|
||||
systemPrompt: defaultAssertionPrompt,
|
||||
}
|
||||
|
||||
switch modelType {
|
||||
case LLMServiceTypeUITARS:
|
||||
config, err := GetArkModelConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if modelConfig.ModelType == option.LLMServiceTypeUITARS {
|
||||
asserter.systemPrompt += "\n\n" + uiTarsAssertionResponseFormat
|
||||
} else if modelConfig.ModelType == option.LLMServiceTypeGPT {
|
||||
// define output format
|
||||
type OutputFormat struct {
|
||||
Thought string `json:"thought"`
|
||||
Pass bool `json:"pass"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
asserter.model, err = ark.NewChatModel(ctx, config)
|
||||
outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
|
||||
}
|
||||
case LLMServiceTypeGPT4Vision, LLMServiceTypeGPT4o:
|
||||
config, err := GetOpenAIModelConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// set structured response format
|
||||
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
|
||||
modelConfig.ChatModelConfig.ResponseFormat = &openai2.ChatCompletionResponseFormat{
|
||||
Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
|
||||
JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
|
||||
Name: "assertion_result",
|
||||
Description: "data that describes assertion result",
|
||||
Schema: outputFormatSchema.Value,
|
||||
Strict: false,
|
||||
},
|
||||
}
|
||||
asserter.model, err = openai.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("not supported model type for asserter")
|
||||
} else {
|
||||
asserter.systemPrompt += "\n\n" + defaultAssertionResponseJsonFormat
|
||||
}
|
||||
|
||||
var err error
|
||||
asserter.model, err = openai.NewChatModel(ctx, modelConfig.ChatModelConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
|
||||
}
|
||||
|
||||
return asserter, nil
|
||||
}
|
||||
|
||||
// getAssertionSystemPrompt returns the appropriate system prompt for the given model type
|
||||
func getAssertionSystemPrompt(modelType LLMServiceType) string {
|
||||
if modelType == LLMServiceTypeUITARS {
|
||||
return defaultAssertionPrompt + "\n\n" + uiTarsAssertionResponseFormat
|
||||
}
|
||||
return defaultAssertionPrompt + "\n\n" + defaultAssertionResponseJsonFormat
|
||||
}
|
||||
|
||||
// Assert performs the assertion check on the screenshot
|
||||
func (a *Asserter) Assert(opts *AssertOptions) (*AssertionResponse, error) {
|
||||
// Validate input parameters
|
||||
@@ -133,7 +138,7 @@ Here is the assertion. Please tell whether it is truthy according to the screens
|
||||
startTime := time.Now()
|
||||
resp, err := a.model.Generate(a.ctx, a.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", string(a.modelType)).Msg("call model service for assertion")
|
||||
Str("model", string(a.modelConfig.ModelType)).Msg("call model service for assertion")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
|
||||
}
|
||||
|
||||
@@ -1,23 +1,27 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func createAIService(t *testing.T) *AIServices {
|
||||
aiService := NewAIService(WithLLMService(LLMServiceTypeUITARS))
|
||||
require.NotNil(t, aiService)
|
||||
require.NotNil(t, aiService.ILLMService)
|
||||
return aiService
|
||||
func createAsserter(t *testing.T) *Asserter {
|
||||
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
|
||||
require.NoError(t, err)
|
||||
asserter, err := NewAsserter(context.Background(), modelConfig)
|
||||
require.NoError(t, err)
|
||||
return asserter
|
||||
}
|
||||
|
||||
// 测试有效断言
|
||||
func TestValidAssertions(t *testing.T) {
|
||||
aiService := createAIService(t)
|
||||
asserter := createAsserter(t)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
@@ -33,7 +37,7 @@ func TestValidAssertions(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "深度思考功能未开启",
|
||||
assertion: "输入框下方的「深度思考」文字是灰色的",
|
||||
assertion: "输入框下方的「深度思考」文字不是蓝色的",
|
||||
imagePath: "testdata/deepseek_think_off.png",
|
||||
expectPass: true,
|
||||
},
|
||||
@@ -47,10 +51,10 @@ func TestValidAssertions(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
imageBase64, size, err := loadImage(tc.imagePath)
|
||||
imageBase64, size, err := builtin.LoadImage(tc.imagePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := aiService.ILLMService.Assert(&AssertOptions{
|
||||
result, err := asserter.Assert(&AssertOptions{
|
||||
Assertion: tc.assertion,
|
||||
Screenshot: imageBase64,
|
||||
Size: size,
|
||||
@@ -58,14 +62,13 @@ func TestValidAssertions(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, tc.expectPass, result.Pass)
|
||||
assert.NotEmpty(t, result.Thought)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 测试无效参数
|
||||
func TestInvalidParameters(t *testing.T) {
|
||||
aiService := createAIService(t)
|
||||
asserter := createAsserter(t)
|
||||
testCases := []struct {
|
||||
name string
|
||||
assertion string
|
||||
@@ -91,7 +94,7 @@ func TestInvalidParameters(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := aiService.ILLMService.Assert(&AssertOptions{
|
||||
_, err := asserter.Assert(&AssertOptions{
|
||||
Assertion: tc.assertion,
|
||||
Screenshot: tc.screenshot,
|
||||
Size: tc.size,
|
||||
|
||||
@@ -22,6 +22,13 @@ type ICVService interface {
|
||||
ReadFromPath(imagePath string, opts ...option.ActionOption) (*CVResult, error)
|
||||
}
|
||||
|
||||
func NewCVService(modelType option.CVServiceType) (ICVService, error) {
|
||||
if modelType == option.CVServiceTypeVEDEM {
|
||||
return NewVEDEMImageService()
|
||||
}
|
||||
return nil, errors.New("invalid cv service type")
|
||||
}
|
||||
|
||||
type CVResult struct {
|
||||
URL string `json:"url,omitempty"` // image uploaded url
|
||||
OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts
|
||||
|
||||
@@ -19,9 +19,8 @@ func TestGetImageFromBuffer(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
buf.Read(file)
|
||||
|
||||
service := NewAIService(
|
||||
WithCVService(CVServiceTypeVEDEM),
|
||||
)
|
||||
service, err := NewVEDEMImageService()
|
||||
require.Nil(t, err)
|
||||
cvResult, err := service.ReadFromBuffer(buf)
|
||||
assert.Nil(t, err)
|
||||
fmt.Println(fmt.Sprintf("cvResult: %v", cvResult))
|
||||
@@ -29,9 +28,8 @@ func TestGetImageFromBuffer(t *testing.T) {
|
||||
|
||||
func TestGetImageFromPath(t *testing.T) {
|
||||
imagePath := "/Users/debugtalk/Downloads/s1.png"
|
||||
service := NewAIService(
|
||||
WithCVService(CVServiceTypeVEDEM),
|
||||
)
|
||||
service, err := NewVEDEMImageService()
|
||||
require.Nil(t, err)
|
||||
cvResult, err := service.ReadFromPath(imagePath)
|
||||
assert.Nil(t, err)
|
||||
fmt.Println(fmt.Sprintf("cvResult: %v", cvResult))
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/png"
|
||||
"os"
|
||||
"strings"
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type IPlanner interface {
|
||||
@@ -36,30 +32,110 @@ type PlanningResult struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ParsedAction represents a parsed action from the VLM response
|
||||
type ParsedAction struct {
|
||||
ActionType ActionType `json:"actionType"`
|
||||
ActionInputs map[string]interface{} `json:"actionInputs"`
|
||||
Thought string `json:"thought"`
|
||||
func NewPlanner(ctx context.Context, modelConfig *ModelConfig) (*Planner, error) {
|
||||
planner := &Planner{
|
||||
ctx: ctx,
|
||||
modelConfig: modelConfig,
|
||||
}
|
||||
|
||||
if modelConfig.ModelType == option.LLMServiceTypeUITARS {
|
||||
planner.systemPrompt = uiTarsPlanningPrompt
|
||||
} else {
|
||||
planner.systemPrompt = defaultPlanningResponseJsonFormat
|
||||
}
|
||||
|
||||
var err error
|
||||
planner.model, err = openai.NewChatModel(ctx, modelConfig.ChatModelConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
|
||||
}
|
||||
|
||||
return planner, nil
|
||||
}
|
||||
|
||||
type ActionType string
|
||||
type Planner struct {
|
||||
ctx context.Context
|
||||
modelConfig *ModelConfig
|
||||
model model.ToolCallingChatModel
|
||||
systemPrompt string
|
||||
history ConversationHistory
|
||||
}
|
||||
|
||||
const (
|
||||
ActionTypeClick ActionType = "click"
|
||||
ActionTypeTap ActionType = "tap"
|
||||
ActionTypeDrag ActionType = "drag"
|
||||
ActionTypeSwipe ActionType = "swipe"
|
||||
ActionTypeWait ActionType = "wait"
|
||||
ActionTypeFinished ActionType = "finished"
|
||||
ActionTypeCallUser ActionType = "call_user"
|
||||
ActionTypeType ActionType = "type"
|
||||
ActionTypeScroll ActionType = "scroll"
|
||||
)
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validatePlanningInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate planning parameters failed")
|
||||
}
|
||||
|
||||
const (
|
||||
defaultTimeout = 30 * time.Second
|
||||
)
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
p.history = ConversationHistory{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: p.systemPrompt + opts.UserInstruction,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
p.history.Append(opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", string(p.modelConfig.ModelType)).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
p.history.Append(&schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
var parseActions []ParsedAction
|
||||
var err error
|
||||
if p.modelConfig.ModelType == option.LLMServiceTypeUITARS {
|
||||
// parse Thought/Action format from UI-TARS
|
||||
parseActions, err = parseThoughtAction(msg.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// parse JSON format, from VLM like openai/gpt-4o
|
||||
parseActions, err = parseJSON(msg.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validatePlanningInput(opts *PlanningOptions) error {
|
||||
if opts.UserInstruction == "" {
|
||||
@@ -83,79 +159,3 @@ func validatePlanningInput(opts *PlanningOptions) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SavePositionImg saves an image with position markers
|
||||
func SavePositionImg(params struct {
|
||||
InputImgBase64 string
|
||||
Rect struct {
|
||||
X float64
|
||||
Y float64
|
||||
}
|
||||
OutputPath string
|
||||
}) error {
|
||||
// 解码Base64图像
|
||||
imgData := params.InputImgBase64
|
||||
// 如果包含了数据URL前缀,去掉它
|
||||
if strings.HasPrefix(imgData, "data:image/") {
|
||||
parts := strings.Split(imgData, ",")
|
||||
if len(parts) > 1 {
|
||||
imgData = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
// 解码Base64
|
||||
unbased, err := base64.StdEncoding.DecodeString(imgData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法解码Base64图像: %w", err)
|
||||
}
|
||||
|
||||
// 解码图像
|
||||
reader := bytes.NewReader(unbased)
|
||||
img, _, err := image.Decode(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法解码图像数据: %w", err)
|
||||
}
|
||||
|
||||
// 创建一个可以在其上绘制的图像
|
||||
bounds := img.Bounds()
|
||||
rgba := image.NewRGBA(bounds)
|
||||
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
|
||||
|
||||
// 在点击/拖动位置绘制标记
|
||||
markRadius := 10
|
||||
x, y := int(params.Rect.X), int(params.Rect.Y)
|
||||
|
||||
// 绘制红色圆圈
|
||||
for i := -markRadius; i <= markRadius; i++ {
|
||||
for j := -markRadius; j <= markRadius; j++ {
|
||||
if i*i+j*j <= markRadius*markRadius {
|
||||
if x+i >= 0 && x+i < bounds.Max.X && y+j >= 0 && y+j < bounds.Max.Y {
|
||||
rgba.Set(x+i, y+j, color.RGBA{255, 0, 0, 255})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 保存图像
|
||||
outFile, err := os.Create(params.OutputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("无法创建输出文件: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// 编码为PNG并保存
|
||||
if err := png.Encode(outFile, rgba); err != nil {
|
||||
return fmt.Errorf("无法编码和保存图像: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// maskAPIKey masks the API key
|
||||
func maskAPIKey(key string) string {
|
||||
if len(key) <= 8 {
|
||||
return "******"
|
||||
}
|
||||
|
||||
return key[:4] + "******" + key[len(key)-4:]
|
||||
}
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
_ "image/jpeg"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
|
||||
func NewPlanner(ctx context.Context) (*Planner, error) {
|
||||
config, err := GetOpenAIModelConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
|
||||
}
|
||||
model, err := openai.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
|
||||
}
|
||||
return &Planner{
|
||||
ctx: ctx,
|
||||
model: model,
|
||||
modelType: LLMServiceTypeGPT4o,
|
||||
systemPrompt: uiTarsPlanningPrompt, // TODO: change prompt with function calling
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Planner struct {
|
||||
ctx context.Context
|
||||
model model.ToolCallingChatModel
|
||||
systemPrompt string
|
||||
modelType LLMServiceType
|
||||
history ConversationHistory
|
||||
}
|
||||
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validatePlanningInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate planning parameters failed")
|
||||
}
|
||||
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
|
||||
p.history = ConversationHistory{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
p.history.Append(opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", string(p.modelType)).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
p.history.Append(&schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse JSON format, from VLM like openai/gpt-4o
|
||||
parseActions, jsonErr := parseJSON(msg.Content)
|
||||
if jsonErr != nil {
|
||||
return nil, jsonErr
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// parseJSON tries to parse the response as JSON format
|
||||
func parseJSON(predictionText string) ([]ParsedAction, error) {
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
|
||||
predictionText = strings.TrimPrefix(predictionText, "```json")
|
||||
predictionText = strings.TrimSuffix(predictionText, "```")
|
||||
}
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
|
||||
var response PlanningResult
|
||||
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
|
||||
if len(response.NextActions) == 0 {
|
||||
return nil, errors.New("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// normalize actions
|
||||
var normalizedActions []ParsedAction
|
||||
for i := range response.NextActions {
|
||||
// create a new variable, avoid implicit memory aliasing in for loop.
|
||||
action := response.NextActions[i]
|
||||
if err := normalizeAction(&action); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to normalize action")
|
||||
}
|
||||
normalizedActions = append(normalizedActions, action)
|
||||
}
|
||||
|
||||
return normalizedActions, nil
|
||||
}
|
||||
|
||||
// normalizeAction normalizes the coordinates in the action
|
||||
func normalizeAction(action *ParsedAction) error {
|
||||
switch action.ActionType {
|
||||
case "click", "drag":
|
||||
// handle click and drag action coordinates
|
||||
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(startBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize startBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["startBox"] = normalized
|
||||
}
|
||||
|
||||
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(endBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize endBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["endBox"] = normalized
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,116 +1,38 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/ark"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func NewUITarsPlanner(ctx context.Context) (*UITarsPlanner, error) {
|
||||
config, err := GetArkModelConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chatModel, err := ark.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &UITarsPlanner{
|
||||
ctx: ctx,
|
||||
model: chatModel,
|
||||
modelType: LLMServiceTypeUITARS,
|
||||
systemPrompt: uiTarsPlanningPrompt,
|
||||
}, nil
|
||||
// ParsedAction represents a parsed action from the VLM response
|
||||
type ParsedAction struct {
|
||||
ActionType ActionType `json:"actionType"`
|
||||
ActionInputs map[string]interface{} `json:"actionInputs"`
|
||||
Thought string `json:"thought"`
|
||||
}
|
||||
|
||||
type UITarsPlanner struct {
|
||||
ctx context.Context
|
||||
model model.ToolCallingChatModel
|
||||
systemPrompt string
|
||||
modelType LLMServiceType
|
||||
history ConversationHistory
|
||||
}
|
||||
type ActionType string
|
||||
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *UITarsPlanner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validatePlanningInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate planning parameters failed")
|
||||
}
|
||||
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
|
||||
p.history = ConversationHistory{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
p.history.Append(opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", string(p.modelType)).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
p.history.Append(&schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *UITarsPlanner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse Thought/Action format from UI-TARS
|
||||
parseActions, thoughtErr := parseThoughtAction(msg.Content)
|
||||
if thoughtErr != nil {
|
||||
return nil, thoughtErr
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
const (
|
||||
ActionTypeClick ActionType = "click"
|
||||
ActionTypeTap ActionType = "tap"
|
||||
ActionTypeDrag ActionType = "drag"
|
||||
ActionTypeSwipe ActionType = "swipe"
|
||||
ActionTypeWait ActionType = "wait"
|
||||
ActionTypeFinished ActionType = "finished"
|
||||
ActionTypeCallUser ActionType = "call_user"
|
||||
ActionTypeType ActionType = "type"
|
||||
ActionTypeScroll ActionType = "scroll"
|
||||
)
|
||||
|
||||
// parseThoughtAction parses the Thought/Action format response
|
||||
func parseThoughtAction(predictionText string) ([]ParsedAction, error) {
|
||||
@@ -396,3 +318,64 @@ func validateTypeContent(action *ParsedAction) {
|
||||
log.Warn().Msg("type action missing content parameter, set to default")
|
||||
}
|
||||
}
|
||||
|
||||
// parseJSON tries to parse the response as JSON format
|
||||
func parseJSON(predictionText string) ([]ParsedAction, error) {
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
|
||||
predictionText = strings.TrimPrefix(predictionText, "```json")
|
||||
predictionText = strings.TrimSuffix(predictionText, "```")
|
||||
}
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
|
||||
var response PlanningResult
|
||||
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
|
||||
if len(response.NextActions) == 0 {
|
||||
return nil, errors.New("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// normalize actions
|
||||
var normalizedActions []ParsedAction
|
||||
for i := range response.NextActions {
|
||||
// create a new variable, avoid implicit memory aliasing in for loop.
|
||||
action := response.NextActions[i]
|
||||
if err := normalizeAction(&action); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to normalize action")
|
||||
}
|
||||
normalizedActions = append(normalizedActions, action)
|
||||
}
|
||||
|
||||
return normalizedActions, nil
|
||||
}
|
||||
|
||||
// normalizeAction normalizes the coordinates in the action
|
||||
func normalizeAction(action *ParsedAction) error {
|
||||
switch action.ActionType {
|
||||
case "click", "drag":
|
||||
// handle click and drag action coordinates
|
||||
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(startBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize startBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["startBox"] = normalized
|
||||
}
|
||||
|
||||
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(endBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize endBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["endBox"] = normalized
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -27,3 +27,5 @@ finished(content='xxx') # Use escape characters \\', \\", and \\n in content par
|
||||
|
||||
## User Instruction
|
||||
`
|
||||
|
||||
const defaultPlanningResponseJsonFormat = ``
|
||||
|
||||
@@ -1,25 +1,20 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVLMPlanning(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/llk_1.png")
|
||||
imageBase64, size, err := builtin.LoadImage("testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := `连连看是一款经典的益智消除类小游戏,通常以图案或图标为主要元素。以下是连连看的基本规则说明:
|
||||
@@ -35,7 +30,10 @@ func TestVLMPlanning(t *testing.T) {
|
||||
|
||||
userInstruction += "\n\n请基于以上游戏规则,给出下一步可点击的两个图标坐标"
|
||||
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
|
||||
require.NoError(t, err)
|
||||
|
||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -100,12 +98,15 @@ func TestVLMPlanning(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestXHSPlanning(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/xhs-feed.jpeg")
|
||||
imageBase64, size, err := builtin.LoadImage("testdata/xhs-feed.jpeg")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := "点击第二个帖子的作者头像"
|
||||
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
|
||||
require.NoError(t, err)
|
||||
|
||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -170,12 +171,15 @@ func TestXHSPlanning(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestChatList(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/chat_list.jpeg")
|
||||
imageBase64, size, err := builtin.LoadImage("testdata/chat_list.jpeg")
|
||||
require.NoError(t, err)
|
||||
|
||||
userInstruction := "请结合图片的文字信息,请告诉我一共有多少个群聊,哪些群聊右下角有绿点"
|
||||
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
|
||||
require.NoError(t, err)
|
||||
|
||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -206,7 +210,10 @@ func TestHandleSwitch(t *testing.T) {
|
||||
userInstruction := "发送框下方的联网搜索开关是开启状态" // 点击开启联网搜索开关
|
||||
// 检查发送框下方的联网搜索开关,蓝色为开启状态,灰色为关闭状态;若开关处于关闭状态,则点击进行开启
|
||||
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
|
||||
require.NoError(t, err)
|
||||
|
||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
||||
require.NoError(t, err)
|
||||
|
||||
testCases := []struct {
|
||||
@@ -219,7 +226,7 @@ func TestHandleSwitch(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
imageBase64, size, err := loadImage(tc.imageFile)
|
||||
imageBase64, size, err := builtin.LoadImage(tc.imageFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -250,7 +257,7 @@ func TestHandleSwitch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateInput(t *testing.T) {
|
||||
imageBase64, size, err := loadImage("testdata/popup_risk_warning.png")
|
||||
imageBase64, size, err := builtin.LoadImage("testdata/popup_risk_warning.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
@@ -375,87 +382,18 @@ func TestProcessVLMResponse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSavePositionImg(t *testing.T) {
|
||||
imageBase64, _, err := loadImage("testdata/popup_risk_warning.png")
|
||||
require.NoError(t, err)
|
||||
|
||||
params := struct {
|
||||
InputImgBase64 string
|
||||
Rect struct {
|
||||
X float64
|
||||
Y float64
|
||||
}
|
||||
OutputPath string
|
||||
}{
|
||||
InputImgBase64: imageBase64,
|
||||
Rect: struct {
|
||||
X float64
|
||||
Y float64
|
||||
}{
|
||||
X: 100,
|
||||
Y: 100,
|
||||
},
|
||||
OutputPath: "testdata/output.png",
|
||||
}
|
||||
|
||||
err = SavePositionImg(params)
|
||||
require.NoError(t, err)
|
||||
|
||||
// cleanup
|
||||
defer os.Remove(params.OutputPath)
|
||||
}
|
||||
|
||||
func TestLoadImage(t *testing.T) {
|
||||
// Test PNG image
|
||||
pngBase64, pngSize, err := loadImage("testdata/llk_1.png")
|
||||
pngBase64, pngSize, err := builtin.LoadImage("testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, pngBase64)
|
||||
assert.Greater(t, pngSize.Width, 0)
|
||||
assert.Greater(t, pngSize.Height, 0)
|
||||
|
||||
// Test JPEG image
|
||||
jpegBase64, jpegSize, err := loadImage("testdata/xhs-feed.jpeg")
|
||||
jpegBase64, jpegSize, err := builtin.LoadImage("testdata/xhs-feed.jpeg")
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, jpegBase64)
|
||||
assert.Greater(t, jpegSize.Width, 0)
|
||||
assert.Greater(t, jpegSize.Height, 0)
|
||||
}
|
||||
|
||||
// loadImage loads image and returns base64 encoded string
|
||||
func loadImage(imagePath string) (base64Str string, size types.Size, err error) {
|
||||
// Read the image file
|
||||
imageFile, err := os.OpenFile(imagePath, os.O_RDONLY, 0o600)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
|
||||
}
|
||||
defer imageFile.Close()
|
||||
|
||||
// Decode the image to get its resolution
|
||||
imageData, format, err := image.Decode(imageFile)
|
||||
if err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
|
||||
}
|
||||
|
||||
// Get the resolution of the image
|
||||
width := imageData.Bounds().Dx()
|
||||
height := imageData.Bounds().Dy()
|
||||
size = types.Size{Width: width, Height: height}
|
||||
|
||||
// Convert image to base64
|
||||
buf := new(bytes.Buffer)
|
||||
// 根据图像格式选择正确的编码器
|
||||
if format == "jpeg" || format == "jpg" {
|
||||
if err := jpeg.Encode(buf, imageData, nil); err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
|
||||
}
|
||||
} else {
|
||||
// 默认使用 PNG 编码
|
||||
if err := png.Encode(buf, imageData); err != nil {
|
||||
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
|
||||
}
|
||||
}
|
||||
base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
|
||||
base64.StdEncoding.EncodeToString(buf.Bytes()))
|
||||
|
||||
return base64Str, size, nil
|
||||
}
|
||||
|
||||
@@ -307,14 +307,16 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
|
||||
|
||||
func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY")
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
var err error
|
||||
x, y, err = handlerTapAbsXY(ad, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// adb shell input tap x y
|
||||
xStr := fmt.Sprintf("%.1f", x)
|
||||
yStr := fmt.Sprintf("%.1f", y)
|
||||
_, err := ad.runShellCommand(
|
||||
"input", "tap", xStr, yStr)
|
||||
_, err = ad.runShellCommand("input", "tap", xStr, yStr)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr))
|
||||
}
|
||||
@@ -324,12 +326,10 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
|
||||
func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap")
|
||||
var err error
|
||||
x, y, err = convertToAbsolutePoint(ad, x, y)
|
||||
x, y, err = handlerDoubleTap(ad, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
// adb shell input tap x y
|
||||
xStr := fmt.Sprintf("%.1f", x)
|
||||
@@ -372,13 +372,13 @@ func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er
|
||||
func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) {
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag")
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY)
|
||||
|
||||
fromX, fromY, toX, toY, err = handlerDrag(ad, fromX, fromY, toX, toY, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
duration := 200.0
|
||||
if actionOptions.Duration > 0 {
|
||||
duration = actionOptions.Duration * 1000
|
||||
@@ -404,12 +404,10 @@ func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe")
|
||||
var err error
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY)
|
||||
fromX, fromY, toX, toY, err = handlerSwipe(ad, fromX, fromY, toX, toY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
// adb shell input swipe fromX fromY toX toY
|
||||
_, err = ad.runShellCommand(
|
||||
@@ -701,7 +699,7 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) {
|
||||
return pointRes, nil
|
||||
}
|
||||
|
||||
reader, err := os.OpenFile(files[0], os.O_RDONLY, 0o600)
|
||||
reader, err := os.Open(files[0])
|
||||
if err != nil {
|
||||
log.Info().Msg("open File error")
|
||||
return pointRes, nil
|
||||
|
||||
@@ -256,12 +256,10 @@ func (ud *UIA2Driver) Orientation() (orientation types.Orientation, err error) {
|
||||
func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap")
|
||||
var err error
|
||||
x, y, err = convertToAbsolutePoint(ud, x, y)
|
||||
x, y, err = handlerDoubleTap(ud, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
data := map[string]interface{}{
|
||||
"actions": []interface{}{
|
||||
@@ -298,13 +296,19 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error {
|
||||
func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY")
|
||||
// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
var err error
|
||||
x, y, err = handlerTapAbsXY(ud, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
duration := 100.0
|
||||
if actionOptions.PressDuration > 0 {
|
||||
duration = actionOptions.PressDuration * 1000 // convert to ms
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"actions": []interface{}{
|
||||
map[string]interface{}{
|
||||
@@ -323,7 +327,7 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error
|
||||
option.MergeOptions(data, opts...)
|
||||
|
||||
urlStr := fmt.Sprintf("/session/%s/actions/tap", ud.Session.ID)
|
||||
_, err := ud.Session.POST(data, urlStr)
|
||||
_, err = ud.Session.POST(data, urlStr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -355,13 +359,12 @@ func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (e
|
||||
func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag")
|
||||
|
||||
var err error
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY)
|
||||
fromX, fromY, toX, toY, err = handlerDrag(ud, fromX, fromY, toX, toY, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
data := map[string]interface{}{
|
||||
"startX": fromX,
|
||||
@@ -387,13 +390,11 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe")
|
||||
var err error
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY)
|
||||
fromX, fromY, toX, toY, err = handlerSwipe(ud, fromX, fromY, toX, toY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
duration := 200.0
|
||||
if actionOptions.PressDuration > 0 {
|
||||
duration = actionOptions.PressDuration * 1000 // ms
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
@@ -23,10 +22,12 @@ func setupADBDriverExt(t *testing.T) *XTDriver {
|
||||
device.Options.LogOn = false
|
||||
driver, err := device.NewDriver()
|
||||
require.Nil(t, err)
|
||||
return NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM),
|
||||
ai.WithLLMService(ai.LLMServiceTypeUITARS),
|
||||
driverExt, err := NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM),
|
||||
option.WithLLMService(option.LLMServiceTypeUITARS),
|
||||
)
|
||||
require.Nil(t, err)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
func setupUIA2DriverExt(t *testing.T) *XTDriver {
|
||||
@@ -36,8 +37,10 @@ func setupUIA2DriverExt(t *testing.T) *XTDriver {
|
||||
device.Options.LogOn = false
|
||||
driver, err := device.NewDriver()
|
||||
require.Nil(t, err)
|
||||
return NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
func TestDevice_Android_GetPackageInfo(t *testing.T) {
|
||||
|
||||
@@ -103,7 +103,13 @@ func NewBrowserDriver(device *BrowserDevice) (driver *BrowserDriver, err error)
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option.ActionOption) (err error) {
|
||||
func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
|
||||
var err error
|
||||
fromX, fromY, toX, toY, err = handlerDrag(wd, fromX, fromY, toX, toY, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"from_x": fromX,
|
||||
"from_y": fromY,
|
||||
@@ -111,14 +117,13 @@ func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option.
|
||||
"to_y": toY,
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(options...)
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
if actionOptions.Duration > 0 {
|
||||
data["duration"] = actionOptions.Duration
|
||||
}
|
||||
|
||||
_, err = wd.HttpPOST(data, wd.sessionId, "ui/drag")
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) AppLaunch(packageName string) (err error) {
|
||||
@@ -517,28 +522,40 @@ func (wd *BrowserDriver) Tap(x, y float64, options ...option.ActionOption) error
|
||||
return wd.TapFloat(x, y, options...)
|
||||
}
|
||||
|
||||
func (wd *BrowserDriver) TapFloat(x, y float64, options ...option.ActionOption) error {
|
||||
actionOptions := option.NewActionOptions(options...)
|
||||
func (wd *BrowserDriver) TapFloat(x, y float64, opts ...option.ActionOption) error {
|
||||
var err error
|
||||
x, y, err = handlerTapAbsXY(wd, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
duration := 0.1
|
||||
if actionOptions.Duration > 0 {
|
||||
duration = actionOptions.Duration
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"x": x,
|
||||
"y": y,
|
||||
"duration": duration,
|
||||
}
|
||||
_, err := wd.HttpPOST(data, wd.sessionId, "ui/tap")
|
||||
_, err = wd.HttpPOST(data, wd.sessionId, "ui/tap")
|
||||
return err
|
||||
}
|
||||
|
||||
// DoubleTap Sends a double tap event at the coordinate.
|
||||
func (wd *BrowserDriver) DoubleTap(x, y float64, options ...option.ActionOption) error {
|
||||
var err error
|
||||
x, y, err = handlerDoubleTap(wd, x, y, options...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data := map[string]interface{}{
|
||||
"x": x,
|
||||
"y": y,
|
||||
}
|
||||
_, err := wd.HttpPOST(data, wd.sessionId, "ui/double_tap")
|
||||
_, err = wd.HttpPOST(data, wd.sessionId, "ui/double_tap")
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -23,9 +22,10 @@ func TestIOSDemo(t *testing.T) {
|
||||
|
||||
driver, err := device.NewDriver()
|
||||
assert.Nil(t, err)
|
||||
driverExt := uixt.NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM),
|
||||
driverExt, err := uixt.NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// release session
|
||||
defer func() {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -81,16 +82,30 @@ type IDriver interface {
|
||||
StopCaptureLog() (result interface{}, err error)
|
||||
}
|
||||
|
||||
func NewXTDriver(driver IDriver, opts ...ai.AIServiceOption) *XTDriver {
|
||||
services := ai.NewAIService(opts...)
|
||||
func NewXTDriver(driver IDriver, opts ...option.AIServiceOption) (*XTDriver, error) {
|
||||
driverExt := &XTDriver{
|
||||
IDriver: driver,
|
||||
CVService: services.ICVService,
|
||||
LLMService: services.ILLMService,
|
||||
|
||||
screenResults: make([]*ScreenResult, 0),
|
||||
IDriver: driver,
|
||||
}
|
||||
return driverExt
|
||||
|
||||
services := option.NewAIServiceOptions(opts...)
|
||||
|
||||
var err error
|
||||
if services.CVService != "" {
|
||||
driverExt.CVService, err = ai.NewCVService(services.CVService)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init vedem image service failed")
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if services.LLMService != "" {
|
||||
driverExt.LLMService, err = ai.NewLLMService(services.LLMService)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init llm service failed")
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return driverExt, nil
|
||||
}
|
||||
|
||||
// XTDriver = IDriver + AI
|
||||
@@ -98,7 +113,4 @@ type XTDriver struct {
|
||||
IDriver
|
||||
CVService ai.ICVService // OCR/CV
|
||||
LLMService ai.ILLMService // LLM
|
||||
|
||||
// cache screenshot results
|
||||
screenResults []*ScreenResult
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ const (
|
||||
ACTION_TapByCV ActionMethod = "tap_cv"
|
||||
ACTION_DoubleTapXY ActionMethod = "double_tap_xy"
|
||||
ACTION_Swipe ActionMethod = "swipe"
|
||||
ACTION_Drag ActionMethod = "drag"
|
||||
ACTION_Input ActionMethod = "input"
|
||||
ACTION_Back ActionMethod = "back"
|
||||
ACTION_KeyCode ActionMethod = "keycode"
|
||||
|
||||
@@ -4,12 +4,16 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
@@ -129,8 +133,9 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
|
||||
}
|
||||
}
|
||||
|
||||
// cache screen result
|
||||
dExt.screenResults = append(dExt.screenResults, screenResult)
|
||||
// save screen result to session
|
||||
session := dExt.GetSession()
|
||||
session.screenResults = append(session.screenResults, screenResult)
|
||||
|
||||
log.Debug().
|
||||
Str("imagePath", imagePath).
|
||||
@@ -294,3 +299,255 @@ func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error
|
||||
// return compressed image buffer
|
||||
return &buf, nil
|
||||
}
|
||||
|
||||
// MarkUIOperation add operation mark for UI operation
|
||||
func MarkUIOperation(driver IDriver, actionType ActionMethod, actionCoordinates []float64) error {
|
||||
if actionType == "" || len(actionCoordinates) == 0 {
|
||||
return nil
|
||||
}
|
||||
start := time.Now()
|
||||
|
||||
// get screenshot
|
||||
compressedBufSource, err := driver.ScreenShot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create screenshot save path
|
||||
timestamp := builtin.GenNameWithTimestamp("action_%d")
|
||||
var imagePath string
|
||||
|
||||
if actionType == ACTION_TapAbsXY || actionType == ACTION_DoubleTapXY {
|
||||
if len(actionCoordinates) != 2 {
|
||||
return fmt.Errorf("invalid tap action coordinates: %v", actionCoordinates)
|
||||
}
|
||||
imagePath = filepath.Join(
|
||||
config.GetConfig().ScreenShotsPath,
|
||||
fmt.Sprintf("%s_%s.png", timestamp, actionType),
|
||||
)
|
||||
x, y := actionCoordinates[0], actionCoordinates[1]
|
||||
point := image.Point{X: int(x), Y: int(y)}
|
||||
err = SaveImageWithCircleMarker(compressedBufSource, point, imagePath)
|
||||
} else if actionType == ACTION_Swipe || actionType == ACTION_Drag {
|
||||
if len(actionCoordinates) != 4 {
|
||||
return fmt.Errorf("invalid swipe action coordinates: %v", actionCoordinates)
|
||||
}
|
||||
imagePath = filepath.Join(
|
||||
config.GetConfig().ScreenShotsPath,
|
||||
fmt.Sprintf("%s_%s.png", timestamp, actionType),
|
||||
)
|
||||
fromX, fromY := actionCoordinates[0], actionCoordinates[1]
|
||||
toX, toY := actionCoordinates[2], actionCoordinates[3]
|
||||
from := image.Point{X: int(fromX), Y: int(fromY)}
|
||||
to := image.Point{X: int(toX), Y: int(toY)}
|
||||
err = SaveImageWithArrowMarker(compressedBufSource, from, to, imagePath)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error().Err(err).
|
||||
Int64("duration(ms)", time.Since(start).Milliseconds()).
|
||||
Msg("mark UI operation failed")
|
||||
return err
|
||||
}
|
||||
|
||||
if imagePath != "" {
|
||||
log.Info().Str("operation", string(actionType)).
|
||||
Str("imagePath", imagePath).
|
||||
Int64("duration(ms)", time.Since(start).Milliseconds()).
|
||||
Msg("mark UI operation success")
|
||||
|
||||
// save screenshot to session
|
||||
session := driver.GetSession()
|
||||
session.screenResults = append(session.screenResults, &ScreenResult{
|
||||
bufSource: compressedBufSource,
|
||||
ImagePath: imagePath,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveImageWithCircleMarker saves an image with circle marker
|
||||
func SaveImageWithCircleMarker(imgBuf *bytes.Buffer, point image.Point, outputPath string) error {
|
||||
img, _, err := image.Decode(imgBuf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode image data: %w", err)
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
rgba := image.NewRGBA(bounds)
|
||||
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
|
||||
|
||||
// draw a red circle at the tap point
|
||||
centerX := point.X
|
||||
centerY := point.Y
|
||||
radius := 20
|
||||
lineWidth := 5
|
||||
red := color.RGBA{255, 0, 0, 255}
|
||||
|
||||
for angle := 0.0; angle < 2*math.Pi; angle += 0.01 {
|
||||
for w := 0; w < lineWidth; w++ {
|
||||
r := float64(radius - w)
|
||||
x := int(float64(centerX) + r*math.Cos(angle))
|
||||
y := int(float64(centerY) + r*math.Sin(angle))
|
||||
if x >= 0 && x < bounds.Max.X && y >= 0 && y < bounds.Max.Y {
|
||||
rgba.Set(x, y, red)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
outFile, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
if err := png.Encode(outFile, rgba); err != nil {
|
||||
return fmt.Errorf("failed to encode and save image: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveImageWithArrowMarker saves an image with an arrow marker
|
||||
func SaveImageWithArrowMarker(imgBuf *bytes.Buffer, from, to image.Point, outputPath string) error {
|
||||
img, _, err := image.Decode(imgBuf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode image data: %w", err)
|
||||
}
|
||||
bounds := img.Bounds()
|
||||
rgba := image.NewRGBA(bounds)
|
||||
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
|
||||
drawArrow(rgba, from, to, color.RGBA{255, 0, 0, 255}, 5)
|
||||
outFile, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file: %w", err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
if err := png.Encode(outFile, rgba); err != nil {
|
||||
return fmt.Errorf("failed to encode and save image: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// drawArrow draws an arrow from 'from' to 'to' on the image
|
||||
func drawArrow(rgba *image.RGBA, from, to image.Point, color color.RGBA, lineWidth int) {
|
||||
bounds := rgba.Bounds()
|
||||
dx, dy := to.X-from.X, to.Y-from.Y
|
||||
steps := int(math.Sqrt(float64(dx*dx + dy*dy)))
|
||||
if steps == 0 {
|
||||
steps = 1
|
||||
}
|
||||
stepX, stepY := float64(dx)/float64(steps), float64(dy)/float64(steps)
|
||||
// main line
|
||||
for i := 0; i < steps; i++ {
|
||||
x := int(float64(from.X) + stepX*float64(i))
|
||||
y := int(float64(from.Y) + stepY*float64(i))
|
||||
for w := 0; w < lineWidth; w++ {
|
||||
offsetX, offsetY := 0, 0
|
||||
if math.Abs(stepX) > math.Abs(stepY) {
|
||||
offsetY = w - lineWidth/2
|
||||
} else {
|
||||
offsetX = w - lineWidth/2
|
||||
}
|
||||
drawX, drawY := x+offsetX, y+offsetY
|
||||
if drawX >= 0 && drawX < bounds.Max.X && drawY >= 0 && drawY < bounds.Max.Y {
|
||||
rgba.Set(drawX, drawY, color)
|
||||
}
|
||||
}
|
||||
}
|
||||
// arrow head
|
||||
arrowLength := float64(steps) * 0.15
|
||||
if arrowLength < 10 {
|
||||
arrowLength = 10
|
||||
} else if arrowLength > 30 {
|
||||
arrowLength = 30
|
||||
}
|
||||
head := calculateArrowHead(float64(from.X), float64(from.Y), float64(to.X), float64(to.Y), arrowLength)
|
||||
if head != nil {
|
||||
for _, point := range head[:2] {
|
||||
drawLineInImage(rgba, to.X, to.Y, int(point.X), int(point.Y), color, lineWidth, bounds)
|
||||
}
|
||||
for _, point := range head[1:] {
|
||||
drawLineInImage(rgba, to.X, to.Y, int(point.X), int(point.Y), color, lineWidth, bounds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// calculateArrowHead calculates the endpoint and arrowhead coordinates
|
||||
func calculateArrowHead(fromX, fromY, toX, toY float64, arrowLength float64) []struct{ X, Y float64 } {
|
||||
// calculate direction vector
|
||||
dx, dy := toX-fromX, toY-fromY
|
||||
// calculate distance
|
||||
length := math.Sqrt(dx*dx + dy*dy)
|
||||
if length < 1e-6 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// unit vector
|
||||
dx, dy = dx/length, dy/length
|
||||
|
||||
// calculate orthogonal vector of arrow direction (counterclockwise 90 degrees)
|
||||
orthX, orthY := -dy, dx
|
||||
|
||||
// calculate two wing points of arrow
|
||||
headWidth := arrowLength * 0.5
|
||||
backX, backY := toX-dx*arrowLength, toY-dy*arrowLength
|
||||
|
||||
// two wing points of arrow
|
||||
leftWingX, leftWingY := backX+orthX*headWidth, backY+orthY*headWidth
|
||||
rightWingX, rightWingY := backX-orthX*headWidth, backY-orthY*headWidth
|
||||
|
||||
return []struct{ X, Y float64 }{
|
||||
{leftWingX, leftWingY},
|
||||
{toX, toY},
|
||||
{rightWingX, rightWingY},
|
||||
}
|
||||
}
|
||||
|
||||
// drawLineInImage draws a line on the image
|
||||
func drawLineInImage(img *image.RGBA, x0, y0, x1, y1 int, lineColor color.RGBA, lineWidth int, bounds image.Rectangle) {
|
||||
// use Bresenham algorithm to draw line
|
||||
dx, dy := math.Abs(float64(x1-x0)), math.Abs(float64(y1-y0))
|
||||
sx, sy := 1, 1
|
||||
if x0 >= x1 {
|
||||
sx = -1
|
||||
}
|
||||
if y0 >= y1 {
|
||||
sy = -1
|
||||
}
|
||||
err := dx - dy
|
||||
|
||||
for {
|
||||
// draw point (consider line width)
|
||||
for w := 0; w < lineWidth; w++ {
|
||||
offsetX, offsetY := 0, 0
|
||||
|
||||
// decide offset direction based on line angle
|
||||
if dx > dy {
|
||||
// more horizontal line
|
||||
offsetY = w - lineWidth/2
|
||||
} else {
|
||||
// more vertical line
|
||||
offsetX = w - lineWidth/2
|
||||
}
|
||||
|
||||
drawX, drawY := x0+offsetX, y0+offsetY
|
||||
if drawX >= 0 && drawX < bounds.Max.X && drawY >= 0 && drawY < bounds.Max.Y {
|
||||
img.Set(drawX, drawY, lineColor)
|
||||
}
|
||||
}
|
||||
|
||||
// end of line
|
||||
if x0 == x1 && y0 == y1 {
|
||||
break
|
||||
}
|
||||
|
||||
// calculate next point
|
||||
e2 := 2 * err
|
||||
if e2 > -dy {
|
||||
err = err - dy
|
||||
x0 = x0 + sx
|
||||
}
|
||||
if e2 < dx {
|
||||
err = err + dx
|
||||
y0 = y0 + sy
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,10 +3,12 @@
|
||||
package uixt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -17,8 +19,9 @@ func TestDriverExt_NewMethod1(t *testing.T) {
|
||||
require.Nil(t, err)
|
||||
driver, err := device.NewDriver()
|
||||
require.Nil(t, err)
|
||||
driverExt := NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
driverExt.TapByOCR("推荐")
|
||||
}
|
||||
|
||||
@@ -27,16 +30,18 @@ func TestDriverExt_NewMethod2(t *testing.T) {
|
||||
require.Nil(t, err)
|
||||
driver, err := NewUIA2Driver(device)
|
||||
require.Nil(t, err)
|
||||
driverExt := NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
driverExt.TapByOCR("推荐")
|
||||
}
|
||||
|
||||
func TestDriverExt(t *testing.T) {
|
||||
device, _ := NewAndroidDevice()
|
||||
driver, _ := NewADBDriver(device)
|
||||
driverExt := NewXTDriver(driver,
|
||||
ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(driver,
|
||||
option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
|
||||
// call IDriver methods
|
||||
driverExt.TapXY(0.2, 0.5)
|
||||
@@ -50,7 +55,7 @@ func TestDriverExt(t *testing.T) {
|
||||
textRect, _ := driverExt.FindScreenText("hello")
|
||||
t.Log(textRect)
|
||||
|
||||
err := driverExt.TapByCV(
|
||||
err = driverExt.TapByCV(
|
||||
option.WithScreenShotUITypes("deepseek_send"),
|
||||
option.WithScope(0.8, 0.5, 1, 1))
|
||||
assert.Nil(t, err)
|
||||
@@ -249,3 +254,51 @@ func TestDriverExt_Action_Offset(t *testing.T) {
|
||||
option.WithTapRandomRect(true))
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestSaveImageWithCircle(t *testing.T) {
|
||||
imgBytes, err := os.ReadFile("ai/testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
imgBuf := bytes.NewBuffer(imgBytes)
|
||||
|
||||
point := image.Point{X: 500, Y: 500}
|
||||
outputPath := "ai/testdata/output.png"
|
||||
|
||||
err = SaveImageWithCircleMarker(imgBuf, point, outputPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer os.Remove(outputPath)
|
||||
}
|
||||
|
||||
func TestSaveImageWithArrow(t *testing.T) {
|
||||
imgBytes, err := os.ReadFile("ai/testdata/llk_1.png")
|
||||
require.NoError(t, err)
|
||||
imgBuf := bytes.NewBuffer(imgBytes)
|
||||
|
||||
from := image.Point{X: 500, Y: 500}
|
||||
to := image.Point{X: 1000, Y: 1000}
|
||||
outputPath := "ai/testdata/output.png"
|
||||
|
||||
err = SaveImageWithArrowMarker(imgBuf, from, to, outputPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer os.Remove(outputPath)
|
||||
}
|
||||
|
||||
func TestMarkOperation(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
opts := []option.ActionOption{option.WithMarkOperationEnabled(true)}
|
||||
|
||||
// tap point
|
||||
err := driver.TapXY(0.5, 0.5, opts...)
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = driver.TapAbsXY(500, 800, opts...)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// swipe
|
||||
err = driver.Swipe(0.2, 0.5, 0.8, 0.5, opts...)
|
||||
assert.Nil(t, err)
|
||||
err = driver.Swipe(0.3, 0.7, 0.3, 0.3, opts...)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
83
uixt/driver_handler.go
Normal file
83
uixt/driver_handler.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package uixt
|
||||
|
||||
import (
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func handlerTapAbsXY(driver IDriver, rawX, rawY float64, opts ...option.ActionOption) (
|
||||
x, y float64, err error) {
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(rawX, rawY)
|
||||
|
||||
// mark UI operation
|
||||
if actionOptions.MarkOperationEnabled {
|
||||
if markErr := MarkUIOperation(driver, ACTION_TapAbsXY, []float64{x, y}); markErr != nil {
|
||||
log.Warn().Err(markErr).Msg("Failed to mark tap operation")
|
||||
}
|
||||
}
|
||||
|
||||
return x, y, nil
|
||||
}
|
||||
|
||||
func handlerDoubleTap(driver IDriver, rawX, rawY float64, opts ...option.ActionOption) (
|
||||
x, y float64, err error) {
|
||||
|
||||
x, y, err = convertToAbsolutePoint(driver, rawX, rawY)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
// mark UI operation
|
||||
if actionOptions.MarkOperationEnabled {
|
||||
if markErr := MarkUIOperation(driver, ACTION_DoubleTapXY, []float64{x, y}); markErr != nil {
|
||||
log.Warn().Err(markErr).Msg("Failed to mark double tap operation")
|
||||
}
|
||||
}
|
||||
|
||||
return x, y, nil
|
||||
}
|
||||
|
||||
func handlerDrag(driver IDriver, rawFomX, rawFromY, rawToX, rawToY float64, opts ...option.ActionOption) (
|
||||
fromX, fromY, toX, toY float64, err error) {
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(driver, rawFomX, rawFromY, rawToX, rawToY)
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
// mark UI operation
|
||||
if actionOptions.MarkOperationEnabled {
|
||||
if markErr := MarkUIOperation(driver, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil {
|
||||
log.Warn().Err(markErr).Msg("Failed to mark drag operation")
|
||||
}
|
||||
}
|
||||
|
||||
return fromX, fromY, toX, toY, nil
|
||||
}
|
||||
|
||||
func handlerSwipe(driver IDriver, rawFomX, rawFromY, rawToX, rawToY float64, opts ...option.ActionOption) (
|
||||
fromX, fromY, toX, toY float64, err error) {
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(driver, rawFomX, rawFromY, rawToX, rawToY)
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
// mark UI operation
|
||||
if actionOptions.MarkOperationEnabled {
|
||||
if markErr := MarkUIOperation(driver, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil {
|
||||
log.Warn().Err(markErr).Msg("Failed to mark swipe operation")
|
||||
}
|
||||
}
|
||||
|
||||
return fromX, fromY, toX, toY, nil
|
||||
}
|
||||
@@ -47,9 +47,9 @@ func NewDriverSession() *DriverSession {
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
requests: make([]*DriverRequests, 0),
|
||||
maxRetry: 5,
|
||||
}
|
||||
session.Reset()
|
||||
return session
|
||||
}
|
||||
|
||||
@@ -66,10 +66,14 @@ type DriverSession struct {
|
||||
|
||||
// cache driver request and response
|
||||
requests []*DriverRequests
|
||||
|
||||
// cache screenshot results
|
||||
screenResults []*ScreenResult
|
||||
}
|
||||
|
||||
func (s *DriverSession) Reset() {
|
||||
s.requests = make([]*DriverRequests, 0)
|
||||
s.screenResults = make([]*ScreenResult, 0)
|
||||
}
|
||||
|
||||
func (s *DriverSession) SetBaseURL(baseUrl string) {
|
||||
|
||||
@@ -117,11 +117,10 @@ func (dExt *XTDriver) GetData(withReset bool) map[string]interface{} {
|
||||
session := dExt.GetSession()
|
||||
data := map[string]interface{}{
|
||||
"requests": session.History(),
|
||||
"screen_results": dExt.screenResults,
|
||||
"screen_results": session.screenResults,
|
||||
}
|
||||
if withReset {
|
||||
session.Reset()
|
||||
dExt.screenResults = make([]*ScreenResult, 0)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
@@ -154,9 +154,14 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
|
||||
|
||||
func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY")
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
var err error
|
||||
x, y, err = handlerTapAbsXY(hd, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
if actionOptions.Identifier != "" {
|
||||
startTime := int(time.Now().UnixMilli())
|
||||
hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100})
|
||||
@@ -182,13 +187,11 @@ func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe")
|
||||
var err error
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(hd, fromX, fromY, toX, toY)
|
||||
fromX, fromY, toX, toY, err = handlerSwipe(hd, fromX, fromY, toX, toY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
duration := 200
|
||||
if actionOptions.PressDuration > 0 {
|
||||
duration = int(actionOptions.PressDuration * 1000)
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -16,7 +16,9 @@ func setupHDCDriverExt(t *testing.T) *XTDriver {
|
||||
require.Nil(t, err)
|
||||
hdcDriver, err := NewHDCDriver(device)
|
||||
require.Nil(t, err)
|
||||
return NewXTDriver(hdcDriver, ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(hdcDriver, option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
func TestWindowSize(t *testing.T) {
|
||||
|
||||
@@ -593,32 +593,40 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error {
|
||||
func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY")
|
||||
// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
|
||||
x = wd.toScale(x)
|
||||
y = wd.toScale(y)
|
||||
|
||||
var err error
|
||||
x, y, err = handlerTapAbsXY(wd, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"x": wd.toScale(x),
|
||||
"y": wd.toScale(y),
|
||||
"x": x,
|
||||
"y": y,
|
||||
}
|
||||
option.MergeOptions(data, opts...)
|
||||
|
||||
urlStr := fmt.Sprintf("/session/%s/wda/tap/0", wd.Session.ID)
|
||||
_, err := wd.Session.POST(data, urlStr)
|
||||
_, err = wd.Session.POST(data, urlStr)
|
||||
return err
|
||||
}
|
||||
|
||||
func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
|
||||
log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap")
|
||||
// [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)]
|
||||
|
||||
x = wd.toScale(x)
|
||||
y = wd.toScale(y)
|
||||
|
||||
var err error
|
||||
x, y, err = convertToAbsolutePoint(wd, x, y)
|
||||
x, y, err = handlerDoubleTap(wd, x, y, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
x, y = actionOptions.ApplyTapOffset(x, y)
|
||||
x = wd.toScale(x)
|
||||
y = wd.toScale(y)
|
||||
data := map[string]interface{}{
|
||||
"x": x,
|
||||
"y": y,
|
||||
@@ -643,17 +651,17 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO
|
||||
log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
|
||||
Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag")
|
||||
// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
|
||||
var err error
|
||||
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(wd, fromX, fromY, toX, toY)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fromX = wd.toScale(fromX)
|
||||
fromY = wd.toScale(fromY)
|
||||
toX = wd.toScale(toX)
|
||||
toY = wd.toScale(toY)
|
||||
actionOptions := option.NewActionOptions(opts...)
|
||||
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
|
||||
|
||||
var err error
|
||||
fromX, fromY, toX, toY, err = handlerDrag(wd, fromX, fromY, toX, toY, opts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"fromX": math.Round(fromX*10) / 10,
|
||||
@@ -970,7 +978,7 @@ func (wd *WDADriver) StartCaptureLog(identifier ...string) error {
|
||||
|
||||
func (wd *WDADriver) PushImage(localPath string) error {
|
||||
log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage")
|
||||
localFile, err := os.OpenFile(localPath, os.O_RDONLY, 0o600)
|
||||
localFile, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
@@ -23,7 +22,9 @@ func setupWDADriverExt(t *testing.T) *XTDriver {
|
||||
require.Nil(t, err)
|
||||
driver, err := device.NewDriver()
|
||||
require.Nil(t, err)
|
||||
return NewXTDriver(driver, ai.WithCVService(ai.CVServiceTypeVEDEM))
|
||||
driverExt, err := NewXTDriver(driver, option.WithCVService(option.CVServiceTypeVEDEM))
|
||||
require.Nil(t, err)
|
||||
return driverExt
|
||||
}
|
||||
|
||||
func TestDevice_IOS_Install(t *testing.T) {
|
||||
|
||||
@@ -132,6 +132,7 @@ func (o *ActionOptions) Options() []ActionOption {
|
||||
|
||||
options = append(options, o.GetScreenShotOptions()...)
|
||||
options = append(options, o.GetScreenRecordOptions()...)
|
||||
options = append(options, o.GetMarkOperationOptions()...)
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
43
uixt/option/ai.go
Normal file
43
uixt/option/ai.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package option
|
||||
|
||||
func NewAIServiceOptions(opts ...AIServiceOption) *AIServiceOptions {
|
||||
services := &AIServiceOptions{}
|
||||
for _, option := range opts {
|
||||
option(services)
|
||||
}
|
||||
return services
|
||||
}
|
||||
|
||||
type AIServiceOptions struct {
|
||||
CVService CVServiceType
|
||||
LLMService LLMServiceType
|
||||
}
|
||||
|
||||
type AIServiceOption func(*AIServiceOptions)
|
||||
|
||||
type CVServiceType string
|
||||
|
||||
const (
|
||||
CVServiceTypeVEDEM CVServiceType = "vedem"
|
||||
CVServiceTypeOpenCV CVServiceType = "opencv"
|
||||
)
|
||||
|
||||
func WithCVService(service CVServiceType) AIServiceOption {
|
||||
return func(opts *AIServiceOptions) {
|
||||
opts.CVService = service
|
||||
}
|
||||
}
|
||||
|
||||
type LLMServiceType string
|
||||
|
||||
const (
|
||||
LLMServiceTypeUITARS LLMServiceType = "ui-tars"
|
||||
LLMServiceTypeGPT LLMServiceType = "gpt"
|
||||
LLMServiceTypeQwenVL LLMServiceType = "qwen-vl"
|
||||
)
|
||||
|
||||
func WithLLMService(modelType LLMServiceType) AIServiceOption {
|
||||
return func(opts *AIServiceOptions) {
|
||||
opts.LLMService = modelType
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ type ScreenOptions struct {
|
||||
ScreenShotOptions
|
||||
ScreenRecordOptions
|
||||
ScreenFilterOptions
|
||||
MarkOperationOptions
|
||||
}
|
||||
|
||||
type ScreenShotOptions struct {
|
||||
@@ -273,3 +274,29 @@ func WithIndex(index int) ActionOption {
|
||||
o.Index = index
|
||||
}
|
||||
}
|
||||
|
||||
// MarkOperationOptions contains options for marking UI operations
|
||||
type MarkOperationOptions struct {
|
||||
// mark UI operation, enable/disable UI operation marking
|
||||
MarkOperationEnabled bool `json:"mark_operation_enabled,omitempty" yaml:"mark_operation_enabled,omitempty"`
|
||||
}
|
||||
|
||||
func (o *MarkOperationOptions) GetMarkOperationOptions() []ActionOption {
|
||||
options := make([]ActionOption, 0)
|
||||
if o == nil {
|
||||
return options
|
||||
}
|
||||
|
||||
if o.MarkOperationEnabled {
|
||||
options = append(options, WithMarkOperationEnabled(true))
|
||||
}
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
// WithMarkOperationEnabled enables or disables UI operation marking
|
||||
func WithMarkOperationEnabled(enabled bool) ActionOption {
|
||||
return func(o *ActionOptions) {
|
||||
o.MarkOperationEnabled = enabled
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user