GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-06-06 16:29:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat-ai' into 'master'

Browse Source 
refactor

See merge request iesqa/httprunner!82
This commit is contained in:
李隆
2025-05-07 12:03:17 +00:00
parent f6e421fc34 032a130418
commit ca0282a6bf
47 changed files with 1089 additions and 933 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
config.go
View File

@@ -4,7 +4,6 @@ import (
"reflect" "reflect"
"github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
) )
@@ -43,8 +42,8 @@ type TConfig struct {
Path string `json:"path,omitempty" yaml:"path,omitempty"` // testcase file path Path string `json:"path,omitempty" yaml:"path,omitempty"` // testcase file path
PluginSetting *PluginConfig `json:"plugin,omitempty" yaml:"plugin,omitempty"` // plugin config PluginSetting *PluginConfig `json:"plugin,omitempty" yaml:"plugin,omitempty"` // plugin config
IgnorePopup bool `json:"ignore_popup,omitempty" yaml:"ignore_popup,omitempty"` IgnorePopup bool `json:"ignore_popup,omitempty" yaml:"ignore_popup,omitempty"`
LLMService ai.LLMServiceType `json:"llm_service,omitempty" yaml:"llm_service,omitempty"` LLMService option.LLMServiceType `json:"llm_service,omitempty" yaml:"llm_service,omitempty"`
CVService ai.CVServiceType `json:"cv_service,omitempty" yaml:"cv_service,omitempty"` CVService option.CVServiceType `json:"cv_service,omitempty" yaml:"cv_service,omitempty"`
} }
func (c *TConfig) Get() *TConfig { func (c *TConfig) Get() *TConfig {
@@ -112,13 +111,13 @@ func (c *TConfig) SetWeight(weight int) *TConfig {
} }
// SetLLMService sets LLM service for current testcase. // SetLLMService sets LLM service for current testcase.
func (c *TConfig) SetLLMService(llmService ai.LLMServiceType) *TConfig { func (c *TConfig) SetLLMService(llmService option.LLMServiceType) *TConfig {
c.LLMService = llmService c.LLMService = llmService
return c return c
} }
// SetCVService sets CV service for current testcase. // SetCVService sets CV service for current testcase.
func (c *TConfig) SetCVService(cvService ai.CVServiceType) *TConfig { func (c *TConfig) SetCVService(cvService option.CVServiceType) *TConfig {
c.CVService = cvService c.CVService = cvService
return c return c
} }

2
convert/from_curl.go
View File

@@ -119,7 +119,7 @@ func LoadCurlCase(path string) (*hrp.TestCaseDef, error) {
} }
func readFileLines(path string) ([]string, error) { func readFileLines(path string) ([]string, error) {
file, err := os.OpenFile(path, os.O_RDONLY, 0o600) file, err := os.Open(path)
if err != nil { if err != nil {
log.Error().Err(err).Str("path", path).Msg("open file failed") log.Error().Err(err).Str("path", path).Msg("open file failed")
return nil, err return nil, err

31
docs/uixt/ui_mark.md Normal file
View File

@@ -0,0 +1,31 @@
### UI操作标注
针对 UI 操作的位置进行标注，帮助用户直观地了解操作发生的位置。
#### 功能说明
- 点击操作（tap）：使用红色矩形框标注点击位置
- 滑动操作（swipe）：使用红色箭头标注滑动方向，从起始点指向结束点
#### 使用方法
只需在操作函数中添加 `WithMarkOperationEnabled(true)` 选项即可启用操作标注功能：
```go
// 启用操作标注功能
opts := []option.ActionOption{option.WithMarkOperationEnabled(true)}
// 执行点击操作，会自动用红色矩形标注点击位置
err := driver.TapXY(0.5, 0.5, opts...)
// 执行滑动操作，会自动用红色箭头标注滑动方向
err = driver.Swipe(0.2, 0.5, 0.8, 0.5, opts...)
// 可以同时使用其他选项
opts = append(opts, option.WithScreenShotFileName("custom_name"))
err = driver.TapXY(0.3, 0.7, opts...)
```
#### 标注结果
标注后的图片会保存在截图目录中，文件名格式为：`{timestamp}_{tap|swipe}_marked.png`

2
examples/uitest/bili/android/cli.go
View File

@@ -52,7 +52,7 @@ func launchAppDriver(pkgName string) (driverExt *uixt.XTDriver, err error) {
time.Sleep(15 * time.Second) time.Sleep(15 * time.Second)
driverExt = uixt.NewXTDriver(driver) driverExt, _ = uixt.NewXTDriver(driver)
// 处理弹窗 // 处理弹窗
err = driverExt.ClosePopupsHandler() err = driverExt.ClosePopupsHandler()

5
examples/uitest/bili/ios/cli.go
View File

@@ -7,7 +7,6 @@ import (
"time" "time"
"github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
) )
@@ -53,8 +52,8 @@ func launchAppDriver(pkgName string) (driverExt *uixt.XTDriver, err error) {
time.Sleep(15 * time.Second) time.Sleep(15 * time.Second)
driverExt = uixt.NewXTDriver(driver, driverExt, _ = uixt.NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
// 处理弹窗 // 处理弹窗
err = driverExt.ClosePopupsHandler() err = driverExt.ClosePopupsHandler()

5
examples/worldcup/cli.go
View File

@@ -8,7 +8,6 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
) )
@@ -39,8 +38,8 @@ var rootCmd = &cobra.Command{
return errors.New("android or ios app bundldID is required") return errors.New("android or ios app bundldID is required")
} }
driverExt := uixt.NewXTDriver(driver, driverExt, _ := uixt.NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM), option.WithCVService(option.CVServiceTypeVEDEM),
) )
wc := NewWorldCupLive(driverExt, matchName, bundleID, duration, interval) wc := NewWorldCupLive(driverExt, matchName, bundleID, duration, interval)

4
examples/worldcup/main.go
View File

@@ -48,7 +48,7 @@ func initIOSDriver(uuid string) *uixt.XTDriver {
log.Fatal().Err(err).Msg("failed to init ios device") log.Fatal().Err(err).Msg("failed to init ios device")
} }
driver, _ := device.NewDriver() driver, _ := device.NewDriver()
driverExt := uixt.NewXTDriver(driver) driverExt, _ := uixt.NewXTDriver(driver)
return driverExt return driverExt
} }
@@ -58,7 +58,7 @@ func initAndroidDriver(uuid string) *uixt.XTDriver {
log.Fatal().Err(err).Msg("failed to init android device") log.Fatal().Err(err).Msg("failed to init android device")
} }
driver, _ := device.NewDriver() driver, _ := device.NewDriver()
driverExt := uixt.NewXTDriver(driver) driverExt, _ := uixt.NewXTDriver(driver)
return driverExt return driverExt
} }

4
go.mod
View File

@@ -9,7 +9,6 @@ require (
github.com/andybalholm/brotli v1.0.4 github.com/andybalholm/brotli v1.0.4
github.com/bytedance/sonic v1.13.2 github.com/bytedance/sonic v1.13.2
github.com/cloudwego/eino v0.3.26 github.com/cloudwego/eino v0.3.26
github.com/cloudwego/eino-ext/components/model/ark v0.1.6
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd
@@ -42,6 +41,7 @@ require (
) )
require ( require (
github.com/bytedance/mockey v1.2.14 // indirect
github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cloudwego/base64x v0.1.5 // indirect github.com/cloudwego/base64x v0.1.5 // indirect
@@ -100,8 +100,6 @@ require (
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 // indirect github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect github.com/ugorji/go/codec v1.2.12 // indirect
github.com/volcengine/volc-sdk-golang v1.0.23 // indirect
github.com/volcengine/volcengine-go-sdk v1.0.185 // indirect
github.com/yargevad/filepathx v1.0.0 // indirect github.com/yargevad/filepathx v1.0.0 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect

67
go.sum
View File

@@ -1,11 +1,8 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
@@ -19,18 +16,14 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/eino v0.3.26 h1:FdJJTCdNrc9xPcYkLZiEyr7AA+WgyCKCbY+VNDXIaCE= github.com/cloudwego/eino v0.3.26 h1:FdJJTCdNrc9xPcYkLZiEyr7AA+WgyCKCbY+VNDXIaCE=
github.com/cloudwego/eino v0.3.26/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY= github.com/cloudwego/eino v0.3.26/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY=
github.com/cloudwego/eino-ext/components/model/ark v0.1.6 h1:k17Z9VIRBL0/t7Ty1drGgY9tVOraM5xuO6gy7Qx7xus=
github.com/cloudwego/eino-ext/components/model/ark v0.1.6/go.mod h1:13kQjYGLMgla6xTbejlpqhuk3i5BPlNv5S+1pmknlOo=
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd h1:XEI7RezzV/cnOnhc1YeBJi6a0UoM41JTph4AZZR7+D8= github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd h1:XEI7RezzV/cnOnhc1YeBJi6a0UoM41JTph4AZZR7+D8=
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd/go.mod h1:8gMakAGQUR+IaWTSD0cpcD4U5FYq5puZ73/QjXqs1oU= github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd/go.mod h1:8gMakAGQUR+IaWTSD0cpcD4U5FYq5puZ73/QjXqs1oU=
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa h1:Jrmw8Q9g1WcE+x5t3o0TsEBM8RoMRURJI6P52I/ld74= github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa h1:Jrmw8Q9g1WcE+x5t3o0TsEBM8RoMRURJI6P52I/ld74=
@@ -53,8 +46,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0= github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg0Kys0ZbaNmDDzZ2R/C7DTi+bbsJ0=
github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo= github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
@@ -106,27 +97,12 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -135,7 +111,6 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18=
@@ -186,7 +161,6 @@ github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQe
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -259,7 +233,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs= github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 h1:I4N3ZRnkZPbDN935Tg8QDf8fRpHp3bZ0U0/L42jBgNE= github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 h1:I4N3ZRnkZPbDN935Tg8QDf8fRpHp3bZ0U0/L42jBgNE=
@@ -299,7 +272,6 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -316,10 +288,6 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8=
github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU=
github.com/volcengine/volcengine-go-sdk v1.0.185 h1:MIH+YgdWZhO1fNg/vxLohl8ad7hlklaf46wpaTS1TN0=
github.com/volcengine/volcengine-go-sdk v1.0.185/go.mod h1:gfEDc1s7SYaGoY+WH2dRrS3qiuDJMkwqyfXWCa7+7oA=
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
@@ -338,20 +306,12 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -359,13 +319,10 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -396,10 +353,6 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
@@ -407,34 +360,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -453,8 +388,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5 h1:DOUDfNS+CFMM46k18FRF5k/0yz5NhZYMiUQxf4xglIU= gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5 h1:DOUDfNS+CFMM46k18FRF5k/0yz5NhZYMiUQxf4xglIU=
gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0= gvisor.dev/gvisor v0.0.0-20240405191320-0878b34101b5/go.mod h1:NQHVAzMwvZ+Qe3ElSiHmq9RUm1MdNHpUZ52fiEqvn+0=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=

43
internal/builtin/utils.go
View File

@@ -6,9 +6,13 @@ import (
"context" "context"
"crypto/hmac" "crypto/hmac"
"crypto/sha256" "crypto/sha256"
"encoding/base64"
"encoding/csv" "encoding/csv"
builtinJSON "encoding/json" builtinJSON "encoding/json"
"fmt" "fmt"
"image"
"image/jpeg"
"image/png"
"math" "math"
"math/rand" "math/rand"
"net" "net"
@@ -26,6 +30,7 @@ import (
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/json" "github.com/httprunner/httprunner/v5/internal/json"
"github.com/httprunner/httprunner/v5/uixt/types"
) )
func Dump2JSON(data interface{}, path string) error { func Dump2JSON(data interface{}, path string) error {
@@ -484,3 +489,41 @@ func RunCommandWithCallback(cmdName string, args []string, callback LineCallback
return err return err
} }
} }
// LoadImage loads image file and returns base64 encoded string and image size
func LoadImage(imagePath string) (base64Str string, size types.Size, err error) {
// Read the image file
imageFile, err := os.Open(imagePath)
if err != nil {
return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
}
defer imageFile.Close()
// Decode the image to get its resolution
imageData, format, err := image.Decode(imageFile)
if err != nil {
return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
}
// Get the resolution of the image
width := imageData.Bounds().Dx()
height := imageData.Bounds().Dy()
size = types.Size{Width: width, Height: height}
// Convert image to base64
buf := new(bytes.Buffer)
if format == "jpeg" || format == "jpg" {
if err := jpeg.Encode(buf, imageData, nil); err != nil {
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
}
} else {
// default use png
if err := png.Encode(buf, imageData); err != nil {
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
}
}
base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
base64.StdEncoding.EncodeToString(buf.Bytes()))
return base64Str, size, nil
}

2
internal/version/VERSION
View File

@@ -1 +1 @@
v5.0.0-beta-2504292008 v5.0.0-beta-2505071715

4
pkg/gadb/device.go
View File

@@ -536,7 +536,7 @@ func (d *Device) List(remotePath string) (devFileInfos []DeviceFileInfo, err err
} }
func (d *Device) PushFile(localPath, remotePath string, modification ...time.Time) (err error) { func (d *Device) PushFile(localPath, remotePath string, modification ...time.Time) (err error) {
localFile, err := os.OpenFile(localPath, os.O_RDONLY, 0o600) localFile, err := os.Open(localPath)
if err != nil { if err != nil {
return err return err
} }
@@ -645,7 +645,7 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt
} }
func (d *Device) InstallAPK(apkPath string, args ...string) (string, error) { func (d *Device) InstallAPK(apkPath string, args ...string) (string, error) {
apkFile, err := os.OpenFile(apkPath, os.O_RDONLY, 0o600) apkFile, err := os.Open(apkPath)
if err != nil { if err != nil {
return "", errors.Wrap(err, fmt.Sprintf("open apk file %s failed", apkPath)) return "", errors.Wrap(err, fmt.Sprintf("open apk file %s failed", apkPath))
} }

25
runner.go
View File

@@ -28,7 +28,7 @@ import (
"github.com/httprunner/httprunner/v5/internal/sdk" "github.com/httprunner/httprunner/v5/internal/sdk"
"github.com/httprunner/httprunner/v5/internal/version" "github.com/httprunner/httprunner/v5/internal/version"
"github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt"
"github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option"
) )
// Run starts to run testcase with default configs. // Run starts to run testcase with default configs.
@@ -419,15 +419,15 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
r.parametersIterator = parametersIterator r.parametersIterator = parametersIterator
// ai options // ai options
aiOpts := []ai.AIServiceOption{} aiOpts := []option.AIServiceOption{}
if parsedConfig.LLMService != "" { if parsedConfig.LLMService != "" {
aiOpts = append(aiOpts, ai.WithLLMService(parsedConfig.LLMService)) aiOpts = append(aiOpts, option.WithLLMService(option.LLMServiceType(parsedConfig.LLMService)))
} }
if parsedConfig.CVService == "" { if parsedConfig.CVService == "" {
// default to vedem // default to vedem
parsedConfig.CVService = ai.CVServiceTypeVEDEM parsedConfig.CVService = option.CVServiceTypeVEDEM
} }
aiOpts = append(aiOpts, ai.WithCVService(parsedConfig.CVService)) aiOpts = append(aiOpts, option.WithCVService(parsedConfig.CVService))
// parse android devices config // parse android devices config
for _, androidDeviceOptions := range parsedConfig.Android { for _, androidDeviceOptions := range parsedConfig.Android {
@@ -446,7 +446,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
return nil, errors.Wrap(err, "init android driver failed") return nil, errors.Wrap(err, "init android driver failed")
} }
driverExt := uixt.NewXTDriver(driver, aiOpts...) driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
if err != nil {
return nil, errors.Wrap(err, "init android XTDriver failed")
}
r.uixtDrivers[androidDeviceOptions.SerialNumber] = driverExt r.uixtDrivers[androidDeviceOptions.SerialNumber] = driverExt
} }
// parse iOS devices config // parse iOS devices config
@@ -466,7 +469,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
return nil, errors.Wrap(err, "init ios driver failed") return nil, errors.Wrap(err, "init ios driver failed")
} }
driverExt := uixt.NewXTDriver(driver, aiOpts...) driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
if err != nil {
return nil, errors.Wrap(err, "init ios XTDriver failed")
}
r.uixtDrivers[iosDeviceOptions.UDID] = driverExt r.uixtDrivers[iosDeviceOptions.UDID] = driverExt
} }
// parse harmony devices config // parse harmony devices config
@@ -486,7 +492,10 @@ func (r *CaseRunner) parseConfig() (parsedConfig *TConfig, err error) {
return nil, errors.Wrap(err, "init harmony driver failed") return nil, errors.Wrap(err, "init harmony driver failed")
} }
driverExt := uixt.NewXTDriver(driver, aiOpts...) driverExt, err := uixt.NewXTDriver(driver, aiOpts...)
if err != nil {
return nil, errors.Wrap(err, "init harmony XTDriver failed")
}
r.uixtDrivers[harmonyDeviceOptions.ConnectKey] = driverExt r.uixtDrivers[harmonyDeviceOptions.ConnectKey] = driverExt
} }

9
server/context.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
) )
@@ -33,8 +32,12 @@ func (r *Router) GetDriver(c *gin.Context) (driverExt *uixt.XTDriver, err error)
return return
} }
driverExt = uixt.NewXTDriver(driver, driverExt, err = uixt.NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
if err != nil {
RenderErrorInitDriver(c, err)
return
}
c.Set("driver", driverExt) c.Set("driver", driverExt)
return driverExt, nil return driverExt, nil
} }

2
summary.go
View File

@@ -95,7 +95,7 @@ func (s *Summary) GenHTMLReport() error {
} }
reportPath := filepath.Join(reportsDir, "report.html") reportPath := filepath.Join(reportsDir, "report.html")
file, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE, 0o600) file, err := os.Open(reportPath)
if err != nil { if err != nil {
log.Error().Err(err).Msg("open file failed") log.Error().Err(err).Msg("open file failed")
return err return err

169
uixt/ai/ai.go
View File

@@ -3,55 +3,14 @@ package ai
import ( import (
"context" "context"
"os" "os"
"time"
"github.com/rs/zerolog/log" "github.com/cloudwego/eino-ext/components/model/openai"
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
) "github.com/httprunner/httprunner/v5/internal/config"
"github.com/httprunner/httprunner/v5/uixt/option"
func NewAIService(opts ...AIServiceOption) *AIServices { "github.com/pkg/errors"
services := &AIServices{} "github.com/rs/zerolog/log"
for _, option := range opts {
option(services)
}
return services
}
type AIServices struct {
ICVService
ILLMService
}
type AIServiceOption func(*AIServices)
type CVServiceType string
const (
CVServiceTypeVEDEM CVServiceType = "vedem"
CVServiceTypeOpenCV CVServiceType = "opencv"
)
func WithCVService(service CVServiceType) AIServiceOption {
return func(opts *AIServices) {
if service == CVServiceTypeVEDEM {
var err error
opts.ICVService, err = NewVEDEMImageService()
if err != nil {
log.Error().Err(err).Msg("init vedem image service failed")
os.Exit(code.GetErrorCode(err))
}
}
}
}
type LLMServiceType string
const (
LLMServiceTypeUITARS LLMServiceType = "ui-tars"
LLMServiceTypeGPT4o LLMServiceType = "gpt-4o"
LLMServiceTypeGPT4Vision LLMServiceType = "gpt-4-vision"
LLMServiceTypeQwenVL LLMServiceType = "qwen-vl"
LLMServiceTypeDeepSeekV3 LLMServiceType = "deepseek-v3"
) )
// ILLMService 定义了 LLM 服务接口，包括规划和断言功能 // ILLMService 定义了 LLM 服务接口，包括规划和断言功能
@@ -60,38 +19,29 @@ type ILLMService interface {
Assert(opts *AssertOptions) (*AssertionResponse, error) Assert(opts *AssertOptions) (*AssertionResponse, error)
} }
func WithLLMService(modelType LLMServiceType) AIServiceOption { func NewLLMService(modelType option.LLMServiceType) (ILLMService, error) {
return func(opts *AIServices) { modelConfig, err := GetModelConfig(modelType)
// init planner if err != nil {
var planner IPlanner return nil, err
var err error
switch modelType {
case LLMServiceTypeGPT4o:
// TODO: implement gpt-4o planner and asserter
planner, err = NewPlanner(context.Background())
case LLMServiceTypeUITARS:
planner, err = NewUITarsPlanner(context.Background())
}
if err != nil {
log.Error().Err(err).Msgf("init %s planner failed", modelType)
os.Exit(code.GetErrorCode(err))
}
// init asserter
asserter, err := NewAsserter(context.Background(), modelType)
if err != nil {
log.Error().Err(err).Msgf("init %s asserter failed", modelType)
os.Exit(code.GetErrorCode(err))
}
opts.ILLMService = &combinedLLMService{
planner: planner,
asserter: asserter,
}
} }
planner, err := NewPlanner(context.Background(), modelConfig)
if err != nil {
return nil, err
}
asserter, err := NewAsserter(context.Background(), modelConfig)
if err != nil {
return nil, err
}
return &combinedLLMService{
planner: planner,
asserter: asserter,
}, nil
} }
// combinedLLMService 实现了 ILLMService 接口，组合了规划和断言功能 // combinedLLMService 实现了 ILLMService 接口，组合了规划和断言功能
// ⭐️支持采用不同的模型服务进行规划和断言
type combinedLLMService struct { type combinedLLMService struct {
planner IPlanner // 提供规划功能 planner IPlanner // 提供规划功能
asserter IAsserter // 提供断言功能 asserter IAsserter // 提供断言功能
@@ -106,3 +56,72 @@ func (c *combinedLLMService) Call(opts *PlanningOptions) (*PlanningResult, error
func (c *combinedLLMService) Assert(opts *AssertOptions) (*AssertionResponse, error) { func (c *combinedLLMService) Assert(opts *AssertOptions) (*AssertionResponse, error) {
return c.asserter.Assert(opts) return c.asserter.Assert(opts)
} }
// LLM model config env variables
const (
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
EnvOpenAIAPIKey = "OPENAI_API_KEY"
EnvModelName = "LLM_MODEL_NAME"
)
const (
defaultTimeout = 30 * time.Second
)
type ModelConfig struct {
*openai.ChatModelConfig
ModelType option.LLMServiceType
}
// GetModelConfig get OpenAI config
func GetModelConfig(modelType option.LLMServiceType) (*ModelConfig, error) {
if err := config.LoadEnv(); err != nil {
return nil, errors.Wrap(code.LoadEnvError, err.Error())
}
openaiBaseURL := os.Getenv(EnvOpenAIBaseURL)
if openaiBaseURL == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvOpenAIBaseURL)
}
openaiAPIKey := os.Getenv(EnvOpenAIAPIKey)
if openaiAPIKey == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvOpenAIAPIKey)
}
modelName := os.Getenv(EnvModelName)
if modelName == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvModelName)
}
temperature := float32(0.01)
modelConfig := &openai.ChatModelConfig{
BaseURL: openaiBaseURL,
APIKey: openaiAPIKey,
Model: modelName,
Timeout: defaultTimeout,
Temperature: &temperature,
}
// log config info
log.Info().Str("model", modelConfig.Model).
Str("baseURL", modelConfig.BaseURL).
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
Str("timeout", defaultTimeout.String()).
Msg("get model config")
return &ModelConfig{
ChatModelConfig: modelConfig,
ModelType: modelType,
}, nil
}
// maskAPIKey masks the API key
func maskAPIKey(key string) string {
if len(key) <= 8 {
return "******"
}
return key[:4] + "******" + key[len(key)-4:]
}

62
uixt/ai/ai_ark.go
View File

@@ -1,62 +0,0 @@
package ai
import (
"os"
"github.com/cloudwego/eino-ext/components/model/ark"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/config"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
const (
EnvArkBaseURL = "ARK_BASE_URL"
EnvArkAPIKey = "ARK_API_KEY"
EnvArkModelID = "ARK_MODEL_ID"
)
func GetArkModelConfig() (*ark.ChatModelConfig, error) {
if err := config.LoadEnv(); err != nil {
return nil, errors.Wrap(code.LoadEnvError, err.Error())
}
arkBaseURL := os.Getenv(EnvArkBaseURL)
arkAPIKey := os.Getenv(EnvArkAPIKey)
if arkAPIKey == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvArkAPIKey)
}
modelName := os.Getenv(EnvArkModelID)
if modelName == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvArkModelID)
}
timeout := defaultTimeout
// https://www.volcengine.com/docs/82379/1494384?redirect=1
temperature := float32(0.01) // [0, 2] 采样温度。控制了生成文本时对每个候选词的概率分布进行平滑的程度。
// topP := float32(0.7) // [0, 1] 核采样概率阈值。模型会考虑概率质量在 top_p 内的 token 结果。
// maxTokens := int(4096) // 模型可以生成的最大 token 数量。输入 token 和输出 token 的总长度还受模型的上下文长度限制。
// frequencyPenalty := float32(0) // [-2, 2] 频率惩罚系数。如果值为正，会根据新 token 在文本中的出现频率对其进行惩罚，从而降低模型逐字重复的可能性。
modelConfig := &ark.ChatModelConfig{
BaseURL: arkBaseURL,
APIKey: arkAPIKey,
Model: modelName,
Timeout: &timeout,
Temperature: &temperature,
// TopP: &topP,
// MaxTokens: &maxTokens,
// FrequencyPenalty: &frequencyPenalty,
}
// log config info
log.Info().Str("model", modelConfig.Model).
Str("baseURL", modelConfig.BaseURL).
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
Str("timeout", defaultTimeout.String()).
Msg("get model config")
return modelConfig, nil
}

79
uixt/ai/ai_openai.go
View File

@@ -1,79 +0,0 @@
package ai
import (
"os"
"github.com/cloudwego/eino-ext/components/model/openai"
openai2 "github.com/cloudwego/eino-ext/libs/acl/openai"
"github.com/getkin/kin-openapi/openapi3gen"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/config"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
const (
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
EnvOpenAIAPIKey = "OPENAI_API_KEY"
EnvModelName = "LLM_MODEL_NAME"
)
// GetOpenAIModelConfig get OpenAI config
func GetOpenAIModelConfig() (*openai.ChatModelConfig, error) {
if err := config.LoadEnv(); err != nil {
return nil, errors.Wrap(code.LoadEnvError, err.Error())
}
openaiBaseURL := os.Getenv(EnvOpenAIBaseURL)
if openaiBaseURL == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvOpenAIBaseURL)
}
openaiAPIKey := os.Getenv(EnvOpenAIAPIKey)
if openaiAPIKey == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvOpenAIAPIKey)
}
modelName := os.Getenv(EnvModelName)
if modelName == "" {
return nil, errors.Wrapf(code.LLMEnvMissedError,
"env %s missed", EnvModelName)
}
type OutputFormat struct {
Thought string `json:"thought"`
Action string `json:"action"`
Error string `json:"error,omitempty"`
}
outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
if err != nil {
return nil, err
}
modelConfig := &openai.ChatModelConfig{
BaseURL: openaiBaseURL,
APIKey: openaiAPIKey,
Model: modelName,
Timeout: defaultTimeout,
// set structured response format
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
ResponseFormat: &openai2.ChatCompletionResponseFormat{
Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
Name: "thought_and_action",
Description: "data that describes planning thought and action",
Schema: outputFormatSchema.Value,
Strict: false,
},
},
}
// log config info
log.Info().Str("model", modelConfig.Model).
Str("baseURL", modelConfig.BaseURL).
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
Str("timeout", defaultTimeout.String()).
Msg("get model config")
return modelConfig, nil
}

13
uixt/ai/ai_test.go
View File

@@ -1,13 +0,0 @@
package ai
import (
"testing"
)
func TestOption(t *testing.T) {
options := NewAIService(
WithCVService(CVServiceTypeOpenCV),
WithLLMService(LLMServiceTypeUITARS),
)
t.Log(options)
}

67
uixt/ai/asserter.go
View File

@@ -7,12 +7,14 @@ import (
"strings" "strings"
"time" "time"
"github.com/cloudwego/eino-ext/components/model/ark"
"github.com/cloudwego/eino-ext/components/model/openai" "github.com/cloudwego/eino-ext/components/model/openai"
openai2 "github.com/cloudwego/eino-ext/libs/acl/openai"
"github.com/cloudwego/eino/components/model" "github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/schema" "github.com/cloudwego/eino/schema"
"github.com/getkin/kin-openapi/openapi3gen"
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/json" "github.com/httprunner/httprunner/v5/internal/json"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@@ -39,54 +41,57 @@ type AssertionResponse struct {
// Asserter handles assertion using different AI models // Asserter handles assertion using different AI models
type Asserter struct { type Asserter struct {
ctx context.Context ctx context.Context
modelConfig *ModelConfig
model model.ToolCallingChatModel model model.ToolCallingChatModel
systemPrompt string systemPrompt string
history ConversationHistory history ConversationHistory
modelType LLMServiceType
} }
// NewAsserter creates a new Asserter instance // NewAsserter creates a new Asserter instance
func NewAsserter(ctx context.Context, modelType LLMServiceType) (*Asserter, error) { func NewAsserter(ctx context.Context, modelConfig *ModelConfig) (*Asserter, error) {
asserter := &Asserter{ asserter := &Asserter{
ctx: ctx, ctx: ctx,
modelType: modelType, modelConfig: modelConfig,
systemPrompt: getAssertionSystemPrompt(modelType), systemPrompt: defaultAssertionPrompt,
} }
switch modelType { if modelConfig.ModelType == option.LLMServiceTypeUITARS {
case LLMServiceTypeUITARS: asserter.systemPrompt += "\n\n" + uiTarsAssertionResponseFormat
config, err := GetArkModelConfig() } else if modelConfig.ModelType == option.LLMServiceTypeGPT {
if err != nil { // define output format
return nil, err type OutputFormat struct {
Thought string `json:"thought"`
Pass bool `json:"pass"`
Error string `json:"error,omitempty"`
} }
asserter.model, err = ark.NewChatModel(ctx, config) outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
if err != nil { if err != nil {
return nil, err return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
} }
case LLMServiceTypeGPT4Vision, LLMServiceTypeGPT4o: // set structured response format
config, err := GetOpenAIModelConfig() // https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
if err != nil { modelConfig.ChatModelConfig.ResponseFormat = &openai2.ChatCompletionResponseFormat{
return nil, err Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
Name: "assertion_result",
Description: "data that describes assertion result",
Schema: outputFormatSchema.Value,
Strict: false,
},
} }
asserter.model, err = openai.NewChatModel(ctx, config) } else {
if err != nil { asserter.systemPrompt += "\n\n" + defaultAssertionResponseJsonFormat
return nil, err }
}
default: var err error
return nil, errors.New("not supported model type for asserter") asserter.model, err = openai.NewChatModel(ctx, modelConfig.ChatModelConfig)
if err != nil {
return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
} }
return asserter, nil return asserter, nil
} }
// getAssertionSystemPrompt returns the appropriate system prompt for the given model type
func getAssertionSystemPrompt(modelType LLMServiceType) string {
if modelType == LLMServiceTypeUITARS {
return defaultAssertionPrompt + "\n\n" + uiTarsAssertionResponseFormat
}
return defaultAssertionPrompt + "\n\n" + defaultAssertionResponseJsonFormat
}
// Assert performs the assertion check on the screenshot // Assert performs the assertion check on the screenshot
func (a *Asserter) Assert(opts *AssertOptions) (*AssertionResponse, error) { func (a *Asserter) Assert(opts *AssertOptions) (*AssertionResponse, error) {
// Validate input parameters // Validate input parameters
@@ -133,7 +138,7 @@ Here is the assertion. Please tell whether it is truthy according to the screens
startTime := time.Now() startTime := time.Now()
resp, err := a.model.Generate(a.ctx, a.history) resp, err := a.model.Generate(a.ctx, a.history)
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()). log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
Str("model", string(a.modelType)).Msg("call model service for assertion") Str("model", string(a.modelConfig.ModelType)).Msg("call model service for assertion")
if err != nil { if err != nil {
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error()) return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
} }

27
uixt/ai/asserter_test.go
View File

@@ -1,23 +1,27 @@
package ai package ai
import ( import (
"context"
"testing" "testing"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func createAIService(t *testing.T) *AIServices { func createAsserter(t *testing.T) *Asserter {
aiService := NewAIService(WithLLMService(LLMServiceTypeUITARS)) modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NotNil(t, aiService) require.NoError(t, err)
require.NotNil(t, aiService.ILLMService) asserter, err := NewAsserter(context.Background(), modelConfig)
return aiService require.NoError(t, err)
return asserter
} }
// 测试有效断言 // 测试有效断言
func TestValidAssertions(t *testing.T) { func TestValidAssertions(t *testing.T) {
aiService := createAIService(t) asserter := createAsserter(t)
testCases := []struct { testCases := []struct {
name string name string
@@ -33,7 +37,7 @@ func TestValidAssertions(t *testing.T) {
}, },
{ {
name: "深度思考功能未开启", name: "深度思考功能未开启",
assertion: "输入框下方的「深度思考」文字是灰色的", assertion: "输入框下方的「深度思考」文字不是蓝色的",
imagePath: "testdata/deepseek_think_off.png", imagePath: "testdata/deepseek_think_off.png",
expectPass: true, expectPass: true,
}, },
@@ -47,10 +51,10 @@ func TestValidAssertions(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
imageBase64, size, err := loadImage(tc.imagePath) imageBase64, size, err := builtin.LoadImage(tc.imagePath)
require.NoError(t, err) require.NoError(t, err)
result, err := aiService.ILLMService.Assert(&AssertOptions{ result, err := asserter.Assert(&AssertOptions{
Assertion: tc.assertion, Assertion: tc.assertion,
Screenshot: imageBase64, Screenshot: imageBase64,
Size: size, Size: size,
@@ -58,14 +62,13 @@ func TestValidAssertions(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.NotNil(t, result) assert.NotNil(t, result)
assert.Equal(t, tc.expectPass, result.Pass) assert.Equal(t, tc.expectPass, result.Pass)
assert.NotEmpty(t, result.Thought)
}) })
} }
} }
// 测试无效参数 // 测试无效参数
func TestInvalidParameters(t *testing.T) { func TestInvalidParameters(t *testing.T) {
aiService := createAIService(t) asserter := createAsserter(t)
testCases := []struct { testCases := []struct {
name string name string
assertion string assertion string
@@ -91,7 +94,7 @@ func TestInvalidParameters(t *testing.T) {
for _, tc := range testCases { for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := aiService.ILLMService.Assert(&AssertOptions{ _, err := asserter.Assert(&AssertOptions{
Assertion: tc.assertion, Assertion: tc.assertion,
Screenshot: tc.screenshot, Screenshot: tc.screenshot,
Size: tc.size, Size: tc.size,

7
uixt/ai/cv.go
View File

@@ -22,6 +22,13 @@ type ICVService interface {
ReadFromPath(imagePath string, opts ...option.ActionOption) (*CVResult, error) ReadFromPath(imagePath string, opts ...option.ActionOption) (*CVResult, error)
} }
func NewCVService(modelType option.CVServiceType) (ICVService, error) {
if modelType == option.CVServiceTypeVEDEM {
return NewVEDEMImageService()
}
return nil, errors.New("invalid cv service type")
}
type CVResult struct { type CVResult struct {
URL string `json:"url,omitempty"` // image uploaded url URL string `json:"url,omitempty"` // image uploaded url
OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts OCRResult OCRResults `json:"ocrResult,omitempty"` // OCR texts

10
uixt/ai/cv_vedem_test.go
View File

@@ -19,9 +19,8 @@ func TestGetImageFromBuffer(t *testing.T) {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
buf.Read(file) buf.Read(file)
service := NewAIService( service, err := NewVEDEMImageService()
WithCVService(CVServiceTypeVEDEM), require.Nil(t, err)
)
cvResult, err := service.ReadFromBuffer(buf) cvResult, err := service.ReadFromBuffer(buf)
assert.Nil(t, err) assert.Nil(t, err)
fmt.Println(fmt.Sprintf("cvResult: %v", cvResult)) fmt.Println(fmt.Sprintf("cvResult: %v", cvResult))
@@ -29,9 +28,8 @@ func TestGetImageFromBuffer(t *testing.T) {
func TestGetImageFromPath(t *testing.T) { func TestGetImageFromPath(t *testing.T) {
imagePath := "/Users/debugtalk/Downloads/s1.png" imagePath := "/Users/debugtalk/Downloads/s1.png"
service := NewAIService( service, err := NewVEDEMImageService()
WithCVService(CVServiceTypeVEDEM), require.Nil(t, err)
)
cvResult, err := service.ReadFromPath(imagePath) cvResult, err := service.ReadFromPath(imagePath)
assert.Nil(t, err) assert.Nil(t, err)
fmt.Println(fmt.Sprintf("cvResult: %v", cvResult)) fmt.Println(fmt.Sprintf("cvResult: %v", cvResult))

210
uixt/ai/planner.go
View File

@@ -1,21 +1,17 @@
package ai package ai
import ( import (
"bytes" "context"
"encoding/base64"
"fmt"
"image"
"image/color"
"image/draw"
"image/png"
"os"
"strings"
"time" "time"
"github.com/cloudwego/eino-ext/components/model/openai"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/schema" "github.com/cloudwego/eino/schema"
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log"
) )
type IPlanner interface { type IPlanner interface {
@@ -36,30 +32,110 @@ type PlanningResult struct {
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
} }
// ParsedAction represents a parsed action from the VLM response func NewPlanner(ctx context.Context, modelConfig *ModelConfig) (*Planner, error) {
type ParsedAction struct { planner := &Planner{
ActionType ActionType `json:"actionType"` ctx: ctx,
ActionInputs map[string]interface{} `json:"actionInputs"` modelConfig: modelConfig,
Thought string `json:"thought"` }
if modelConfig.ModelType == option.LLMServiceTypeUITARS {
planner.systemPrompt = uiTarsPlanningPrompt
} else {
planner.systemPrompt = defaultPlanningResponseJsonFormat
}
var err error
planner.model, err = openai.NewChatModel(ctx, modelConfig.ChatModelConfig)
if err != nil {
return nil, errors.Wrap(code.LLMPrepareRequestError, err.Error())
}
return planner, nil
} }
type ActionType string type Planner struct {
ctx context.Context
modelConfig *ModelConfig
model model.ToolCallingChatModel
systemPrompt string
history ConversationHistory
}
const ( // Call performs UI planning using Vision Language Model
ActionTypeClick ActionType = "click" func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
ActionTypeTap ActionType = "tap" // validate input parameters
ActionTypeDrag ActionType = "drag" if err := validatePlanningInput(opts); err != nil {
ActionTypeSwipe ActionType = "swipe" return nil, errors.Wrap(err, "validate planning parameters failed")
ActionTypeWait ActionType = "wait" }
ActionTypeFinished ActionType = "finished"
ActionTypeCallUser ActionType = "call_user"
ActionTypeType ActionType = "type"
ActionTypeScroll ActionType = "scroll"
)
const ( // prepare prompt
defaultTimeout = 30 * time.Second if len(p.history) == 0 {
) // add system message
p.history = ConversationHistory{
{
Role: schema.System,
Content: p.systemPrompt + opts.UserInstruction,
},
}
}
// append user image message
p.history.Append(opts.Message)
// call model service, generate response
logRequest(p.history)
startTime := time.Now()
resp, err := p.model.Generate(p.ctx, p.history)
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
Str("model", string(p.modelConfig.ModelType)).Msg("call model service")
if err != nil {
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
}
logResponse(resp)
// parse result
result, err := p.parseResult(resp, opts.Size)
if err != nil {
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
}
// append assistant message
p.history.Append(&schema.Message{
Role: schema.Assistant,
Content: result.ActionSummary,
})
return result, nil
}
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
var parseActions []ParsedAction
var err error
if p.modelConfig.ModelType == option.LLMServiceTypeUITARS {
// parse Thought/Action format from UI-TARS
parseActions, err = parseThoughtAction(msg.Content)
if err != nil {
return nil, err
}
} else {
// parse JSON format, from VLM like openai/gpt-4o
parseActions, err = parseJSON(msg.Content)
if err != nil {
return nil, err
}
}
// process response
result, err := processVLMResponse(parseActions, size)
if err != nil {
return nil, errors.Wrap(err, "process VLM response failed")
}
log.Info().
Interface("summary", result.ActionSummary).
Interface("actions", result.NextActions).
Msg("get VLM planning result")
return result, nil
}
func validatePlanningInput(opts *PlanningOptions) error { func validatePlanningInput(opts *PlanningOptions) error {
if opts.UserInstruction == "" { if opts.UserInstruction == "" {
@@ -83,79 +159,3 @@ func validatePlanningInput(opts *PlanningOptions) error {
return nil return nil
} }
// SavePositionImg saves an image with position markers
func SavePositionImg(params struct {
InputImgBase64 string
Rect struct {
X float64
Y float64
}
OutputPath string
}) error {
// 解码Base64图像
imgData := params.InputImgBase64
// 如果包含了数据URL前缀，去掉它
if strings.HasPrefix(imgData, "data:image/") {
parts := strings.Split(imgData, ",")
if len(parts) > 1 {
imgData = parts[1]
}
}
// 解码Base64
unbased, err := base64.StdEncoding.DecodeString(imgData)
if err != nil {
return fmt.Errorf("无法解码Base64图像: %w", err)
}
// 解码图像
reader := bytes.NewReader(unbased)
img, _, err := image.Decode(reader)
if err != nil {
return fmt.Errorf("无法解码图像数据: %w", err)
}
// 创建一个可以在其上绘制的图像
bounds := img.Bounds()
rgba := image.NewRGBA(bounds)
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
// 在点击/拖动位置绘制标记
markRadius := 10
x, y := int(params.Rect.X), int(params.Rect.Y)
// 绘制红色圆圈
for i := -markRadius; i <= markRadius; i++ {
for j := -markRadius; j <= markRadius; j++ {
if i*i+j*j <= markRadius*markRadius {
if x+i >= 0 && x+i < bounds.Max.X && y+j >= 0 && y+j < bounds.Max.Y {
rgba.Set(x+i, y+j, color.RGBA{255, 0, 0, 255})
}
}
}
}
// 保存图像
outFile, err := os.Create(params.OutputPath)
if err != nil {
return fmt.Errorf("无法创建输出文件: %w", err)
}
defer outFile.Close()
// 编码为PNG并保存
if err := png.Encode(outFile, rgba); err != nil {
return fmt.Errorf("无法编码和保存图像: %w", err)
}
return nil
}
// maskAPIKey masks the API key
func maskAPIKey(key string) string {
if len(key) <= 8 {
return "******"
}
return key[:4] + "******" + key[len(key)-4:]
}

172
uixt/ai/planner_gpt.go
View File

@@ -1,172 +0,0 @@
package ai
import (
"context"
"fmt"
_ "image/jpeg"
"strings"
"time"
"github.com/cloudwego/eino-ext/components/model/openai"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/schema"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/json"
"github.com/httprunner/httprunner/v5/uixt/types"
)
func NewPlanner(ctx context.Context) (*Planner, error) {
config, err := GetOpenAIModelConfig()
if err != nil {
return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
}
model, err := openai.NewChatModel(ctx, config)
if err != nil {
return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
}
return &Planner{
ctx: ctx,
model: model,
modelType: LLMServiceTypeGPT4o,
systemPrompt: uiTarsPlanningPrompt, // TODO: change prompt with function calling
}, nil
}
type Planner struct {
ctx context.Context
model model.ToolCallingChatModel
systemPrompt string
modelType LLMServiceType
history ConversationHistory
}
// Call performs UI planning using Vision Language Model
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
// validate input parameters
if err := validatePlanningInput(opts); err != nil {
return nil, errors.Wrap(err, "validate planning parameters failed")
}
// prepare prompt
if len(p.history) == 0 {
// add system message
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
p.history = ConversationHistory{
{
Role: schema.System,
Content: systemPrompt,
},
}
}
// append user image message
p.history.Append(opts.Message)
// call model service, generate response
logRequest(p.history)
startTime := time.Now()
resp, err := p.model.Generate(p.ctx, p.history)
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
Str("model", string(p.modelType)).Msg("call model service")
if err != nil {
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
}
logResponse(resp)
// parse result
result, err := p.parseResult(resp, opts.Size)
if err != nil {
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
}
// append assistant message
p.history.Append(&schema.Message{
Role: schema.Assistant,
Content: result.ActionSummary,
})
return result, nil
}
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
// parse JSON format, from VLM like openai/gpt-4o
parseActions, jsonErr := parseJSON(msg.Content)
if jsonErr != nil {
return nil, jsonErr
}
// process response
result, err := processVLMResponse(parseActions, size)
if err != nil {
return nil, errors.Wrap(err, "process VLM response failed")
}
log.Info().
Interface("summary", result.ActionSummary).
Interface("actions", result.NextActions).
Msg("get VLM planning result")
return result, nil
}
// parseJSON tries to parse the response as JSON format
func parseJSON(predictionText string) ([]ParsedAction, error) {
predictionText = strings.TrimSpace(predictionText)
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
predictionText = strings.TrimPrefix(predictionText, "```json")
predictionText = strings.TrimSuffix(predictionText, "```")
}
predictionText = strings.TrimSpace(predictionText)
var response PlanningResult
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
}
if response.Error != "" {
return nil, errors.New(response.Error)
}
if len(response.NextActions) == 0 {
return nil, errors.New("no actions returned from VLM")
}
// normalize actions
var normalizedActions []ParsedAction
for i := range response.NextActions {
// create a new variable, avoid implicit memory aliasing in for loop.
action := response.NextActions[i]
if err := normalizeAction(&action); err != nil {
return nil, errors.Wrap(err, "failed to normalize action")
}
normalizedActions = append(normalizedActions, action)
}
return normalizedActions, nil
}
// normalizeAction normalizes the coordinates in the action
func normalizeAction(action *ParsedAction) error {
switch action.ActionType {
case "click", "drag":
// handle click and drag action coordinates
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
normalized, err := normalizeCoordinates(startBox)
if err != nil {
return fmt.Errorf("failed to normalize startBox: %w", err)
}
action.ActionInputs["startBox"] = normalized
}
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
normalized, err := normalizeCoordinates(endBox)
if err != nil {
return fmt.Errorf("failed to normalize endBox: %w", err)
}
action.ActionInputs["endBox"] = normalized
}
}
return nil
}

173
uixt/ai/planner_ui_tars.go → uixt/ai/planner_parser.go
View File

@@ -1,116 +1,38 @@
package ai package ai
import ( import (
"context"
"fmt" "fmt"
"math" "math"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/cloudwego/eino-ext/components/model/ark"
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/schema"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/json" "github.com/httprunner/httprunner/v5/internal/json"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
func NewUITarsPlanner(ctx context.Context) (*UITarsPlanner, error) { // ParsedAction represents a parsed action from the VLM response
config, err := GetArkModelConfig() type ParsedAction struct {
if err != nil { ActionType ActionType `json:"actionType"`
return nil, err ActionInputs map[string]interface{} `json:"actionInputs"`
} Thought string `json:"thought"`
chatModel, err := ark.NewChatModel(ctx, config)
if err != nil {
return nil, err
}
return &UITarsPlanner{
ctx: ctx,
model: chatModel,
modelType: LLMServiceTypeUITARS,
systemPrompt: uiTarsPlanningPrompt,
}, nil
} }
type UITarsPlanner struct { type ActionType string
ctx context.Context
model model.ToolCallingChatModel
systemPrompt string
modelType LLMServiceType
history ConversationHistory
}
// Call performs UI planning using Vision Language Model const (
func (p *UITarsPlanner) Call(opts *PlanningOptions) (*PlanningResult, error) { ActionTypeClick ActionType = "click"
// validate input parameters ActionTypeTap ActionType = "tap"
if err := validatePlanningInput(opts); err != nil { ActionTypeDrag ActionType = "drag"
return nil, errors.Wrap(err, "validate planning parameters failed") ActionTypeSwipe ActionType = "swipe"
} ActionTypeWait ActionType = "wait"
ActionTypeFinished ActionType = "finished"
// prepare prompt ActionTypeCallUser ActionType = "call_user"
if len(p.history) == 0 { ActionTypeType ActionType = "type"
// add system message ActionTypeScroll ActionType = "scroll"
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction )
p.history = ConversationHistory{
{
Role: schema.System,
Content: systemPrompt,
},
}
}
// append user image message
p.history.Append(opts.Message)
// call model service, generate response
logRequest(p.history)
startTime := time.Now()
resp, err := p.model.Generate(p.ctx, p.history)
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
Str("model", string(p.modelType)).Msg("call model service")
if err != nil {
return nil, errors.Wrap(code.LLMRequestServiceError, err.Error())
}
logResponse(resp)
// parse result
result, err := p.parseResult(resp, opts.Size)
if err != nil {
return nil, errors.Wrap(code.LLMParsePlanningResponseError, err.Error())
}
// append assistant message
p.history.Append(&schema.Message{
Role: schema.Assistant,
Content: result.ActionSummary,
})
return result, nil
}
func (p *UITarsPlanner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
// parse Thought/Action format from UI-TARS
parseActions, thoughtErr := parseThoughtAction(msg.Content)
if thoughtErr != nil {
return nil, thoughtErr
}
// process response
result, err := processVLMResponse(parseActions, size)
if err != nil {
return nil, errors.Wrap(err, "process VLM response failed")
}
log.Info().
Interface("summary", result.ActionSummary).
Interface("actions", result.NextActions).
Msg("get VLM planning result")
return result, nil
}
// parseThoughtAction parses the Thought/Action format response // parseThoughtAction parses the Thought/Action format response
func parseThoughtAction(predictionText string) ([]ParsedAction, error) { func parseThoughtAction(predictionText string) ([]ParsedAction, error) {
@@ -396,3 +318,64 @@ func validateTypeContent(action *ParsedAction) {
log.Warn().Msg("type action missing content parameter, set to default") log.Warn().Msg("type action missing content parameter, set to default")
} }
} }
// parseJSON tries to parse the response as JSON format
func parseJSON(predictionText string) ([]ParsedAction, error) {
predictionText = strings.TrimSpace(predictionText)
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
predictionText = strings.TrimPrefix(predictionText, "```json")
predictionText = strings.TrimSuffix(predictionText, "```")
}
predictionText = strings.TrimSpace(predictionText)
var response PlanningResult
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
}
if response.Error != "" {
return nil, errors.New(response.Error)
}
if len(response.NextActions) == 0 {
return nil, errors.New("no actions returned from VLM")
}
// normalize actions
var normalizedActions []ParsedAction
for i := range response.NextActions {
// create a new variable, avoid implicit memory aliasing in for loop.
action := response.NextActions[i]
if err := normalizeAction(&action); err != nil {
return nil, errors.Wrap(err, "failed to normalize action")
}
normalizedActions = append(normalizedActions, action)
}
return normalizedActions, nil
}
// normalizeAction normalizes the coordinates in the action
func normalizeAction(action *ParsedAction) error {
switch action.ActionType {
case "click", "drag":
// handle click and drag action coordinates
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
normalized, err := normalizeCoordinates(startBox)
if err != nil {
return fmt.Errorf("failed to normalize startBox: %w", err)
}
action.ActionInputs["startBox"] = normalized
}
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
normalized, err := normalizeCoordinates(endBox)
if err != nil {
return fmt.Errorf("failed to normalize endBox: %w", err)
}
action.ActionInputs["endBox"] = normalized
}
}
return nil
}

2
uixt/ai/planner_prompts.go
View File

@@ -27,3 +27,5 @@ finished(content='xxx') # Use escape characters \\', \\", and \\n in content par
## User Instruction ## User Instruction
` `
const defaultPlanningResponseJsonFormat = ``

112
uixt/ai/planner_test.go
View File

@@ -1,25 +1,20 @@
package ai package ai
import ( import (
"bytes"
"context" "context"
"encoding/base64"
"fmt"
"image"
"image/jpeg"
"image/png"
"os"
"testing" "testing"
"github.com/cloudwego/eino/schema" "github.com/cloudwego/eino/schema"
"github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestVLMPlanning(t *testing.T) { func TestVLMPlanning(t *testing.T) {
imageBase64, size, err := loadImage("testdata/llk_1.png") imageBase64, size, err := builtin.LoadImage("testdata/llk_1.png")
require.NoError(t, err) require.NoError(t, err)
userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明： userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
@@ -35,7 +30,10 @@ func TestVLMPlanning(t *testing.T) {
userInstruction += "\n\n请基于以上游戏规则，给出下一步可点击的两个图标坐标" userInstruction += "\n\n请基于以上游戏规则，给出下一步可点击的两个图标坐标"
planner, err := NewUITarsPlanner(context.Background()) modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NoError(t, err)
planner, err := NewPlanner(context.Background(), modelConfig)
require.NoError(t, err) require.NoError(t, err)
opts := &PlanningOptions{ opts := &PlanningOptions{
@@ -100,12 +98,15 @@ func TestVLMPlanning(t *testing.T) {
} }
func TestXHSPlanning(t *testing.T) { func TestXHSPlanning(t *testing.T) {
imageBase64, size, err := loadImage("testdata/xhs-feed.jpeg") imageBase64, size, err := builtin.LoadImage("testdata/xhs-feed.jpeg")
require.NoError(t, err) require.NoError(t, err)
userInstruction := "点击第二个帖子的作者头像" userInstruction := "点击第二个帖子的作者头像"
planner, err := NewUITarsPlanner(context.Background()) modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NoError(t, err)
planner, err := NewPlanner(context.Background(), modelConfig)
require.NoError(t, err) require.NoError(t, err)
opts := &PlanningOptions{ opts := &PlanningOptions{
@@ -170,12 +171,15 @@ func TestXHSPlanning(t *testing.T) {
} }
func TestChatList(t *testing.T) { func TestChatList(t *testing.T) {
imageBase64, size, err := loadImage("testdata/chat_list.jpeg") imageBase64, size, err := builtin.LoadImage("testdata/chat_list.jpeg")
require.NoError(t, err) require.NoError(t, err)
userInstruction := "请结合图片的文字信息，请告诉我一共有多少个群聊，哪些群聊右下角有绿点" userInstruction := "请结合图片的文字信息，请告诉我一共有多少个群聊，哪些群聊右下角有绿点"
planner, err := NewUITarsPlanner(context.Background()) modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NoError(t, err)
planner, err := NewPlanner(context.Background(), modelConfig)
require.NoError(t, err) require.NoError(t, err)
opts := &PlanningOptions{ opts := &PlanningOptions{
@@ -206,7 +210,10 @@ func TestHandleSwitch(t *testing.T) {
userInstruction := "发送框下方的联网搜索开关是开启状态" // 点击开启联网搜索开关 userInstruction := "发送框下方的联网搜索开关是开启状态" // 点击开启联网搜索开关
// 检查发送框下方的联网搜索开关，蓝色为开启状态，灰色为关闭状态；若开关处于关闭状态，则点击进行开启 // 检查发送框下方的联网搜索开关，蓝色为开启状态，灰色为关闭状态；若开关处于关闭状态，则点击进行开启
planner, err := NewUITarsPlanner(context.Background()) modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NoError(t, err)
planner, err := NewPlanner(context.Background(), modelConfig)
require.NoError(t, err) require.NoError(t, err)
testCases := []struct { testCases := []struct {
@@ -219,7 +226,7 @@ func TestHandleSwitch(t *testing.T) {
} }
for _, tc := range testCases { for _, tc := range testCases {
imageBase64, size, err := loadImage(tc.imageFile) imageBase64, size, err := builtin.LoadImage(tc.imageFile)
require.NoError(t, err) require.NoError(t, err)
opts := &PlanningOptions{ opts := &PlanningOptions{
@@ -250,7 +257,7 @@ func TestHandleSwitch(t *testing.T) {
} }
func TestValidateInput(t *testing.T) { func TestValidateInput(t *testing.T) {
imageBase64, size, err := loadImage("testdata/popup_risk_warning.png") imageBase64, size, err := builtin.LoadImage("testdata/popup_risk_warning.png")
require.NoError(t, err) require.NoError(t, err)
tests := []struct { tests := []struct {
@@ -375,87 +382,18 @@ func TestProcessVLMResponse(t *testing.T) {
} }
} }
func TestSavePositionImg(t *testing.T) {
imageBase64, _, err := loadImage("testdata/popup_risk_warning.png")
require.NoError(t, err)
params := struct {
InputImgBase64 string
Rect struct {
X float64
Y float64
}
OutputPath string
}{
InputImgBase64: imageBase64,
Rect: struct {
X float64
Y float64
}{
X: 100,
Y: 100,
},
OutputPath: "testdata/output.png",
}
err = SavePositionImg(params)
require.NoError(t, err)
// cleanup
defer os.Remove(params.OutputPath)
}
func TestLoadImage(t *testing.T) { func TestLoadImage(t *testing.T) {
// Test PNG image // Test PNG image
pngBase64, pngSize, err := loadImage("testdata/llk_1.png") pngBase64, pngSize, err := builtin.LoadImage("testdata/llk_1.png")
require.NoError(t, err) require.NoError(t, err)
assert.NotEmpty(t, pngBase64) assert.NotEmpty(t, pngBase64)
assert.Greater(t, pngSize.Width, 0) assert.Greater(t, pngSize.Width, 0)
assert.Greater(t, pngSize.Height, 0) assert.Greater(t, pngSize.Height, 0)
// Test JPEG image // Test JPEG image
jpegBase64, jpegSize, err := loadImage("testdata/xhs-feed.jpeg") jpegBase64, jpegSize, err := builtin.LoadImage("testdata/xhs-feed.jpeg")
require.NoError(t, err) require.NoError(t, err)
assert.NotEmpty(t, jpegBase64) assert.NotEmpty(t, jpegBase64)
assert.Greater(t, jpegSize.Width, 0) assert.Greater(t, jpegSize.Width, 0)
assert.Greater(t, jpegSize.Height, 0) assert.Greater(t, jpegSize.Height, 0)
} }
// loadImage loads image and returns base64 encoded string
func loadImage(imagePath string) (base64Str string, size types.Size, err error) {
// Read the image file
imageFile, err := os.OpenFile(imagePath, os.O_RDONLY, 0o600)
if err != nil {
return "", types.Size{}, fmt.Errorf("failed to open image file: %w", err)
}
defer imageFile.Close()
// Decode the image to get its resolution
imageData, format, err := image.Decode(imageFile)
if err != nil {
return "", types.Size{}, fmt.Errorf("failed to decode image: %w", err)
}
// Get the resolution of the image
width := imageData.Bounds().Dx()
height := imageData.Bounds().Dy()
size = types.Size{Width: width, Height: height}
// Convert image to base64
buf := new(bytes.Buffer)
// 根据图像格式选择正确的编码器
if format == "jpeg" || format == "jpg" {
if err := jpeg.Encode(buf, imageData, nil); err != nil {
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
}
} else {
// 默认使用 PNG 编码
if err := png.Encode(buf, imageData); err != nil {
return "", types.Size{}, fmt.Errorf("failed to encode image to buffer: %w", err)
}
}
base64Str = fmt.Sprintf("data:image/%s;base64,%s", format,
base64.StdEncoding.EncodeToString(buf.Bytes()))
return base64Str, size, nil
}

26
uixt/android_driver_adb.go
View File

@@ -307,14 +307,16 @@ func (ad *ADBDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY") log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.TapAbsXY")
actionOptions := option.NewActionOptions(opts...) var err error
x, y = actionOptions.ApplyTapOffset(x, y) x, y, err = handlerTapAbsXY(ad, x, y, opts...)
if err != nil {
return err
}
// adb shell input tap x y // adb shell input tap x y
xStr := fmt.Sprintf("%.1f", x) xStr := fmt.Sprintf("%.1f", x)
yStr := fmt.Sprintf("%.1f", y) yStr := fmt.Sprintf("%.1f", y)
_, err := ad.runShellCommand( _, err = ad.runShellCommand("input", "tap", xStr, yStr)
"input", "tap", xStr, yStr)
if err != nil { if err != nil {
return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr))
} }
@@ -324,12 +326,10 @@ func (ad *ADBDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap") log.Info().Float64("x", x).Float64("y", y).Msg("ADBDriver.DoubleTap")
var err error var err error
x, y, err = convertToAbsolutePoint(ad, x, y) x, y, err = handlerDoubleTap(ad, x, y, opts...)
if err != nil { if err != nil {
return err return err
} }
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
// adb shell input tap x y // adb shell input tap x y
xStr := fmt.Sprintf("%.1f", x) xStr := fmt.Sprintf("%.1f", x)
@@ -372,13 +372,13 @@ func (ad *ADBDriver) TouchAndHold(x, y float64, opts ...option.ActionOption) (er
func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) { func (ad *ADBDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) (err error) {
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag") Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Drag")
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) fromX, fromY, toX, toY, err = handlerDrag(ad, fromX, fromY, toX, toY, opts...)
if err != nil { if err != nil {
return err return err
} }
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
actionOptions := option.NewActionOptions(opts...)
duration := 200.0 duration := 200.0
if actionOptions.Duration > 0 { if actionOptions.Duration > 0 {
duration = actionOptions.Duration * 1000 duration = actionOptions.Duration * 1000
@@ -404,12 +404,10 @@ func (ad *ADBDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe") Float64("toX", toX).Float64("toY", toY).Msg("ADBDriver.Swipe")
var err error var err error
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ad, fromX, fromY, toX, toY) fromX, fromY, toX, toY, err = handlerSwipe(ad, fromX, fromY, toX, toY)
if err != nil { if err != nil {
return err return err
} }
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
// adb shell input swipe fromX fromY toX toY // adb shell input swipe fromX fromY toX toY
_, err = ad.runShellCommand( _, err = ad.runShellCommand(
@@ -701,7 +699,7 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) {
return pointRes, nil return pointRes, nil
} }
reader, err := os.OpenFile(files[0], os.O_RDONLY, 0o600) reader, err := os.Open(files[0])
if err != nil { if err != nil {
log.Info().Msg("open File error") log.Info().Msg("open File error")
return pointRes, nil return pointRes, nil

27
uixt/android_driver_uia2.go
View File

@@ -256,12 +256,10 @@ func (ud *UIA2Driver) Orientation() (orientation types.Orientation, err error) {
func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error { func (ud *UIA2Driver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap") log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.DoubleTap")
var err error var err error
x, y, err = convertToAbsolutePoint(ud, x, y) x, y, err = handlerDoubleTap(ud, x, y, opts...)
if err != nil { if err != nil {
return err return err
} }
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
data := map[string]interface{}{ data := map[string]interface{}{
"actions": []interface{}{ "actions": []interface{}{
@@ -298,13 +296,19 @@ func (ud *UIA2Driver) TapXY(x, y float64, opts ...option.ActionOption) error {
func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY") log.Info().Float64("x", x).Float64("y", y).Msg("UIA2Driver.TapAbsXY")
// register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap")) // register(postHandler, new Tap("/wd/hub/session/:sessionId/appium/tap"))
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
var err error
x, y, err = handlerTapAbsXY(ud, x, y, opts...)
if err != nil {
return err
}
actionOptions := option.NewActionOptions(opts...)
duration := 100.0 duration := 100.0
if actionOptions.PressDuration > 0 { if actionOptions.PressDuration > 0 {
duration = actionOptions.PressDuration * 1000 // convert to ms duration = actionOptions.PressDuration * 1000 // convert to ms
} }
data := map[string]interface{}{ data := map[string]interface{}{
"actions": []interface{}{ "actions": []interface{}{
map[string]interface{}{ map[string]interface{}{
@@ -323,7 +327,7 @@ func (ud *UIA2Driver) TapAbsXY(x, y float64, opts ...option.ActionOption) error
option.MergeOptions(data, opts...) option.MergeOptions(data, opts...)
urlStr := fmt.Sprintf("/session/%s/actions/tap", ud.Session.ID) urlStr := fmt.Sprintf("/session/%s/actions/tap", ud.Session.ID)
_, err := ud.Session.POST(data, urlStr) _, err = ud.Session.POST(data, urlStr)
return err return err
} }
@@ -355,13 +359,12 @@ func (ud *UIA2Driver) TouchAndHold(x, y float64, opts ...option.ActionOption) (e
func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { func (ud *UIA2Driver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag") Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Drag")
var err error var err error
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY) fromX, fromY, toX, toY, err = handlerDrag(ud, fromX, fromY, toX, toY, opts...)
if err != nil { if err != nil {
return err return err
} }
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
data := map[string]interface{}{ data := map[string]interface{}{
"startX": fromX, "startX": fromX,
@@ -387,13 +390,11 @@ func (ud *UIA2Driver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Actio
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe") Float64("toX", toX).Float64("toY", toY).Msg("UIA2Driver.Swipe")
var err error var err error
actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = handlerSwipe(ud, fromX, fromY, toX, toY)
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(ud, fromX, fromY, toX, toY)
if err != nil { if err != nil {
return err return err
} }
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) actionOptions := option.NewActionOptions(opts...)
duration := 200.0 duration := 200.0
if actionOptions.PressDuration > 0 { if actionOptions.PressDuration > 0 {
duration = actionOptions.PressDuration * 1000 // ms duration = actionOptions.PressDuration * 1000 // ms

15
uixt/android_test.go
View File

@@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
) )
@@ -23,10 +22,12 @@ func setupADBDriverExt(t *testing.T) *XTDriver {
device.Options.LogOn = false device.Options.LogOn = false
driver, err := device.NewDriver() driver, err := device.NewDriver()
require.Nil(t, err) require.Nil(t, err)
return NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM), option.WithCVService(option.CVServiceTypeVEDEM),
ai.WithLLMService(ai.LLMServiceTypeUITARS), option.WithLLMService(option.LLMServiceTypeUITARS),
) )
require.Nil(t, err)
return driverExt
} }
func setupUIA2DriverExt(t *testing.T) *XTDriver { func setupUIA2DriverExt(t *testing.T) *XTDriver {
@@ -36,8 +37,10 @@ func setupUIA2DriverExt(t *testing.T) *XTDriver {
device.Options.LogOn = false device.Options.LogOn = false
driver, err := device.NewDriver() driver, err := device.NewDriver()
require.Nil(t, err) require.Nil(t, err)
return NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
return driverExt
} }
func TestDevice_Android_GetPackageInfo(t *testing.T) { func TestDevice_Android_GetPackageInfo(t *testing.T) {

33
uixt/browser_driver.go
View File

@@ -103,7 +103,13 @@ func NewBrowserDriver(device *BrowserDevice) (driver *BrowserDriver, err error)
return driver, nil return driver, nil
} }
func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option.ActionOption) (err error) { func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
var err error
fromX, fromY, toX, toY, err = handlerDrag(wd, fromX, fromY, toX, toY, opts...)
if err != nil {
return err
}
data := map[string]interface{}{ data := map[string]interface{}{
"from_x": fromX, "from_x": fromX,
"from_y": fromY, "from_y": fromY,
@@ -111,14 +117,13 @@ func (wd *BrowserDriver) Drag(fromX, fromY, toX, toY float64, options ...option.
"to_y": toY, "to_y": toY,
} }
actionOptions := option.NewActionOptions(options...) actionOptions := option.NewActionOptions(opts...)
if actionOptions.Duration > 0 { if actionOptions.Duration > 0 {
data["duration"] = actionOptions.Duration data["duration"] = actionOptions.Duration
} }
_, err = wd.HttpPOST(data, wd.sessionId, "ui/drag") _, err = wd.HttpPOST(data, wd.sessionId, "ui/drag")
return return err
} }
func (wd *BrowserDriver) AppLaunch(packageName string) (err error) { func (wd *BrowserDriver) AppLaunch(packageName string) (err error) {
@@ -517,28 +522,40 @@ func (wd *BrowserDriver) Tap(x, y float64, options ...option.ActionOption) error
return wd.TapFloat(x, y, options...) return wd.TapFloat(x, y, options...)
} }
func (wd *BrowserDriver) TapFloat(x, y float64, options ...option.ActionOption) error { func (wd *BrowserDriver) TapFloat(x, y float64, opts ...option.ActionOption) error {
actionOptions := option.NewActionOptions(options...) var err error
x, y, err = handlerTapAbsXY(wd, x, y, opts...)
if err != nil {
return err
}
actionOptions := option.NewActionOptions(opts...)
duration := 0.1 duration := 0.1
if actionOptions.Duration > 0 { if actionOptions.Duration > 0 {
duration = actionOptions.Duration duration = actionOptions.Duration
} }
data := map[string]interface{}{ data := map[string]interface{}{
"x": x, "x": x,
"y": y, "y": y,
"duration": duration, "duration": duration,
} }
_, err := wd.HttpPOST(data, wd.sessionId, "ui/tap") _, err = wd.HttpPOST(data, wd.sessionId, "ui/tap")
return err return err
} }
// DoubleTap Sends a double tap event at the coordinate. // DoubleTap Sends a double tap event at the coordinate.
func (wd *BrowserDriver) DoubleTap(x, y float64, options ...option.ActionOption) error { func (wd *BrowserDriver) DoubleTap(x, y float64, options ...option.ActionOption) error {
var err error
x, y, err = handlerDoubleTap(wd, x, y, options...)
if err != nil {
return err
}
data := map[string]interface{}{ data := map[string]interface{}{
"x": x, "x": x,
"y": y, "y": y,
} }
_, err := wd.HttpPOST(data, wd.sessionId, "ui/double_tap") _, err = wd.HttpPOST(data, wd.sessionId, "ui/double_tap")
return err return err
} }

6
uixt/demo/main_test.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
) )
@@ -23,9 +22,10 @@ func TestIOSDemo(t *testing.T) {
driver, err := device.NewDriver() driver, err := device.NewDriver()
assert.Nil(t, err) assert.Nil(t, err)
driverExt := uixt.NewXTDriver(driver, driverExt, err := uixt.NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM), option.WithCVService(option.CVServiceTypeVEDEM),
) )
assert.Nil(t, err)
// release session // release session
defer func() { defer func() {

34
uixt/driver.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
"github.com/rs/zerolog/log"
) )
var ( var (
@@ -81,16 +82,30 @@ type IDriver interface {
StopCaptureLog() (result interface{}, err error) StopCaptureLog() (result interface{}, err error)
} }
func NewXTDriver(driver IDriver, opts ...ai.AIServiceOption) *XTDriver { func NewXTDriver(driver IDriver, opts ...option.AIServiceOption) (*XTDriver, error) {
services := ai.NewAIService(opts...)
driverExt := &XTDriver{ driverExt := &XTDriver{
IDriver: driver, IDriver: driver,
CVService: services.ICVService,
LLMService: services.ILLMService,
screenResults: make([]*ScreenResult, 0),
} }
return driverExt
services := option.NewAIServiceOptions(opts...)
var err error
if services.CVService != "" {
driverExt.CVService, err = ai.NewCVService(services.CVService)
if err != nil {
log.Error().Err(err).Msg("init vedem image service failed")
return nil, err
}
}
if services.LLMService != "" {
driverExt.LLMService, err = ai.NewLLMService(services.LLMService)
if err != nil {
log.Error().Err(err).Msg("init llm service failed")
return nil, err
}
}
return driverExt, nil
} }
// XTDriver = IDriver + AI // XTDriver = IDriver + AI
@@ -98,7 +113,4 @@ type XTDriver struct {
IDriver IDriver
CVService ai.ICVService // OCR/CV CVService ai.ICVService // OCR/CV
LLMService ai.ILLMService // LLM LLMService ai.ILLMService // LLM
// cache screenshot results
screenResults []*ScreenResult
} }

1
uixt/driver_action.go
View File

@@ -42,6 +42,7 @@ const (
ACTION_TapByCV ActionMethod = "tap_cv" ACTION_TapByCV ActionMethod = "tap_cv"
ACTION_DoubleTapXY ActionMethod = "double_tap_xy" ACTION_DoubleTapXY ActionMethod = "double_tap_xy"
ACTION_Swipe ActionMethod = "swipe" ACTION_Swipe ActionMethod = "swipe"
ACTION_Drag ActionMethod = "drag"
ACTION_Input ActionMethod = "input" ACTION_Input ActionMethod = "input"
ACTION_Back ActionMethod = "back" ACTION_Back ActionMethod = "back"
ACTION_KeyCode ActionMethod = "keycode" ACTION_KeyCode ActionMethod = "keycode"

261
uixt/driver_ext_screenshot.go
View File

@@ -4,12 +4,16 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"image" "image"
"image/color"
"image/draw"
"image/gif" "image/gif"
"image/jpeg" "image/jpeg"
"image/png" "image/png"
"math"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@@ -129,8 +133,9 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
} }
} }
// cache screen result // save screen result to session
dExt.screenResults = append(dExt.screenResults, screenResult) session := dExt.GetSession()
session.screenResults = append(session.screenResults, screenResult)
log.Debug(). log.Debug().
Str("imagePath", imagePath). Str("imagePath", imagePath).
@@ -294,3 +299,255 @@ func compressImageBuffer(raw *bytes.Buffer) (compressed *bytes.Buffer, err error
// return compressed image buffer // return compressed image buffer
return &buf, nil return &buf, nil
} }
// MarkUIOperation add operation mark for UI operation
func MarkUIOperation(driver IDriver, actionType ActionMethod, actionCoordinates []float64) error {
if actionType == "" || len(actionCoordinates) == 0 {
return nil
}
start := time.Now()
// get screenshot
compressedBufSource, err := driver.ScreenShot()
if err != nil {
return err
}
// create screenshot save path
timestamp := builtin.GenNameWithTimestamp("action_%d")
var imagePath string
if actionType == ACTION_TapAbsXY || actionType == ACTION_DoubleTapXY {
if len(actionCoordinates) != 2 {
return fmt.Errorf("invalid tap action coordinates: %v", actionCoordinates)
}
imagePath = filepath.Join(
config.GetConfig().ScreenShotsPath,
fmt.Sprintf("%s_%s.png", timestamp, actionType),
)
x, y := actionCoordinates[0], actionCoordinates[1]
point := image.Point{X: int(x), Y: int(y)}
err = SaveImageWithCircleMarker(compressedBufSource, point, imagePath)
} else if actionType == ACTION_Swipe || actionType == ACTION_Drag {
if len(actionCoordinates) != 4 {
return fmt.Errorf("invalid swipe action coordinates: %v", actionCoordinates)
}
imagePath = filepath.Join(
config.GetConfig().ScreenShotsPath,
fmt.Sprintf("%s_%s.png", timestamp, actionType),
)
fromX, fromY := actionCoordinates[0], actionCoordinates[1]
toX, toY := actionCoordinates[2], actionCoordinates[3]
from := image.Point{X: int(fromX), Y: int(fromY)}
to := image.Point{X: int(toX), Y: int(toY)}
err = SaveImageWithArrowMarker(compressedBufSource, from, to, imagePath)
}
if err != nil {
log.Error().Err(err).
Int64("duration(ms)", time.Since(start).Milliseconds()).
Msg("mark UI operation failed")
return err
}
if imagePath != "" {
log.Info().Str("operation", string(actionType)).
Str("imagePath", imagePath).
Int64("duration(ms)", time.Since(start).Milliseconds()).
Msg("mark UI operation success")
// save screenshot to session
session := driver.GetSession()
session.screenResults = append(session.screenResults, &ScreenResult{
bufSource: compressedBufSource,
ImagePath: imagePath,
})
}
return nil
}
// SaveImageWithCircleMarker saves an image with circle marker
func SaveImageWithCircleMarker(imgBuf *bytes.Buffer, point image.Point, outputPath string) error {
img, _, err := image.Decode(imgBuf)
if err != nil {
return fmt.Errorf("failed to decode image data: %w", err)
}
bounds := img.Bounds()
rgba := image.NewRGBA(bounds)
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
// draw a red circle at the tap point
centerX := point.X
centerY := point.Y
radius := 20
lineWidth := 5
red := color.RGBA{255, 0, 0, 255}
for angle := 0.0; angle < 2*math.Pi; angle += 0.01 {
for w := 0; w < lineWidth; w++ {
r := float64(radius - w)
x := int(float64(centerX) + r*math.Cos(angle))
y := int(float64(centerY) + r*math.Sin(angle))
if x >= 0 && x < bounds.Max.X && y >= 0 && y < bounds.Max.Y {
rgba.Set(x, y, red)
}
}
}
outFile, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer outFile.Close()
if err := png.Encode(outFile, rgba); err != nil {
return fmt.Errorf("failed to encode and save image: %w", err)
}
return nil
}
// SaveImageWithArrowMarker saves an image with an arrow marker
func SaveImageWithArrowMarker(imgBuf *bytes.Buffer, from, to image.Point, outputPath string) error {
img, _, err := image.Decode(imgBuf)
if err != nil {
return fmt.Errorf("failed to decode image data: %w", err)
}
bounds := img.Bounds()
rgba := image.NewRGBA(bounds)
draw.Draw(rgba, bounds, img, bounds.Min, draw.Src)
drawArrow(rgba, from, to, color.RGBA{255, 0, 0, 255}, 5)
outFile, err := os.Create(outputPath)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer outFile.Close()
if err := png.Encode(outFile, rgba); err != nil {
return fmt.Errorf("failed to encode and save image: %w", err)
}
return nil
}
// drawArrow draws an arrow from 'from' to 'to' on the image
func drawArrow(rgba *image.RGBA, from, to image.Point, color color.RGBA, lineWidth int) {
bounds := rgba.Bounds()
dx, dy := to.X-from.X, to.Y-from.Y
steps := int(math.Sqrt(float64(dx*dx + dy*dy)))
if steps == 0 {
steps = 1
}
stepX, stepY := float64(dx)/float64(steps), float64(dy)/float64(steps)
// main line
for i := 0; i < steps; i++ {
x := int(float64(from.X) + stepX*float64(i))
y := int(float64(from.Y) + stepY*float64(i))
for w := 0; w < lineWidth; w++ {
offsetX, offsetY := 0, 0
if math.Abs(stepX) > math.Abs(stepY) {
offsetY = w - lineWidth/2
} else {
offsetX = w - lineWidth/2
}
drawX, drawY := x+offsetX, y+offsetY
if drawX >= 0 && drawX < bounds.Max.X && drawY >= 0 && drawY < bounds.Max.Y {
rgba.Set(drawX, drawY, color)
}
}
}
// arrow head
arrowLength := float64(steps) * 0.15
if arrowLength < 10 {
arrowLength = 10
} else if arrowLength > 30 {
arrowLength = 30
}
head := calculateArrowHead(float64(from.X), float64(from.Y), float64(to.X), float64(to.Y), arrowLength)
if head != nil {
for _, point := range head[:2] {
drawLineInImage(rgba, to.X, to.Y, int(point.X), int(point.Y), color, lineWidth, bounds)
}
for _, point := range head[1:] {
drawLineInImage(rgba, to.X, to.Y, int(point.X), int(point.Y), color, lineWidth, bounds)
}
}
}
// calculateArrowHead calculates the endpoint and arrowhead coordinates
func calculateArrowHead(fromX, fromY, toX, toY float64, arrowLength float64) []struct{ X, Y float64 } {
// calculate direction vector
dx, dy := toX-fromX, toY-fromY
// calculate distance
length := math.Sqrt(dx*dx + dy*dy)
if length < 1e-6 {
return nil
}
// unit vector
dx, dy = dx/length, dy/length
// calculate orthogonal vector of arrow direction (counterclockwise 90 degrees)
orthX, orthY := -dy, dx
// calculate two wing points of arrow
headWidth := arrowLength * 0.5
backX, backY := toX-dx*arrowLength, toY-dy*arrowLength
// two wing points of arrow
leftWingX, leftWingY := backX+orthX*headWidth, backY+orthY*headWidth
rightWingX, rightWingY := backX-orthX*headWidth, backY-orthY*headWidth
return []struct{ X, Y float64 }{
{leftWingX, leftWingY},
{toX, toY},
{rightWingX, rightWingY},
}
}
// drawLineInImage draws a line on the image
func drawLineInImage(img *image.RGBA, x0, y0, x1, y1 int, lineColor color.RGBA, lineWidth int, bounds image.Rectangle) {
// use Bresenham algorithm to draw line
dx, dy := math.Abs(float64(x1-x0)), math.Abs(float64(y1-y0))
sx, sy := 1, 1
if x0 >= x1 {
sx = -1
}
if y0 >= y1 {
sy = -1
}
err := dx - dy
for {
// draw point (consider line width)
for w := 0; w < lineWidth; w++ {
offsetX, offsetY := 0, 0
// decide offset direction based on line angle
if dx > dy {
// more horizontal line
offsetY = w - lineWidth/2
} else {
// more vertical line
offsetX = w - lineWidth/2
}
drawX, drawY := x0+offsetX, y0+offsetY
if drawX >= 0 && drawX < bounds.Max.X && drawY >= 0 && drawY < bounds.Max.Y {
img.Set(drawX, drawY, lineColor)
}
}
// end of line
if x0 == x1 && y0 == y1 {
break
}
// calculate next point
e2 := 2 * err
if e2 > -dy {
err = err - dy
x0 = x0 + sx
}
if e2 < dx {
err = err + dx
y0 = y0 + sy
}
}
}

69
uixt/driver_ext_test.go
View File

@@ -3,10 +3,12 @@
package uixt package uixt
import ( import (
"bytes"
"image"
"os"
"testing" "testing"
"time" "time"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -17,8 +19,9 @@ func TestDriverExt_NewMethod1(t *testing.T) {
require.Nil(t, err) require.Nil(t, err)
driver, err := device.NewDriver() driver, err := device.NewDriver()
require.Nil(t, err) require.Nil(t, err)
driverExt := NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
driverExt.TapByOCR("推荐") driverExt.TapByOCR("推荐")
} }
@@ -27,16 +30,18 @@ func TestDriverExt_NewMethod2(t *testing.T) {
require.Nil(t, err) require.Nil(t, err)
driver, err := NewUIA2Driver(device) driver, err := NewUIA2Driver(device)
require.Nil(t, err) require.Nil(t, err)
driverExt := NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
driverExt.TapByOCR("推荐") driverExt.TapByOCR("推荐")
} }
func TestDriverExt(t *testing.T) { func TestDriverExt(t *testing.T) {
device, _ := NewAndroidDevice() device, _ := NewAndroidDevice()
driver, _ := NewADBDriver(device) driver, _ := NewADBDriver(device)
driverExt := NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
ai.WithCVService(ai.CVServiceTypeVEDEM)) option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
// call IDriver methods // call IDriver methods
driverExt.TapXY(0.2, 0.5) driverExt.TapXY(0.2, 0.5)
@@ -50,7 +55,7 @@ func TestDriverExt(t *testing.T) {
textRect, _ := driverExt.FindScreenText("hello") textRect, _ := driverExt.FindScreenText("hello")
t.Log(textRect) t.Log(textRect)
err := driverExt.TapByCV( err = driverExt.TapByCV(
option.WithScreenShotUITypes("deepseek_send"), option.WithScreenShotUITypes("deepseek_send"),
option.WithScope(0.8, 0.5, 1, 1)) option.WithScope(0.8, 0.5, 1, 1))
assert.Nil(t, err) assert.Nil(t, err)
@@ -249,3 +254,51 @@ func TestDriverExt_Action_Offset(t *testing.T) {
option.WithTapRandomRect(true)) option.WithTapRandomRect(true))
assert.Nil(t, err) assert.Nil(t, err)
} }
func TestSaveImageWithCircle(t *testing.T) {
imgBytes, err := os.ReadFile("ai/testdata/llk_1.png")
require.NoError(t, err)
imgBuf := bytes.NewBuffer(imgBytes)
point := image.Point{X: 500, Y: 500}
outputPath := "ai/testdata/output.png"
err = SaveImageWithCircleMarker(imgBuf, point, outputPath)
require.NoError(t, err)
defer os.Remove(outputPath)
}
func TestSaveImageWithArrow(t *testing.T) {
imgBytes, err := os.ReadFile("ai/testdata/llk_1.png")
require.NoError(t, err)
imgBuf := bytes.NewBuffer(imgBytes)
from := image.Point{X: 500, Y: 500}
to := image.Point{X: 1000, Y: 1000}
outputPath := "ai/testdata/output.png"
err = SaveImageWithArrowMarker(imgBuf, from, to, outputPath)
require.NoError(t, err)
defer os.Remove(outputPath)
}
func TestMarkOperation(t *testing.T) {
driver := setupDriverExt(t)
opts := []option.ActionOption{option.WithMarkOperationEnabled(true)}
// tap point
err := driver.TapXY(0.5, 0.5, opts...)
assert.Nil(t, err)
err = driver.TapAbsXY(500, 800, opts...)
assert.Nil(t, err)
// swipe
err = driver.Swipe(0.2, 0.5, 0.8, 0.5, opts...)
assert.Nil(t, err)
err = driver.Swipe(0.3, 0.7, 0.3, 0.3, opts...)
assert.Nil(t, err)
}

83
uixt/driver_handler.go Normal file
View File

@@ -0,0 +1,83 @@
package uixt
import (
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/rs/zerolog/log"
)
func handlerTapAbsXY(driver IDriver, rawX, rawY float64, opts ...option.ActionOption) (
x, y float64, err error) {
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(rawX, rawY)
// mark UI operation
if actionOptions.MarkOperationEnabled {
if markErr := MarkUIOperation(driver, ACTION_TapAbsXY, []float64{x, y}); markErr != nil {
log.Warn().Err(markErr).Msg("Failed to mark tap operation")
}
}
return x, y, nil
}
func handlerDoubleTap(driver IDriver, rawX, rawY float64, opts ...option.ActionOption) (
x, y float64, err error) {
x, y, err = convertToAbsolutePoint(driver, rawX, rawY)
if err != nil {
return 0, 0, err
}
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
// mark UI operation
if actionOptions.MarkOperationEnabled {
if markErr := MarkUIOperation(driver, ACTION_DoubleTapXY, []float64{x, y}); markErr != nil {
log.Warn().Err(markErr).Msg("Failed to mark double tap operation")
}
}
return x, y, nil
}
func handlerDrag(driver IDriver, rawFomX, rawFromY, rawToX, rawToY float64, opts ...option.ActionOption) (
fromX, fromY, toX, toY float64, err error) {
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(driver, rawFomX, rawFromY, rawToX, rawToY)
if err != nil {
return 0, 0, 0, 0, err
}
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
// mark UI operation
if actionOptions.MarkOperationEnabled {
if markErr := MarkUIOperation(driver, ACTION_Drag, []float64{fromX, fromY, toX, toY}); markErr != nil {
log.Warn().Err(markErr).Msg("Failed to mark drag operation")
}
}
return fromX, fromY, toX, toY, nil
}
func handlerSwipe(driver IDriver, rawFomX, rawFromY, rawToX, rawToY float64, opts ...option.ActionOption) (
fromX, fromY, toX, toY float64, err error) {
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(driver, rawFomX, rawFromY, rawToX, rawToY)
if err != nil {
return 0, 0, 0, 0, err
}
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY)
// mark UI operation
if actionOptions.MarkOperationEnabled {
if markErr := MarkUIOperation(driver, ACTION_Swipe, []float64{fromX, fromY, toX, toY}); markErr != nil {
log.Warn().Err(markErr).Msg("Failed to mark swipe operation")
}
}
return fromX, fromY, toX, toY, nil
}

6
uixt/driver_session.go
View File

@@ -47,9 +47,9 @@ func NewDriverSession() *DriverSession {
client: &http.Client{ client: &http.Client{
Timeout: timeout, Timeout: timeout,
}, },
requests: make([]*DriverRequests, 0),
maxRetry: 5, maxRetry: 5,
} }
session.Reset()
return session return session
} }
@@ -66,10 +66,14 @@ type DriverSession struct {
// cache driver request and response // cache driver request and response
requests []*DriverRequests requests []*DriverRequests
// cache screenshot results
screenResults []*ScreenResult
} }
func (s *DriverSession) Reset() { func (s *DriverSession) Reset() {
s.requests = make([]*DriverRequests, 0) s.requests = make([]*DriverRequests, 0)
s.screenResults = make([]*ScreenResult, 0)
} }
func (s *DriverSession) SetBaseURL(baseUrl string) { func (s *DriverSession) SetBaseURL(baseUrl string) {

3
uixt/driver_utils.go
View File

@@ -117,11 +117,10 @@ func (dExt *XTDriver) GetData(withReset bool) map[string]interface{} {
session := dExt.GetSession() session := dExt.GetSession()
data := map[string]interface{}{ data := map[string]interface{}{
"requests": session.History(), "requests": session.History(),
"screen_results": dExt.screenResults, "screen_results": session.screenResults,
} }
if withReset { if withReset {
session.Reset() session.Reset()
dExt.screenResults = make([]*ScreenResult, 0)
} }
return data return data
} }

15
uixt/harmony_driver_hdc.go
View File

@@ -154,9 +154,14 @@ func (hd *HDCDriver) TapXY(x, y float64, opts ...option.ActionOption) error {
func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { func (hd *HDCDriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY") log.Info().Float64("x", x).Float64("y", y).Msg("HDCDriver.TapAbsXY")
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
var err error
x, y, err = handlerTapAbsXY(hd, x, y, opts...)
if err != nil {
return err
}
actionOptions := option.NewActionOptions(opts...)
if actionOptions.Identifier != "" { if actionOptions.Identifier != "" {
startTime := int(time.Now().UnixMilli()) startTime := int(time.Now().UnixMilli())
hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100}) hd.points = append(hd.points, ExportPoint{Start: startTime, End: startTime + 100, Ext: actionOptions.Identifier, RunTime: 100})
@@ -182,13 +187,11 @@ func (hd *HDCDriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe") Float64("toX", toX).Float64("toY", toY).Msg("HDCDriver.Swipe")
var err error var err error
actionOptions := option.NewActionOptions(opts...) fromX, fromY, toX, toY, err = handlerSwipe(hd, fromX, fromY, toX, toY)
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(hd, fromX, fromY, toX, toY)
if err != nil { if err != nil {
return err return err
} }
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) actionOptions := option.NewActionOptions(opts...)
duration := 200 duration := 200
if actionOptions.PressDuration > 0 { if actionOptions.PressDuration > 0 {
duration = int(actionOptions.PressDuration * 1000) duration = int(actionOptions.PressDuration * 1000)

6
uixt/harmony_test.go
View File

@@ -6,7 +6,7 @@ import (
"fmt" "fmt"
"testing" "testing"
"github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -16,7 +16,9 @@ func setupHDCDriverExt(t *testing.T) *XTDriver {
require.Nil(t, err) require.Nil(t, err)
hdcDriver, err := NewHDCDriver(device) hdcDriver, err := NewHDCDriver(device)
require.Nil(t, err) require.Nil(t, err)
return NewXTDriver(hdcDriver, ai.WithCVService(ai.CVServiceTypeVEDEM)) driverExt, err := NewXTDriver(hdcDriver, option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
return driverExt
} }
func TestWindowSize(t *testing.T) { func TestWindowSize(t *testing.T) {

44
uixt/ios_driver_wda.go
View File

@@ -593,32 +593,40 @@ func (wd *WDADriver) TapXY(x, y float64, opts ...option.ActionOption) error {
func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error { func (wd *WDADriver) TapAbsXY(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY") log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.TapAbsXY")
// [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)] // [[FBRoute POST:@"/wda/tap/:uuid"] respondWithTarget:self action:@selector(handleTap:)]
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y) x = wd.toScale(x)
y = wd.toScale(y)
var err error
x, y, err = handlerTapAbsXY(wd, x, y, opts...)
if err != nil {
return err
}
data := map[string]interface{}{ data := map[string]interface{}{
"x": wd.toScale(x), "x": x,
"y": wd.toScale(y), "y": y,
} }
option.MergeOptions(data, opts...) option.MergeOptions(data, opts...)
urlStr := fmt.Sprintf("/session/%s/wda/tap/0", wd.Session.ID) urlStr := fmt.Sprintf("/session/%s/wda/tap/0", wd.Session.ID)
_, err := wd.Session.POST(data, urlStr) _, err = wd.Session.POST(data, urlStr)
return err return err
} }
func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error { func (wd *WDADriver) DoubleTap(x, y float64, opts ...option.ActionOption) error {
log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap") log.Info().Float64("x", x).Float64("y", y).Msg("WDADriver.DoubleTap")
// [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)] // [[FBRoute POST:@"/wda/doubleTap"] respondWithTarget:self action:@selector(handleDoubleTapCoordinate:)]
x = wd.toScale(x)
y = wd.toScale(y)
var err error var err error
x, y, err = convertToAbsolutePoint(wd, x, y) x, y, err = handlerDoubleTap(wd, x, y, opts...)
if err != nil { if err != nil {
return err return err
} }
actionOptions := option.NewActionOptions(opts...)
x, y = actionOptions.ApplyTapOffset(x, y)
x = wd.toScale(x)
y = wd.toScale(y)
data := map[string]interface{}{ data := map[string]interface{}{
"x": x, "x": x,
"y": y, "y": y,
@@ -643,17 +651,17 @@ func (wd *WDADriver) Drag(fromX, fromY, toX, toY float64, opts ...option.ActionO
log.Info().Float64("fromX", fromX).Float64("fromY", fromY). log.Info().Float64("fromX", fromX).Float64("fromY", fromY).
Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag") Float64("toX", toX).Float64("toY", toY).Msg("WDADriver.Drag")
// [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)] // [[FBRoute POST:@"/wda/dragfromtoforduration"] respondWithTarget:self action:@selector(handleDragCoordinate:)]
var err error
fromX, fromY, toX, toY, err = convertToAbsoluteCoordinates(wd, fromX, fromY, toX, toY)
if err != nil {
return err
}
fromX = wd.toScale(fromX) fromX = wd.toScale(fromX)
fromY = wd.toScale(fromY) fromY = wd.toScale(fromY)
toX = wd.toScale(toX) toX = wd.toScale(toX)
toY = wd.toScale(toY) toY = wd.toScale(toY)
actionOptions := option.NewActionOptions(opts...)
fromX, fromY, toX, toY = actionOptions.ApplySwipeOffset(fromX, fromY, toX, toY) var err error
fromX, fromY, toX, toY, err = handlerDrag(wd, fromX, fromY, toX, toY, opts...)
if err != nil {
return err
}
data := map[string]interface{}{ data := map[string]interface{}{
"fromX": math.Round(fromX*10) / 10, "fromX": math.Round(fromX*10) / 10,
@@ -970,7 +978,7 @@ func (wd *WDADriver) StartCaptureLog(identifier ...string) error {
func (wd *WDADriver) PushImage(localPath string) error { func (wd *WDADriver) PushImage(localPath string) error {
log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage") log.Info().Str("localPath", localPath).Msg("WDADriver.PushImage")
localFile, err := os.OpenFile(localPath, os.O_RDONLY, 0o600) localFile, err := os.Open(localPath)
if err != nil { if err != nil {
return err return err
} }

5
uixt/ios_test.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/httprunner/httprunner/v5/uixt/ai"
"github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types" "github.com/httprunner/httprunner/v5/uixt/types"
) )
@@ -23,7 +22,9 @@ func setupWDADriverExt(t *testing.T) *XTDriver {
require.Nil(t, err) require.Nil(t, err)
driver, err := device.NewDriver() driver, err := device.NewDriver()
require.Nil(t, err) require.Nil(t, err)
return NewXTDriver(driver, ai.WithCVService(ai.CVServiceTypeVEDEM)) driverExt, err := NewXTDriver(driver, option.WithCVService(option.CVServiceTypeVEDEM))
require.Nil(t, err)
return driverExt
} }
func TestDevice_IOS_Install(t *testing.T) { func TestDevice_IOS_Install(t *testing.T) {

1
uixt/option/action.go
View File

@@ -132,6 +132,7 @@ func (o *ActionOptions) Options() []ActionOption {
options = append(options, o.GetScreenShotOptions()...) options = append(options, o.GetScreenShotOptions()...)
options = append(options, o.GetScreenRecordOptions()...) options = append(options, o.GetScreenRecordOptions()...)
options = append(options, o.GetMarkOperationOptions()...)
return options return options
} }

43
uixt/option/ai.go Normal file
View File

@@ -0,0 +1,43 @@
package option
func NewAIServiceOptions(opts ...AIServiceOption) *AIServiceOptions {
services := &AIServiceOptions{}
for _, option := range opts {
option(services)
}
return services
}
type AIServiceOptions struct {
CVService CVServiceType
LLMService LLMServiceType
}
type AIServiceOption func(*AIServiceOptions)
type CVServiceType string
const (
CVServiceTypeVEDEM CVServiceType = "vedem"
CVServiceTypeOpenCV CVServiceType = "opencv"
)
func WithCVService(service CVServiceType) AIServiceOption {
return func(opts *AIServiceOptions) {
opts.CVService = service
}
}
type LLMServiceType string
const (
LLMServiceTypeUITARS LLMServiceType = "ui-tars"
LLMServiceTypeGPT LLMServiceType = "gpt"
LLMServiceTypeQwenVL LLMServiceType = "qwen-vl"
)
func WithLLMService(modelType LLMServiceType) AIServiceOption {
return func(opts *AIServiceOptions) {
opts.LLMService = modelType
}
}

27
uixt/option/screen.go
View File

@@ -8,6 +8,7 @@ type ScreenOptions struct {
ScreenShotOptions ScreenShotOptions
ScreenRecordOptions ScreenRecordOptions
ScreenFilterOptions ScreenFilterOptions
MarkOperationOptions
} }
type ScreenShotOptions struct { type ScreenShotOptions struct {
@@ -273,3 +274,29 @@ func WithIndex(index int) ActionOption {
o.Index = index o.Index = index
} }
} }
// MarkOperationOptions contains options for marking UI operations
type MarkOperationOptions struct {
// mark UI operation, enable/disable UI operation marking
MarkOperationEnabled bool `json:"mark_operation_enabled,omitempty" yaml:"mark_operation_enabled,omitempty"`
}
func (o *MarkOperationOptions) GetMarkOperationOptions() []ActionOption {
options := make([]ActionOption, 0)
if o == nil {
return options
}
if o.MarkOperationEnabled {
options = append(options, WithMarkOperationEnabled(true))
}
return options
}
// WithMarkOperationEnabled enables or disables UI operation marking
func WithMarkOperationEnabled(enabled bool) ActionOption {
return func(o *ActionOptions) {
o.MarkOperationEnabled = enabled
}
}