mirror of
https://github.com/httprunner/httprunner.git
synced 2026-05-06 20:32:44 +08:00
refactor: llm planner
This commit is contained in:
lilong.129
44
go.mod
44
go.mod
@@ -1,6 +1,6 @@
|
||||
module github.com/httprunner/httprunner/v5
|
||||
|
||||
go 1.23
|
||||
go 1.23.0
|
||||
|
||||
toolchain go1.23.7
|
||||
|
||||
@@ -8,13 +8,15 @@ require (
|
||||
github.com/Masterminds/semver v1.5.0
|
||||
github.com/andybalholm/brotli v1.0.4
|
||||
github.com/bytedance/sonic v1.13.2
|
||||
github.com/cloudwego/eino v0.3.23
|
||||
github.com/cloudwego/eino v0.3.26
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd
|
||||
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd
|
||||
github.com/danielpaulus/go-ios v1.0.161
|
||||
github.com/denisbrodbeck/machineid v1.0.1
|
||||
github.com/fatih/color v1.16.0
|
||||
github.com/getkin/kin-openapi v0.118.0
|
||||
github.com/getsentry/sentry-go v0.13.0
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/go-openapi/spec v0.20.7
|
||||
@@ -33,8 +35,8 @@ require (
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/spf13/cobra v1.5.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
golang.org/x/net v0.26.0
|
||||
golang.org/x/text v0.21.0
|
||||
golang.org/x/net v0.39.0
|
||||
golang.org/x/text v0.24.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -43,17 +45,15 @@ require (
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/getkin/kin-openapi v0.118.0 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-errors/errors v1.4.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.1 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.0 // indirect
|
||||
github.com/go-openapi/swag v0.22.3 // indirect
|
||||
github.com/go-openapi/swag v0.23.1 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
||||
@@ -71,12 +71,12 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.0.1 // indirect
|
||||
github.com/invopop/yaml v0.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6 // indirect
|
||||
github.com/miekg/dns v1.1.57 // indirect
|
||||
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@@ -85,15 +85,13 @@ require (
|
||||
github.com/nikolalohinski/gonja v1.5.3 // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/perimeterx/marshmallow v1.1.4 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/perimeterx/marshmallow v1.1.5 // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
|
||||
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sashabaranov/go-openai v1.32.5 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect
|
||||
@@ -107,14 +105,14 @@ require (
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
|
||||
go.uber.org/mock v0.4.0 // indirect
|
||||
golang.org/x/arch v0.11.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
|
||||
golang.org/x/mod v0.17.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/arch v0.16.0 // indirect
|
||||
golang.org/x/crypto v0.37.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
|
||||
golang.org/x/mod v0.24.0 // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
golang.org/x/tools v0.32.0 // indirect
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
|
||||
google.golang.org/grpc v1.57.0 // indirect
|
||||
|
||||
84
go.sum
84
go.sum
@@ -27,16 +27,16 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/eino v0.3.23 h1:fPjskHM85I4PPa+GZPoh76bMCbdRKWVKd52gdvcGHt8=
|
||||
github.com/cloudwego/eino v0.3.23/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY=
|
||||
github.com/cloudwego/eino v0.3.26 h1:FdJJTCdNrc9xPcYkLZiEyr7AA+WgyCKCbY+VNDXIaCE=
|
||||
github.com/cloudwego/eino v0.3.26/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6 h1:k17Z9VIRBL0/t7Ty1drGgY9tVOraM5xuO6gy7Qx7xus=
|
||||
github.com/cloudwego/eino-ext/components/model/ark v0.1.6/go.mod h1:13kQjYGLMgla6xTbejlpqhuk3i5BPlNv5S+1pmknlOo=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c h1:04WQpGikdQv6fh5wzMYSQhO0SJraV8+xcb9VQ00+HX4=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250314110024-9e89ba18146c/go.mod h1:YGP4q3uspj5qhkv3CnvlEPSo0YGeWpvkkTUHHpLExas=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd h1:XEI7RezzV/cnOnhc1YeBJi6a0UoM41JTph4AZZR7+D8=
|
||||
github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250417123744-154d7ca4d3cd/go.mod h1:8gMakAGQUR+IaWTSD0cpcD4U5FYq5puZ73/QjXqs1oU=
|
||||
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa h1:Jrmw8Q9g1WcE+x5t3o0TsEBM8RoMRURJI6P52I/ld74=
|
||||
github.com/cloudwego/eino-ext/components/tool/mcp v0.0.0-20250328102648-b47e7f1587fa/go.mod h1:UzVdRk1E+TuDxjuSAdxt5dMeAc6XJGbhJscfvKGQC8Y=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955 h1:fgvkmTqAalDfjdy3b6Ur2mh/KEwB9L2uvqS4MFgTOqc=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250305023926-469de0301955/go.mod h1:6CThw1XQx/ASXNt31yuvp0X4Yp4GprknQuIvP9VKDpw=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd h1:CJkxSpN3+lhV/dye7ui8hoCHU8VV4TecQfca5c8hx9g=
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250417123744-154d7ca4d3cd/go.mod h1:Ye0YAqpESCxMlnALNrjeNJjhS9q2PIdxVdJbtFeni8o=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
|
||||
@@ -79,16 +79,17 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic=
|
||||
github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk=
|
||||
github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
|
||||
github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
|
||||
github.com/go-openapi/spec v0.20.7 h1:1Rlu/ZrOCCob0n+JKKJAWhNWMPW8bOZRg8FJaY+0SKI=
|
||||
github.com/go-openapi/spec v0.20.7/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
|
||||
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
|
||||
github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU=
|
||||
github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -180,8 +181,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@@ -197,8 +198,9 @@ github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjS
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
|
||||
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/maja42/goval v1.2.1 h1:fyEgzddqPgCZsKcFLk4C6SdCHyEaAHYvtZG4mGzQOHU=
|
||||
github.com/maja42/goval v1.2.1/go.mod h1:42LU+BQXL/veE9jnTTUOSj38GRmOTSThYSXRVodI5J4=
|
||||
github.com/mark3labs/mcp-go v0.17.0 h1:5Ps6T7qXr7De/2QTqs9h6BKeZ/qdeUeGrgM5lPzi930=
|
||||
@@ -213,6 +215,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6 h1:nmdXxiUX48DZ2ELC/jSYzyGUVgxVEF2QJRGhLJ933zA=
|
||||
github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6/go.mod h1:kyz7fcXqXtccmRAIARn1Q+cKLNXJHC3AoqqJGeCqNI0=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
@@ -241,15 +245,15 @@ github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3Ro
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
|
||||
github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
|
||||
github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -260,17 +264,14 @@ github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 h1:I4N3ZRnkZPbDN935Tg8QDf8fRpHp3bZ0U0/L42jBgNE=
|
||||
github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sashabaranov/go-openai v1.32.5 h1:/eNVa8KzlE7mJdKPZDj6886MUzZQjoVHyn0sLvIt5qA=
|
||||
github.com/sashabaranov/go-openai v1.32.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
@@ -303,8 +304,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tadglines/go-pkgs v0.0.0-20210623144937-b983b20f54f9 h1:aeN+ghOV0b2VCmKKO3gqnDQ8mLbpABZgRR2FVYx4ouI=
|
||||
@@ -329,23 +328,23 @@ go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdH
|
||||
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4=
|
||||
golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
|
||||
golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
|
||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
|
||||
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -356,14 +355,14 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -381,18 +380,17 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
|
||||
golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -401,8 +399,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
|
||||
golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
|
||||
|
||||
@@ -1 +1 @@
|
||||
v5.0.0-beta-2504211518
|
||||
v5.0.0-beta-2504212133
|
||||
|
||||
@@ -56,7 +56,7 @@ func WithLLMService(service LLMServiceType) AIServiceOption {
|
||||
return func(opts *AIServices) {
|
||||
if service == LLMServiceTypeGPT4o {
|
||||
var err error
|
||||
opts.ILLMService, err = NewGPT4oLLMService()
|
||||
opts.ILLMService, err = NewPlanner(context.Background())
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init gpt-4o llm service failed")
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
@@ -64,9 +64,10 @@ func WithLLMService(service LLMServiceType) AIServiceOption {
|
||||
}
|
||||
if service == LLMServiceTypeUITARS {
|
||||
var err error
|
||||
opts.ILLMService, err = NewPlanner(context.Background())
|
||||
opts.ILLMService, err = NewUITarsPlanner(context.Background())
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("init ui-tars llm service failed")
|
||||
os.Exit(code.GetErrorCode(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
211
uixt/ai/llm.go
211
uixt/ai/llm.go
@@ -1,211 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
|
||||
type ILLMService interface {
|
||||
Call(opts *PlanningOptions) (*PlanningResult, error)
|
||||
}
|
||||
|
||||
func NewGPT4oLLMService() (*openaiLLMService, error) {
|
||||
return &openaiLLMService{}, nil
|
||||
}
|
||||
|
||||
type openaiLLMService struct{}
|
||||
|
||||
func (s openaiLLMService) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// PlanningOptions represents the input options for planning
|
||||
type PlanningOptions struct {
|
||||
UserInstruction string `json:"user_instruction"` // append to system prompt
|
||||
Message *schema.Message `json:"message"`
|
||||
Size types.Size `json:"size"`
|
||||
}
|
||||
|
||||
// PlanningResult represents the result of planning
|
||||
type PlanningResult struct {
|
||||
NextActions []ParsedAction `json:"actions"`
|
||||
ActionSummary string `json:"summary"`
|
||||
}
|
||||
|
||||
// VLMResponse represents the response from the Vision Language Model
|
||||
type VLMResponse struct {
|
||||
Actions []ParsedAction `json:"actions"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ParsedAction represents a parsed action from the VLM response
|
||||
type ParsedAction struct {
|
||||
ActionType ActionType `json:"actionType"`
|
||||
ActionInputs map[string]interface{} `json:"actionInputs"`
|
||||
Thought string `json:"thought"`
|
||||
}
|
||||
|
||||
type ActionType string
|
||||
|
||||
const (
|
||||
ActionTypeClick ActionType = "click"
|
||||
ActionTypeTap ActionType = "tap"
|
||||
ActionTypeDrag ActionType = "drag"
|
||||
ActionTypeSwipe ActionType = "swipe"
|
||||
ActionTypeWait ActionType = "wait"
|
||||
ActionTypeFinished ActionType = "finished"
|
||||
ActionTypeCallUser ActionType = "call_user"
|
||||
ActionTypeType ActionType = "type"
|
||||
ActionTypeScroll ActionType = "scroll"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
type OpenAIInitConfig struct {
|
||||
ReportURL string `json:"REPORT_SERVER_URL"`
|
||||
Headers map[string]string `json:"defaultHeaders"`
|
||||
}
|
||||
|
||||
const (
|
||||
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
|
||||
EnvOpenAIAPIKey = "OPENAI_API_KEY"
|
||||
EnvModelName = "LLM_MODEL_NAME"
|
||||
EnvOpenAIInitConfigJSON = "OPENAI_INIT_CONFIG_JSON"
|
||||
)
|
||||
|
||||
func checkEnvLLM() error {
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
openaiBaseURL := os.Getenv("OPENAI_BASE_URL")
|
||||
if openaiBaseURL == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "OPENAI_BASE_URL missed")
|
||||
}
|
||||
log.Info().Str("OPENAI_BASE_URL", openaiBaseURL).Msg("get env")
|
||||
openaiAPIKey := os.Getenv("OPENAI_API_KEY")
|
||||
if openaiAPIKey == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "OPENAI_API_KEY missed")
|
||||
}
|
||||
log.Info().Str("OPENAI_API_KEY", maskAPIKey(openaiAPIKey)).Msg("get env")
|
||||
modelName := os.Getenv("LLM_MODEL_NAME")
|
||||
if modelName == "" {
|
||||
return errors.Wrap(code.LLMEnvMissedError, "LLM_MODEL_NAME missed")
|
||||
}
|
||||
log.Info().Str("LLM_MODEL_NAME", modelName).Msg("get env")
|
||||
return nil
|
||||
}
|
||||
|
||||
// CustomTransport is a custom RoundTripper that adds headers to every request
|
||||
type CustomTransport struct {
|
||||
Transport http.RoundTripper
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
// RoundTrip executes a single HTTP transaction and adds custom headers
|
||||
func (c *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
for key, value := range c.Headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
return c.Transport.RoundTrip(req)
|
||||
}
|
||||
|
||||
type OutputFormat struct {
|
||||
Thought string `json:"thought"`
|
||||
Action string `json:"action"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// GetModelConfig get OpenAI config
|
||||
func GetModelConfig() (*openai.ChatModelConfig, error) {
|
||||
if err := checkEnvLLM(); err != nil {
|
||||
log.Error().Err(err).Msg("check LLM env failed")
|
||||
return nil, err
|
||||
}
|
||||
envConfig := &OpenAIInitConfig{
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// read from JSON config first
|
||||
jsonStr := config.GetEnvConfig(EnvOpenAIInitConfigJSON)
|
||||
if jsonStr != "" {
|
||||
if err := json.Unmarshal([]byte(jsonStr), envConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
|
||||
// if err != nil {
|
||||
// log.Fatal().Err(err).Msg("NewSchemaRefForValue failed")
|
||||
// }
|
||||
|
||||
modelConfig := &openai.ChatModelConfig{
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: defaultTimeout,
|
||||
Transport: &CustomTransport{
|
||||
Transport: http.DefaultTransport,
|
||||
Headers: envConfig.Headers,
|
||||
},
|
||||
},
|
||||
// TODO: set structured response format
|
||||
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
|
||||
// ResponseFormat: &openai2.ChatCompletionResponseFormat{
|
||||
// Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
|
||||
// JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
|
||||
// Name: "thought_and_action",
|
||||
// Description: "data that describes planning thought and action",
|
||||
// Schema: outputFormatSchema.Value,
|
||||
// Strict: false,
|
||||
// },
|
||||
// },
|
||||
}
|
||||
|
||||
if baseURL := config.GetEnvConfig(EnvOpenAIBaseURL); baseURL != "" {
|
||||
modelConfig.BaseURL = baseURL
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvOpenAIBaseURL)
|
||||
}
|
||||
|
||||
if apiKey := config.GetEnvConfig(EnvOpenAIAPIKey); apiKey != "" {
|
||||
modelConfig.APIKey = apiKey
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvOpenAIAPIKey)
|
||||
}
|
||||
|
||||
if modelName := config.GetEnvConfig(EnvModelName); modelName != "" {
|
||||
modelConfig.Model = modelName
|
||||
} else {
|
||||
return nil, fmt.Errorf("miss env %s", EnvModelName)
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return modelConfig, nil
|
||||
}
|
||||
|
||||
// maskAPIKey masks the API key
|
||||
func maskAPIKey(key string) string {
|
||||
if len(key) <= 8 {
|
||||
return "******"
|
||||
}
|
||||
|
||||
return key[:4] + "******" + key[len(key)-4:]
|
||||
}
|
||||
@@ -1,267 +0,0 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// NewActionParser creates a new ActionParser instance
|
||||
func NewActionParser(factor float64) *ActionParser {
|
||||
return &ActionParser{
|
||||
Factor: factor,
|
||||
}
|
||||
}
|
||||
|
||||
// ActionParser parses VLM responses and converts them to structured actions
|
||||
type ActionParser struct {
|
||||
Factor float64 // TODO
|
||||
}
|
||||
|
||||
// Parse parses the prediction text and extracts actions
|
||||
func (p *ActionParser) Parse(predictionText string) ([]ParsedAction, error) {
|
||||
// try parsing JSON format, from VLM like openai/gpt-4o
|
||||
jsonActions, jsonErr := p.parseJSON(predictionText)
|
||||
if jsonErr == nil {
|
||||
return jsonActions, nil
|
||||
}
|
||||
|
||||
// json parsing failed, try parsing Thought/Action format, from VLM like UI-TARS
|
||||
thoughtActions, thoughtErr := p.parseThoughtAction(predictionText)
|
||||
if thoughtErr == nil {
|
||||
return thoughtActions, nil
|
||||
}
|
||||
|
||||
return nil, errors.Wrap(thoughtErr, "parse planner response failed")
|
||||
}
|
||||
|
||||
// parseJSON tries to parse the response as JSON format
|
||||
func (p *ActionParser) parseJSON(predictionText string) ([]ParsedAction, error) {
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
|
||||
predictionText = strings.TrimPrefix(predictionText, "```json")
|
||||
predictionText = strings.TrimSuffix(predictionText, "```")
|
||||
}
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
|
||||
var response VLMResponse
|
||||
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
|
||||
if len(response.Actions) == 0 {
|
||||
return nil, errors.New("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// normalize actions
|
||||
var normalizedActions []ParsedAction
|
||||
for i := range response.Actions {
|
||||
// create a new variable, avoid implicit memory aliasing in for loop.
|
||||
action := response.Actions[i]
|
||||
if err := p.normalizeAction(&action); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to normalize action")
|
||||
}
|
||||
normalizedActions = append(normalizedActions, action)
|
||||
}
|
||||
|
||||
return normalizedActions, nil
|
||||
}
|
||||
|
||||
// parseThoughtAction parses the Thought/Action format response
|
||||
func (p *ActionParser) parseThoughtAction(predictionText string) ([]ParsedAction, error) {
|
||||
thoughtRegex := regexp.MustCompile(`(?is)Thought:(.+?)Action:`)
|
||||
actionRegex := regexp.MustCompile(`(?is)Action:(.+)`)
|
||||
|
||||
// extract Thought part
|
||||
thoughtMatch := thoughtRegex.FindStringSubmatch(predictionText)
|
||||
var thought string
|
||||
if len(thoughtMatch) > 1 {
|
||||
thought = strings.TrimSpace(thoughtMatch[1])
|
||||
}
|
||||
|
||||
// extract Action part, e.g. "click(start_box='(552,454)')"
|
||||
actionMatch := actionRegex.FindStringSubmatch(predictionText)
|
||||
if len(actionMatch) < 2 {
|
||||
return nil, errors.New("no action found in the response")
|
||||
}
|
||||
|
||||
actionsText := strings.TrimSpace(actionMatch[1])
|
||||
|
||||
// parse action type and parameters
|
||||
return p.parseActionText(actionsText, thought)
|
||||
}
|
||||
|
||||
// parseActionText parses the action text to extract the action type and parameters
|
||||
func (p *ActionParser) parseActionText(actionsText, thought string) ([]ParsedAction, error) {
|
||||
// remove trailing comments
|
||||
if idx := strings.Index(actionsText, "#"); idx > 0 {
|
||||
actionsText = strings.TrimSpace(actionsText[:idx])
|
||||
}
|
||||
|
||||
// supported action types and regexes
|
||||
actionRegexes := map[ActionType]*regexp.Regexp{
|
||||
"click": regexp.MustCompile(`click\(start_box='([^']+)'\)`),
|
||||
"left_double": regexp.MustCompile(`left_double\(start_box='([^']+)'\)`),
|
||||
"right_single": regexp.MustCompile(`right_single\(start_box='([^']+)'\)`),
|
||||
"drag": regexp.MustCompile(`drag\(start_box='([^']+)', end_box='([^']+)'\)`),
|
||||
"type": regexp.MustCompile(`type\(content='([^']+)'\)`),
|
||||
"scroll": regexp.MustCompile(`scroll\(start_box='([^']+)', direction='([^']+)'\)`),
|
||||
"wait": regexp.MustCompile(`wait\(\)`),
|
||||
"finished": regexp.MustCompile(`finished\(content='([^']+)'\)`),
|
||||
"call_user": regexp.MustCompile(`call_user\(\)`),
|
||||
}
|
||||
|
||||
// one or multiple actions, separated by newline
|
||||
// "click(start_box='<bbox>229 379 229 379</bbox>')
|
||||
// "click(start_box='<bbox>229 379 229 379</bbox>')\n\nclick(start_box='<bbox>769 519 769 519</bbox>')"
|
||||
parsedActions := make([]ParsedAction, 0)
|
||||
for _, actionText := range strings.Split(actionsText, "\n") {
|
||||
actionText = strings.TrimSpace(actionText)
|
||||
for actionType, regex := range actionRegexes {
|
||||
matches := regex.FindStringSubmatch(actionText)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var action ParsedAction
|
||||
action.ActionType = actionType
|
||||
action.ActionInputs = make(map[string]interface{})
|
||||
action.Thought = thought
|
||||
|
||||
// parse parameters based on action type
|
||||
switch actionType {
|
||||
case ActionTypeClick:
|
||||
if len(matches) > 1 {
|
||||
coord, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize point failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = coord
|
||||
}
|
||||
case ActionTypeDrag:
|
||||
if len(matches) > 2 {
|
||||
// handle start point
|
||||
startBox, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = startBox
|
||||
|
||||
// handle end point
|
||||
endBox, err := p.normalizeCoordinates(matches[2])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize endBox failed: %s", matches[2])
|
||||
}
|
||||
action.ActionInputs["endBox"] = endBox
|
||||
}
|
||||
case ActionTypeType:
|
||||
if len(matches) > 1 {
|
||||
action.ActionInputs["content"] = matches[1]
|
||||
}
|
||||
case ActionTypeScroll:
|
||||
if len(matches) > 2 {
|
||||
startBox, err := p.normalizeCoordinates(matches[1])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "normalize startBox failed: %s", matches[1])
|
||||
}
|
||||
action.ActionInputs["startBox"] = startBox
|
||||
action.ActionInputs["direction"] = matches[2]
|
||||
}
|
||||
case ActionTypeWait, ActionTypeFinished, ActionTypeCallUser:
|
||||
// 这些动作没有额外参数
|
||||
}
|
||||
|
||||
parsedActions = append(parsedActions, action)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parsedActions) == 0 {
|
||||
return nil, fmt.Errorf("no valid actions returned from VLM")
|
||||
}
|
||||
return parsedActions, nil
|
||||
}
|
||||
|
||||
// normalizeAction normalizes the coordinates in the action
|
||||
func (p *ActionParser) normalizeAction(action *ParsedAction) error {
|
||||
switch action.ActionType {
|
||||
case "click", "drag":
|
||||
// handle click and drag action coordinates
|
||||
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
|
||||
normalized, err := p.normalizeCoordinates(startBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize startBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["startBox"] = normalized
|
||||
}
|
||||
|
||||
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
|
||||
normalized, err := p.normalizeCoordinates(endBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize endBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["endBox"] = normalized
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeCoordinates normalizes the coordinates based on the factor
|
||||
func (p *ActionParser) normalizeCoordinates(coordStr string) (coords []float64, err error) {
|
||||
// check empty string
|
||||
if coordStr == "" {
|
||||
return nil, fmt.Errorf("empty coordinate string")
|
||||
}
|
||||
|
||||
// handle BBox format: <bbox>x1 y1 x2 y2</bbox>
|
||||
bboxRegex := regexp.MustCompile(`<bbox>(\d+\s+\d+\s+\d+\s+\d+)</bbox>`)
|
||||
bboxMatches := bboxRegex.FindStringSubmatch(coordStr)
|
||||
if len(bboxMatches) > 1 {
|
||||
// Extract space-separated values from inside the bbox tags
|
||||
bboxContent := bboxMatches[1]
|
||||
// Split by whitespace
|
||||
parts := strings.Fields(bboxContent)
|
||||
if len(parts) == 4 {
|
||||
coords = make([]float64, 4)
|
||||
for i, part := range parts {
|
||||
val, e := strconv.ParseFloat(part, 64)
|
||||
if e != nil {
|
||||
return nil, fmt.Errorf("failed to parse coordinate value '%s': %w", part, e)
|
||||
}
|
||||
coords[i] = val
|
||||
}
|
||||
// 将 val 转换为 [x,y] 坐标
|
||||
x := (coords[0] + coords[2]) / 2
|
||||
y := (coords[1] + coords[3]) / 2
|
||||
return []float64{x, y}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// handle coordinate string, e.g. "[100, 200]", "(100, 200)"
|
||||
if strings.Contains(coordStr, ",") {
|
||||
// remove possible brackets and split coordinates
|
||||
coordStr = strings.Trim(coordStr, "[]() \t")
|
||||
|
||||
// try parsing JSON array
|
||||
jsonStr := coordStr
|
||||
if !strings.HasPrefix(jsonStr, "[") {
|
||||
jsonStr = "[" + coordStr + "]"
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(jsonStr), &coords)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse coordinate string: %w", err)
|
||||
}
|
||||
return coords, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("invalid coordinate string format: %s", coordStr)
|
||||
}
|
||||
@@ -2,26 +2,64 @@ package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
_ "image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type ILLMService interface {
|
||||
Call(opts *PlanningOptions) (*PlanningResult, error)
|
||||
}
|
||||
|
||||
// PlanningOptions represents the input options for planning
|
||||
type PlanningOptions struct {
|
||||
UserInstruction string `json:"user_instruction"` // append to system prompt
|
||||
Message *schema.Message `json:"message"`
|
||||
Size types.Size `json:"size"`
|
||||
}
|
||||
|
||||
// PlanningResult represents the result of planning
|
||||
type PlanningResult struct {
|
||||
NextActions []ParsedAction `json:"actions"`
|
||||
ActionSummary string `json:"summary"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// ParsedAction represents a parsed action from the VLM response
|
||||
type ParsedAction struct {
|
||||
ActionType ActionType `json:"actionType"`
|
||||
ActionInputs map[string]interface{} `json:"actionInputs"`
|
||||
Thought string `json:"thought"`
|
||||
}
|
||||
|
||||
type ActionType string
|
||||
|
||||
const (
|
||||
ActionTypeClick ActionType = "click"
|
||||
ActionTypeTap ActionType = "tap"
|
||||
ActionTypeDrag ActionType = "drag"
|
||||
ActionTypeSwipe ActionType = "swipe"
|
||||
ActionTypeWait ActionType = "wait"
|
||||
ActionTypeFinished ActionType = "finished"
|
||||
ActionTypeCallUser ActionType = "call_user"
|
||||
ActionTypeType ActionType = "type"
|
||||
ActionTypeScroll ActionType = "scroll"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
// Error types
|
||||
var (
|
||||
ErrEmptyInstruction = fmt.Errorf("user instruction is empty")
|
||||
@@ -29,81 +67,6 @@ var (
|
||||
ErrInvalidImageData = fmt.Errorf("invalid image data")
|
||||
)
|
||||
|
||||
func NewPlanner(ctx context.Context) (*Planner, error) {
|
||||
config, err := GetModelConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
|
||||
}
|
||||
model, err := openai.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
|
||||
}
|
||||
parser := NewActionParser(1000)
|
||||
return &Planner{
|
||||
ctx: ctx,
|
||||
config: config,
|
||||
model: model,
|
||||
systemPrompt: uiTarsPlanningPrompt,
|
||||
parser: parser,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Planner struct {
|
||||
ctx context.Context
|
||||
model model.ChatModel
|
||||
config *openai.ChatModelConfig
|
||||
systemPrompt string
|
||||
parser *ActionParser
|
||||
history []*schema.Message // conversation history
|
||||
}
|
||||
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validateInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate input parameters failed")
|
||||
}
|
||||
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
|
||||
p.history = []*schema.Message{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
p.appendConversationHistory(opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", p.config.Model).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request model service failed: %w", err)
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parse result failed")
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
p.appendConversationHistory(&schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateInput(opts *PlanningOptions) error {
|
||||
if opts.UserInstruction == "" {
|
||||
return ErrEmptyInstruction
|
||||
@@ -169,7 +132,7 @@ func logResponse(resp *schema.Message) {
|
||||
}
|
||||
|
||||
// appendConversationHistory adds a message to the conversation history
|
||||
func (p *Planner) appendConversationHistory(msg *schema.Message) {
|
||||
func appendConversationHistory(history []*schema.Message, msg *schema.Message) {
|
||||
// for user image message:
|
||||
// - keep at most 4 user image messages
|
||||
// - delete the oldest user image message when the limit is reached
|
||||
@@ -179,7 +142,7 @@ func (p *Planner) appendConversationHistory(msg *schema.Message) {
|
||||
firstUserImgIndex := -1
|
||||
|
||||
// calculate the number of user messages and find the index of the first user message
|
||||
for i, item := range p.history {
|
||||
for i, item := range history {
|
||||
if item.Role == schema.User {
|
||||
userImgCount++
|
||||
if firstUserImgIndex == -1 {
|
||||
@@ -191,54 +154,34 @@ func (p *Planner) appendConversationHistory(msg *schema.Message) {
|
||||
// if there are already 4 user messages, delete the first one before adding the new message
|
||||
if userImgCount >= 4 && firstUserImgIndex >= 0 {
|
||||
// delete the first user message
|
||||
p.history = append(
|
||||
p.history[:firstUserImgIndex],
|
||||
p.history[firstUserImgIndex+1:]...,
|
||||
history = append(
|
||||
history[:firstUserImgIndex],
|
||||
history[firstUserImgIndex+1:]...,
|
||||
)
|
||||
}
|
||||
// add the new user message to the history
|
||||
p.history = append(p.history, msg)
|
||||
history = append(history, msg)
|
||||
}
|
||||
|
||||
// for assistant message:
|
||||
// - keep at most the last 10 assistant messages
|
||||
if msg.Role == schema.Assistant {
|
||||
// add the new assistant message to the history
|
||||
p.history = append(p.history, msg)
|
||||
history = append(history, msg)
|
||||
|
||||
// if there are more than 10 assistant messages, remove the oldest ones
|
||||
assistantMsgCount := 0
|
||||
for i := len(p.history) - 1; i >= 0; i-- {
|
||||
if p.history[i].Role == schema.Assistant {
|
||||
for i := len(history) - 1; i >= 0; i-- {
|
||||
if history[i].Role == schema.Assistant {
|
||||
assistantMsgCount++
|
||||
if assistantMsgCount > 10 {
|
||||
p.history = append(p.history[:i], p.history[i+1:]...)
|
||||
history = append(history[:i], history[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse response
|
||||
parseActions, err := p.parser.Parse(msg.Content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse actions: %w", err)
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// SavePositionImg saves an image with position markers
|
||||
func SavePositionImg(params struct {
|
||||
InputImgBase64 string
|
||||
@@ -336,3 +279,12 @@ func loadImage(imagePath string) (base64Str string, size types.Size, err error)
|
||||
|
||||
return base64Str, size, nil
|
||||
}
|
||||
|
||||
// maskAPIKey masks the API key
|
||||
func maskAPIKey(key string) string {
|
||||
if len(key) <= 8 {
|
||||
return "******"
|
||||
}
|
||||
|
||||
return key[:4] + "******" + key[len(key)-4:]
|
||||
}
|
||||
|
||||
242
uixt/ai/planner_gpt.go
Normal file
242
uixt/ai/planner_gpt.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
_ "image/jpeg"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
openai2 "github.com/cloudwego/eino-ext/libs/acl/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/getkin/kin-openapi/openapi3gen"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
)
|
||||
|
||||
const (
|
||||
EnvOpenAIBaseURL = "OPENAI_BASE_URL"
|
||||
EnvOpenAIAPIKey = "OPENAI_API_KEY"
|
||||
EnvModelName = "LLM_MODEL_NAME"
|
||||
)
|
||||
|
||||
// GetOpenAIModelConfig get OpenAI config
|
||||
func GetOpenAIModelConfig() (*openai.ChatModelConfig, error) {
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return nil, errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
|
||||
openaiBaseURL := os.Getenv(EnvOpenAIBaseURL)
|
||||
if openaiBaseURL == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIBaseURL)
|
||||
}
|
||||
openaiAPIKey := os.Getenv(EnvOpenAIAPIKey)
|
||||
if openaiAPIKey == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvOpenAIAPIKey)
|
||||
}
|
||||
modelName := os.Getenv(EnvModelName)
|
||||
if modelName == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvModelName)
|
||||
}
|
||||
|
||||
type OutputFormat struct {
|
||||
Thought string `json:"thought"`
|
||||
Action string `json:"action"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modelConfig := &openai.ChatModelConfig{
|
||||
BaseURL: openaiBaseURL,
|
||||
APIKey: openaiAPIKey,
|
||||
Model: modelName,
|
||||
Timeout: defaultTimeout,
|
||||
// set structured response format
|
||||
// https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go
|
||||
ResponseFormat: &openai2.ChatCompletionResponseFormat{
|
||||
Type: openai2.ChatCompletionResponseFormatTypeJSONSchema,
|
||||
JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{
|
||||
Name: "thought_and_action",
|
||||
Description: "data that describes planning thought and action",
|
||||
Schema: outputFormatSchema.Value,
|
||||
Strict: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return modelConfig, nil
|
||||
}
|
||||
|
||||
func NewPlanner(ctx context.Context) (*Planner, error) {
|
||||
config, err := GetOpenAIModelConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create OpenAI config: %w", err)
|
||||
}
|
||||
model, err := openai.NewChatModel(ctx, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OpenAI model: %w", err)
|
||||
}
|
||||
return &Planner{
|
||||
ctx: ctx,
|
||||
config: config,
|
||||
model: model,
|
||||
systemPrompt: uiTarsPlanningPrompt, // TODO: change prompt with function calling
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Planner struct {
|
||||
ctx context.Context
|
||||
model model.ToolCallingChatModel
|
||||
config *openai.ChatModelConfig
|
||||
systemPrompt string
|
||||
history []*schema.Message // conversation history
|
||||
}
|
||||
|
||||
// Call performs UI planning using Vision Language Model
|
||||
func (p *Planner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
// validate input parameters
|
||||
if err := validateInput(opts); err != nil {
|
||||
return nil, errors.Wrap(err, "validate input parameters failed")
|
||||
}
|
||||
|
||||
// prepare prompt
|
||||
if len(p.history) == 0 {
|
||||
// add system message
|
||||
systemPrompt := uiTarsPlanningPrompt + opts.UserInstruction
|
||||
p.history = []*schema.Message{
|
||||
{
|
||||
Role: schema.System,
|
||||
Content: systemPrompt,
|
||||
},
|
||||
}
|
||||
}
|
||||
// append user image message
|
||||
appendConversationHistory(p.history, opts.Message)
|
||||
|
||||
// call model service, generate response
|
||||
logRequest(p.history)
|
||||
startTime := time.Now()
|
||||
resp, err := p.model.Generate(p.ctx, p.history)
|
||||
log.Info().Float64("elapsed(s)", time.Since(startTime).Seconds()).
|
||||
Str("model", p.config.Model).Msg("call model service")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request model service failed: %w", err)
|
||||
}
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parse result failed")
|
||||
}
|
||||
|
||||
// append assistant message
|
||||
appendConversationHistory(p.history, &schema.Message{
|
||||
Role: schema.Assistant,
|
||||
Content: result.ActionSummary,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *Planner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse JSON format, from VLM like openai/gpt-4o
|
||||
parseActions, jsonErr := parseJSON(msg.Content)
|
||||
if jsonErr != nil {
|
||||
return nil, jsonErr
|
||||
}
|
||||
|
||||
// process response
|
||||
result, err := processVLMResponse(parseActions, size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "process VLM response failed")
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Interface("summary", result.ActionSummary).
|
||||
Interface("actions", result.NextActions).
|
||||
Msg("get VLM planning result")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// parseJSON tries to parse the response as JSON format
|
||||
func parseJSON(predictionText string) ([]ParsedAction, error) {
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
if strings.HasPrefix(predictionText, "```json") && strings.HasSuffix(predictionText, "```") {
|
||||
predictionText = strings.TrimPrefix(predictionText, "```json")
|
||||
predictionText = strings.TrimSuffix(predictionText, "```")
|
||||
}
|
||||
predictionText = strings.TrimSpace(predictionText)
|
||||
|
||||
var response PlanningResult
|
||||
if err := json.Unmarshal([]byte(predictionText), &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse VLM response: %v", err)
|
||||
}
|
||||
|
||||
if response.Error != "" {
|
||||
return nil, errors.New(response.Error)
|
||||
}
|
||||
|
||||
if len(response.NextActions) == 0 {
|
||||
return nil, errors.New("no actions returned from VLM")
|
||||
}
|
||||
|
||||
// normalize actions
|
||||
var normalizedActions []ParsedAction
|
||||
for i := range response.NextActions {
|
||||
// create a new variable, avoid implicit memory aliasing in for loop.
|
||||
action := response.NextActions[i]
|
||||
if err := normalizeAction(&action); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to normalize action")
|
||||
}
|
||||
normalizedActions = append(normalizedActions, action)
|
||||
}
|
||||
|
||||
return normalizedActions, nil
|
||||
}
|
||||
|
||||
// normalizeAction normalizes the coordinates in the action
|
||||
func normalizeAction(action *ParsedAction) error {
|
||||
switch action.ActionType {
|
||||
case "click", "drag":
|
||||
// handle click and drag action coordinates
|
||||
if startBox, ok := action.ActionInputs["startBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(startBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize startBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["startBox"] = normalized
|
||||
}
|
||||
|
||||
if endBox, ok := action.ActionInputs["endBox"].(string); ok {
|
||||
normalized, err := normalizeCoordinates(endBox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize endBox: %w", err)
|
||||
}
|
||||
action.ActionInputs["endBox"] = normalized
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func TestVLMPlanning(t *testing.T) {
|
||||
|
||||
userInstruction += "\n\n请基于以上游戏规则，给出下一步可点击的两个图标坐标"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -98,7 +98,7 @@ func TestXHSPlanning(t *testing.T) {
|
||||
|
||||
userInstruction := "点击第二个帖子的作者头像"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
@@ -168,7 +168,7 @@ func TestChatList(t *testing.T) {
|
||||
|
||||
userInstruction := "请结合图片的文字信息，请告诉我一共有多少个群聊，哪些群聊右下角有绿点"
|
||||
|
||||
planner, err := NewPlanner(context.Background())
|
||||
planner, err := NewUITarsPlanner(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &PlanningOptions{
|
||||
|
||||
@@ -13,17 +13,52 @@ import (
|
||||
"github.com/cloudwego/eino-ext/components/model/ark"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
EnvArkBaseURL = "ARK_BASE_URL"
|
||||
EnvArkAPIKey = "ARK_API_KEY"
|
||||
EnvArkModelID = "ARK_MODEL_ID"
|
||||
)
|
||||
|
||||
func GetArkModelConfig() (*ark.ChatModelConfig, error) {
|
||||
return &ark.ChatModelConfig{
|
||||
APIKey: os.Getenv("ARK_API_KEY"),
|
||||
Model: os.Getenv("ARK_MODEL_ID"),
|
||||
}, nil
|
||||
if err := config.LoadEnv(); err != nil {
|
||||
return nil, errors.Wrap(code.LoadEnvError, err.Error())
|
||||
}
|
||||
|
||||
arkBaseURL := os.Getenv(EnvArkBaseURL)
|
||||
arkAPIKey := os.Getenv(EnvArkAPIKey)
|
||||
if arkAPIKey == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvArkAPIKey)
|
||||
}
|
||||
modelName := os.Getenv(EnvArkModelID)
|
||||
if modelName == "" {
|
||||
return nil, errors.Wrapf(code.LLMEnvMissedError,
|
||||
"env %s missed", EnvArkModelID)
|
||||
}
|
||||
timeout := defaultTimeout
|
||||
modelConfig := &ark.ChatModelConfig{
|
||||
BaseURL: arkBaseURL,
|
||||
APIKey: arkAPIKey,
|
||||
Model: modelName,
|
||||
Timeout: &timeout,
|
||||
}
|
||||
|
||||
// log config info
|
||||
log.Info().Str("model", modelConfig.Model).
|
||||
Str("baseURL", modelConfig.BaseURL).
|
||||
Str("apiKey", maskAPIKey(modelConfig.APIKey)).
|
||||
Str("timeout", defaultTimeout.String()).
|
||||
Msg("get model config")
|
||||
|
||||
return modelConfig, nil
|
||||
}
|
||||
|
||||
func NewUITarsPlanner(ctx context.Context) (*UITarsPlanner, error) {
|
||||
@@ -113,7 +148,7 @@ func (p *UITarsPlanner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
logResponse(resp)
|
||||
|
||||
// parse result
|
||||
result, err := parseResult(resp, opts.Size)
|
||||
result, err := p.parseResult(resp, opts.Size)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parse result failed")
|
||||
}
|
||||
@@ -127,58 +162,7 @@ func (p *UITarsPlanner) Call(opts *PlanningOptions) (*PlanningResult, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// appendConversationHistory adds a message to the conversation history
|
||||
func appendConversationHistory(history []*schema.Message, msg *schema.Message) {
|
||||
// for user image message:
|
||||
// - keep at most 4 user image messages
|
||||
// - delete the oldest user image message when the limit is reached
|
||||
if msg.Role == schema.User {
|
||||
// get all existing user messages
|
||||
userImgCount := 0
|
||||
firstUserImgIndex := -1
|
||||
|
||||
// calculate the number of user messages and find the index of the first user message
|
||||
for i, item := range history {
|
||||
if item.Role == schema.User {
|
||||
userImgCount++
|
||||
if firstUserImgIndex == -1 {
|
||||
firstUserImgIndex = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if there are already 4 user messages, delete the first one before adding the new message
|
||||
if userImgCount >= 4 && firstUserImgIndex >= 0 {
|
||||
// delete the first user message
|
||||
history = append(
|
||||
history[:firstUserImgIndex],
|
||||
history[firstUserImgIndex+1:]...,
|
||||
)
|
||||
}
|
||||
// add the new user message to the history
|
||||
history = append(history, msg)
|
||||
}
|
||||
|
||||
// for assistant message:
|
||||
// - keep at most the last 10 assistant messages
|
||||
if msg.Role == schema.Assistant {
|
||||
// add the new assistant message to the history
|
||||
history = append(history, msg)
|
||||
|
||||
// if there are more than 10 assistant messages, remove the oldest ones
|
||||
assistantMsgCount := 0
|
||||
for i := len(history) - 1; i >= 0; i-- {
|
||||
if history[i].Role == schema.Assistant {
|
||||
assistantMsgCount++
|
||||
if assistantMsgCount > 10 {
|
||||
history = append(history[:i], history[i+1:]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
func (p *UITarsPlanner) parseResult(msg *schema.Message, size types.Size) (*PlanningResult, error) {
|
||||
// parse Thought/Action format from UI-TARS
|
||||
parseActions, thoughtErr := parseThoughtAction(msg.Content)
|
||||
if thoughtErr != nil {
|
||||
Reference in New Issue
Block a user