diff --git a/internal/version/VERSION b/internal/version/VERSION index 13a7f720..7aea0823 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2504292008 +v5.0.0-beta-2504292203 diff --git a/uixt/ai/ai.go b/uixt/ai/ai.go index 3d039820..e29a8931 100644 --- a/uixt/ai/ai.go +++ b/uixt/ai/ai.go @@ -51,7 +51,6 @@ const ( LLMServiceTypeGPT4o LLMServiceType = "gpt-4o" LLMServiceTypeGPT4Vision LLMServiceType = "gpt-4-vision" LLMServiceTypeQwenVL LLMServiceType = "qwen-vl" - LLMServiceTypeDeepSeekV3 LLMServiceType = "deepseek-v3" ) // ILLMService 定义了 LLM 服务接口，包括规划和断言功能 diff --git a/uixt/ai/ai_ark.go b/uixt/ai/ai_ark.go index 265d4afb..72b7a578 100644 --- a/uixt/ai/ai_ark.go +++ b/uixt/ai/ai_ark.go @@ -35,20 +35,13 @@ func GetArkModelConfig() (*ark.ChatModelConfig, error) { timeout := defaultTimeout // https://www.volcengine.com/docs/82379/1494384?redirect=1 - temperature := float32(0.01) // [0, 2] 采样温度。控制了生成文本时对每个候选词的概率分布进行平滑的程度。 - // topP := float32(0.7) // [0, 1] 核采样概率阈值。模型会考虑概率质量在 top_p 内的 token 结果。 - // maxTokens := int(4096) // 模型可以生成的最大 token 数量。输入 token 和输出 token 的总长度还受模型的上下文长度限制。 - // frequencyPenalty := float32(0) // [-2, 2] 频率惩罚系数。如果值为正，会根据新 token 在文本中的出现频率对其进行惩罚，从而降低模型逐字重复的可能性。 - + temperature := float32(0.01) modelConfig := &ark.ChatModelConfig{ BaseURL: arkBaseURL, APIKey: arkAPIKey, Model: modelName, Timeout: &timeout, Temperature: &temperature, - // TopP: &topP, - // MaxTokens: &maxTokens, - // FrequencyPenalty: &frequencyPenalty, } // log config info diff --git a/uixt/ai/ai_openai.go b/uixt/ai/ai_openai.go index 85d35ab4..e275cac1 100644 --- a/uixt/ai/ai_openai.go +++ b/uixt/ai/ai_openai.go @@ -4,8 +4,6 @@ import ( "os" "github.com/cloudwego/eino-ext/components/model/openai" - openai2 "github.com/cloudwego/eino-ext/libs/acl/openai" - "github.com/getkin/kin-openapi/openapi3gen" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/config" "github.com/pkg/errors" @@ -40,32 +38,13 @@ func GetOpenAIModelConfig() (*openai.ChatModelConfig, error) { "env %s missed", EnvModelName) } - type OutputFormat struct { - Thought string `json:"thought"` - Action string `json:"action"` - Error string `json:"error,omitempty"` - } - outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil) - if err != nil { - return nil, err - } - + temperature := float32(0.01) modelConfig := &openai.ChatModelConfig{ - BaseURL: openaiBaseURL, - APIKey: openaiAPIKey, - Model: modelName, - Timeout: defaultTimeout, - // set structured response format - // https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go - ResponseFormat: &openai2.ChatCompletionResponseFormat{ - Type: openai2.ChatCompletionResponseFormatTypeJSONSchema, - JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{ - Name: "thought_and_action", - Description: "data that describes planning thought and action", - Schema: outputFormatSchema.Value, - Strict: false, - }, - }, + BaseURL: openaiBaseURL, + APIKey: openaiAPIKey, + Model: modelName, + Timeout: defaultTimeout, + Temperature: &temperature, } // log config info diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index 9e0d628b..750f2a32 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -9,8 +9,10 @@ import ( "github.com/cloudwego/eino-ext/components/model/ark" "github.com/cloudwego/eino-ext/components/model/openai" + openai2 "github.com/cloudwego/eino-ext/libs/acl/openai" "github.com/cloudwego/eino/components/model" "github.com/cloudwego/eino/schema" + "github.com/getkin/kin-openapi/openapi3gen" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/json" "github.com/httprunner/httprunner/v5/uixt/types" @@ -50,7 +52,7 @@ func NewAsserter(ctx context.Context, modelType LLMServiceType) (*Asserter, erro asserter := &Asserter{ ctx: ctx, modelType: modelType, - systemPrompt: getAssertionSystemPrompt(modelType), + systemPrompt: defaultAssertionPrompt, } switch modelType { @@ -59,19 +61,56 @@ func NewAsserter(ctx context.Context, modelType LLMServiceType) (*Asserter, erro if err != nil { return nil, err } + asserter.systemPrompt += "\n\n" + uiTarsAssertionResponseFormat asserter.model, err = ark.NewChatModel(ctx, config) if err != nil { return nil, err } + case LLMServiceTypeGPT4Vision, LLMServiceTypeGPT4o: config, err := GetOpenAIModelConfig() if err != nil { return nil, err } + + // define output format + type OutputFormat struct { + Thought string `json:"thought"` + Pass bool `json:"pass"` + Error string `json:"error,omitempty"` + } + outputFormatSchema, err := openapi3gen.NewSchemaRefForValue(&OutputFormat{}, nil) + if err != nil { + return nil, err + } + // set structured response format + // https://github.com/cloudwego/eino-ext/blob/main/components/model/openai/examples/structured/structured.go + config.ResponseFormat = &openai2.ChatCompletionResponseFormat{ + Type: openai2.ChatCompletionResponseFormatTypeJSONSchema, + JSONSchema: &openai2.ChatCompletionResponseFormatJSONSchema{ + Name: "assertion_result", + Description: "data that describes assertion result", + Schema: outputFormatSchema.Value, + Strict: false, + }, + } + asserter.model, err = openai.NewChatModel(ctx, config) if err != nil { return nil, err } + + case LLMServiceTypeQwenVL: + config, err := GetOpenAIModelConfig() + if err != nil { + return nil, err + } + asserter.systemPrompt += "\n\n" + defaultAssertionResponseJsonFormat + asserter.model, err = openai.NewChatModel(ctx, config) + if err != nil { + return nil, err + } + default: return nil, errors.New("not supported model type for asserter") } @@ -79,14 +118,6 @@ func NewAsserter(ctx context.Context, modelType LLMServiceType) (*Asserter, erro return asserter, nil } -// getAssertionSystemPrompt returns the appropriate system prompt for the given model type -func getAssertionSystemPrompt(modelType LLMServiceType) string { - if modelType == LLMServiceTypeUITARS { - return defaultAssertionPrompt + "\n\n" + uiTarsAssertionResponseFormat - } - return defaultAssertionPrompt + "\n\n" + defaultAssertionResponseJsonFormat -} - // Assert performs the assertion check on the screenshot func (a *Asserter) Assert(opts *AssertOptions) (*AssertionResponse, error) { // Validate input parameters diff --git a/uixt/ai/asserter_test.go b/uixt/ai/asserter_test.go index 8786914b..2ab36bba 100644 --- a/uixt/ai/asserter_test.go +++ b/uixt/ai/asserter_test.go @@ -1,6 +1,7 @@ package ai import ( + "context" "testing" "github.com/httprunner/httprunner/v5/uixt/types" @@ -8,16 +9,15 @@ import ( "github.com/stretchr/testify/require" ) -func createAIService(t *testing.T) *AIServices { - aiService := NewAIService(WithLLMService(LLMServiceTypeUITARS)) - require.NotNil(t, aiService) - require.NotNil(t, aiService.ILLMService) - return aiService +func createAsserter(t *testing.T) *Asserter { + asserter, err := NewAsserter(context.Background(), LLMServiceTypeUITARS) + require.NoError(t, err) + return asserter } // 测试有效断言 func TestValidAssertions(t *testing.T) { - aiService := createAIService(t) + asserter := createAsserter(t) testCases := []struct { name string @@ -33,7 +33,7 @@ func TestValidAssertions(t *testing.T) { }, { name: "深度思考功能未开启", - assertion: "输入框下方的「深度思考」文字是灰色的", + assertion: "输入框下方的「深度思考」文字不是蓝色的", imagePath: "testdata/deepseek_think_off.png", expectPass: true, }, @@ -50,7 +50,7 @@ func TestValidAssertions(t *testing.T) { imageBase64, size, err := loadImage(tc.imagePath) require.NoError(t, err) - result, err := aiService.ILLMService.Assert(&AssertOptions{ + result, err := asserter.Assert(&AssertOptions{ Assertion: tc.assertion, Screenshot: imageBase64, Size: size, @@ -58,14 +58,13 @@ func TestValidAssertions(t *testing.T) { require.NoError(t, err) assert.NotNil(t, result) assert.Equal(t, tc.expectPass, result.Pass) - assert.NotEmpty(t, result.Thought) }) } } // 测试无效参数 func TestInvalidParameters(t *testing.T) { - aiService := createAIService(t) + asserter := createAsserter(t) testCases := []struct { name string assertion string @@ -91,7 +90,7 @@ func TestInvalidParameters(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := aiService.ILLMService.Assert(&AssertOptions{ + _, err := asserter.Assert(&AssertOptions{ Assertion: tc.assertion, Screenshot: tc.screenshot, Size: tc.size,