mirror of
https://github.com/httprunner/httprunner.git
synced 2026-09-06 07:57:27 +08:00
feat: optimize ILLMService interface to support different models for each component
- Add LLMServiceConfig to support mixed model configuration - Enable Planner, Asserter, Querier to use different optimal models - Provide recommended configurations for various use cases - Maintain backward compatibility with existing API - Update documentation to reflect current state without iteration history - Merge test files and add comprehensive configuration tests - Resolve circular dependency by moving config to option package
This commit is contained in:
@@ -1 +1 @@
|
|||||||
v5.0.0-beta-2506111115
|
v5.0.0-beta-2506111218
|
||||||
|
|||||||
+263
-701
File diff suppressed because it is too large
Load Diff
+25
-4
@@ -16,21 +16,42 @@ type ILLMService interface {
|
|||||||
RegisterTools(tools []*schema.ToolInfo) error
|
RegisterTools(tools []*schema.ToolInfo) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewLLMService creates a new LLM service with the same model for all components (backward compatibility)
|
||||||
func NewLLMService(modelType option.LLMServiceType) (ILLMService, error) {
|
func NewLLMService(modelType option.LLMServiceType) (ILLMService, error) {
|
||||||
modelConfig, err := GetModelConfig(modelType)
|
config := option.NewLLMServiceConfig(modelType)
|
||||||
|
return NewLLMServiceWithOptionConfig(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLLMServiceWithOptionConfig creates a new LLM service with different models for each component
|
||||||
|
func NewLLMServiceWithOptionConfig(config *option.LLMServiceConfig) (ILLMService, error) {
|
||||||
|
// Get model configs for each component
|
||||||
|
plannerModelConfig, err := GetModelConfig(config.PlannerModel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
planner, err := NewPlanner(context.Background(), modelConfig)
|
asserterModelConfig, err := GetModelConfig(config.AsserterModel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
asserter, err := NewAsserter(context.Background(), modelConfig)
|
|
||||||
|
querierModelConfig, err := GetModelConfig(config.QuerierModel)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
querier, err := NewQuerier(context.Background(), modelConfig)
|
|
||||||
|
// Create components with their respective model configs
|
||||||
|
planner, err := NewPlanner(context.Background(), plannerModelConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
asserter, err := NewAsserter(context.Background(), asserterModelConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
querier, err := NewQuerier(context.Background(), querierModelConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -140,3 +140,82 @@ func TestILLMServiceIntegration(t *testing.T) {
|
|||||||
// which is more complex, so we skip it in this integration test
|
// which is more complex, so we skip it in this integration test
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestLLMServiceConfig tests the LLM service configuration functionality
|
||||||
|
func TestLLMServiceConfig(t *testing.T) {
|
||||||
|
t.Run("BasicConfiguration", func(t *testing.T) {
|
||||||
|
// Test creating config with same model for all components
|
||||||
|
modelType := option.DOUBAO_1_5_THINKING_VISION_PRO_250428
|
||||||
|
config := option.NewLLMServiceConfig(modelType)
|
||||||
|
|
||||||
|
assert.Equal(t, modelType, config.PlannerModel)
|
||||||
|
assert.Equal(t, modelType, config.AsserterModel)
|
||||||
|
assert.Equal(t, modelType, config.QuerierModel)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("MixedConfiguration", func(t *testing.T) {
|
||||||
|
// Test configuring different models for each component
|
||||||
|
config := option.NewLLMServiceConfig(option.DOUBAO_1_5_THINKING_VISION_PRO_250428).
|
||||||
|
WithPlannerModel(option.DOUBAO_1_5_UI_TARS_250328).
|
||||||
|
WithAsserterModel(option.OPENAI_GPT_4O).
|
||||||
|
WithQuerierModel(option.DEEPSEEK_R1_250528)
|
||||||
|
|
||||||
|
assert.Equal(t, option.DOUBAO_1_5_UI_TARS_250328, config.PlannerModel)
|
||||||
|
assert.Equal(t, option.OPENAI_GPT_4O, config.AsserterModel)
|
||||||
|
assert.Equal(t, option.DEEPSEEK_R1_250528, config.QuerierModel)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("RecommendedConfigurations", func(t *testing.T) {
|
||||||
|
configs := option.RecommendedConfigurations()
|
||||||
|
|
||||||
|
// Test mixed optimal configuration
|
||||||
|
mixedOptimal := configs["mixed_optimal"]
|
||||||
|
assert.NotNil(t, mixedOptimal)
|
||||||
|
assert.Equal(t, option.DOUBAO_1_5_UI_TARS_250328, mixedOptimal.PlannerModel)
|
||||||
|
assert.Equal(t, option.OPENAI_GPT_4O, mixedOptimal.AsserterModel)
|
||||||
|
assert.Equal(t, option.DEEPSEEK_R1_250528, mixedOptimal.QuerierModel)
|
||||||
|
|
||||||
|
// Test high performance configuration
|
||||||
|
highPerf := configs["high_performance"]
|
||||||
|
assert.NotNil(t, highPerf)
|
||||||
|
assert.Equal(t, option.OPENAI_GPT_4O, highPerf.PlannerModel)
|
||||||
|
assert.Equal(t, option.OPENAI_GPT_4O, highPerf.AsserterModel)
|
||||||
|
assert.Equal(t, option.OPENAI_GPT_4O, highPerf.QuerierModel)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLLMServiceCreation tests service creation with different configurations
|
||||||
|
func TestLLMServiceCreation(t *testing.T) {
|
||||||
|
t.Run("BackwardCompatibility", func(t *testing.T) {
|
||||||
|
// Test that the original NewLLMService function still works
|
||||||
|
modelType := option.DOUBAO_1_5_THINKING_VISION_PRO_250428
|
||||||
|
service, err := NewLLMService(modelType)
|
||||||
|
|
||||||
|
// We expect an error due to missing environment variables in test environment
|
||||||
|
// but the function signature should be correct
|
||||||
|
if err != nil {
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Nil(t, service)
|
||||||
|
} else {
|
||||||
|
assert.NotNil(t, service)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("WithAdvancedConfig", func(t *testing.T) {
|
||||||
|
// Test the new API with different models for each component
|
||||||
|
config := option.NewLLMServiceConfig(option.DOUBAO_1_5_THINKING_VISION_PRO_250428).
|
||||||
|
WithPlannerModel(option.DOUBAO_1_5_UI_TARS_250328).
|
||||||
|
WithAsserterModel(option.OPENAI_GPT_4O)
|
||||||
|
|
||||||
|
service, err := NewLLMServiceWithOptionConfig(config)
|
||||||
|
|
||||||
|
// We expect an error due to missing environment variables in test environment
|
||||||
|
// but the function signature should be correct
|
||||||
|
if err != nil {
|
||||||
|
assert.NotNil(t, err)
|
||||||
|
assert.Nil(t, service)
|
||||||
|
} else {
|
||||||
|
assert.NotNil(t, service)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ func NewAIServiceOptions(opts ...AIServiceOption) *AIServiceOptions {
|
|||||||
type AIServiceOptions struct {
|
type AIServiceOptions struct {
|
||||||
CVService CVServiceType
|
CVService CVServiceType
|
||||||
LLMService LLMServiceType
|
LLMService LLMServiceType
|
||||||
|
LLMConfig *LLMServiceConfig // New field for advanced LLM configuration
|
||||||
}
|
}
|
||||||
|
|
||||||
type AIServiceOption func(*AIServiceOptions)
|
type AIServiceOption func(*AIServiceOptions)
|
||||||
@@ -48,3 +49,65 @@ func WithLLMService(modelType LLMServiceType) AIServiceOption {
|
|||||||
opts.LLMService = modelType
|
opts.LLMService = modelType
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LLMServiceConfig defines configuration for different LLM service components
|
||||||
|
type LLMServiceConfig struct {
|
||||||
|
PlannerModel LLMServiceType `json:"planner_model"` // Model type for planner component
|
||||||
|
AsserterModel LLMServiceType `json:"asserter_model"` // Model type for asserter component
|
||||||
|
QuerierModel LLMServiceType `json:"querier_model"` // Model type for querier component
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLLMServiceConfig creates a new LLMServiceConfig with the same model for all components
|
||||||
|
func NewLLMServiceConfig(modelType LLMServiceType) *LLMServiceConfig {
|
||||||
|
return &LLMServiceConfig{
|
||||||
|
PlannerModel: modelType,
|
||||||
|
AsserterModel: modelType,
|
||||||
|
QuerierModel: modelType,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithPlannerModel sets the model type for planner component
|
||||||
|
func (c *LLMServiceConfig) WithPlannerModel(modelType LLMServiceType) *LLMServiceConfig {
|
||||||
|
c.PlannerModel = modelType
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithAsserterModel sets the model type for asserter component
|
||||||
|
func (c *LLMServiceConfig) WithAsserterModel(modelType LLMServiceType) *LLMServiceConfig {
|
||||||
|
c.AsserterModel = modelType
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithQuerierModel sets the model type for querier component
|
||||||
|
func (c *LLMServiceConfig) WithQuerierModel(modelType LLMServiceType) *LLMServiceConfig {
|
||||||
|
c.QuerierModel = modelType
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithLLMConfig sets the advanced LLM configuration
|
||||||
|
func WithLLMConfig(config *LLMServiceConfig) AIServiceOption {
|
||||||
|
return func(opts *AIServiceOptions) {
|
||||||
|
opts.LLMConfig = config
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecommendedConfigurations provides some recommended model configurations for different use cases
|
||||||
|
func RecommendedConfigurations() map[string]*LLMServiceConfig {
|
||||||
|
return map[string]*LLMServiceConfig{
|
||||||
|
"cost_effective": NewLLMServiceConfig(DOUBAO_1_5_THINKING_VISION_PRO_250428).
|
||||||
|
WithPlannerModel(DOUBAO_1_5_UI_TARS_250328).
|
||||||
|
WithAsserterModel(DOUBAO_1_5_THINKING_VISION_PRO_250428).
|
||||||
|
WithQuerierModel(DOUBAO_1_5_THINKING_VISION_PRO_250428),
|
||||||
|
|
||||||
|
"high_performance": NewLLMServiceConfig(OPENAI_GPT_4O),
|
||||||
|
|
||||||
|
"mixed_optimal": NewLLMServiceConfig(DOUBAO_1_5_THINKING_VISION_PRO_250428).
|
||||||
|
WithPlannerModel(DOUBAO_1_5_UI_TARS_250328). // Best for UI understanding
|
||||||
|
WithAsserterModel(OPENAI_GPT_4O). // Best for reasoning
|
||||||
|
WithQuerierModel(DEEPSEEK_R1_250528), // Cost-effective for queries
|
||||||
|
|
||||||
|
"ui_focused": NewLLMServiceConfig(DOUBAO_1_5_UI_TARS_250328),
|
||||||
|
|
||||||
|
"reasoning_focused": NewLLMServiceConfig(DOUBAO_1_5_THINKING_VISION_PRO_250428),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+13
-2
@@ -33,13 +33,24 @@ func NewXTDriver(driver IDriver, opts ...option.AIServiceOption) (*XTDriver, err
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if services.LLMService != "" {
|
|
||||||
|
// Handle LLM service initialization
|
||||||
|
if services.LLMConfig != nil {
|
||||||
|
// Use advanced LLM configuration if provided
|
||||||
|
driverExt.LLMService, err = ai.NewLLMServiceWithOptionConfig(services.LLMConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "init llm service with config failed")
|
||||||
|
}
|
||||||
|
} else if services.LLMService != "" {
|
||||||
|
// Fallback to simple LLM service if no config provided
|
||||||
driverExt.LLMService, err = ai.NewLLMService(services.LLMService)
|
driverExt.LLMService, err = ai.NewLLMService(services.LLMService)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "init llm service failed")
|
return nil, errors.Wrap(err, "init llm service failed")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Register uixt MCP tools to LLM service
|
// Register uixt MCP tools to LLM service if it exists
|
||||||
|
if driverExt.LLMService != nil {
|
||||||
mcpTools := driverExt.client.Server.ListTools()
|
mcpTools := driverExt.client.Server.ListTools()
|
||||||
einoTools := ai.ConvertMCPToolsToEinoToolInfos(mcpTools, "uixt")
|
einoTools := ai.ConvertMCPToolsToEinoToolInfos(mcpTools, "uixt")
|
||||||
if err := driverExt.LLMService.RegisterTools(einoTools); err != nil {
|
if err := driverExt.LLMService.RegisterTools(einoTools); err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user