feat: add MCP plugin support and optimize AI service configuration

- Add UIXT runner with MCP plugin support
   - Refactor AI service options handling
   - Optimize configuration parsing for LLM and CV services
   - Update dependencies to latest versions
This commit is contained in:
lilong.129
2025-06-13 17:06:28 +08:00
parent 409cd693f0
commit b271e655b1
15 changed files with 490 additions and 259 deletions

View File

@@ -171,6 +171,7 @@ func createXTDriverWithConfig(config DriverCacheConfig) (*XTDriver, error) {
// Default AI options
aiOpts = []option.AIServiceOption{
option.WithCVService(option.CVServiceTypeVEDEM),
option.WithLLMConfig(option.RecommendedConfigurations()["ui_focused"]),
}
}

View File

@@ -316,14 +316,14 @@ type SessionData struct {
ScreenResults []*ScreenResult `json:"screen_results,omitempty"` // store sub-action specific screen_results
}
func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (string, error) {
func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.QueryResult, error) {
if dExt.LLMService == nil {
return "", errors.New("LLM service is not initialized")
return nil, errors.New("LLM service is not initialized")
}
screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize()
if err != nil {
return "", err
return nil, err
}
// parse action options to extract OutputSchema
@@ -338,10 +338,10 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (string,
}
result, err := dExt.LLMService.Query(context.Background(), queryOpts)
if err != nil {
return "", errors.Wrap(err, "AI query failed")
return nil, errors.Wrap(err, "AI query failed")
}
return result.Content, nil
return result, nil
}
func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) error {

View File

@@ -174,7 +174,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc {
message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt)
returnData := ToolAIQuery{
Prompt: unifiedReq.Prompt,
Result: result,
Result: result.Content,
}
return NewMCPSuccessResponse(message, &returnData), nil

View File

@@ -9,9 +9,23 @@ func NewAIServiceOptions(opts ...AIServiceOption) *AIServiceOptions {
}
type AIServiceOptions struct {
CVService CVServiceType
LLMService LLMServiceType
LLMConfig *LLMServiceConfig // New field for advanced LLM configuration
CVService CVServiceType `json:"cv_service,omitempty" yaml:"cv_service,omitempty"`
LLMService LLMServiceType `json:"llm_service,omitempty" yaml:"llm_service,omitempty"`
LLMConfig *LLMServiceConfig `json:"llm_config,omitempty" yaml:"llm_config,omitempty"` // advanced LLM configuration
}
func (opts *AIServiceOptions) Options() []AIServiceOption {
aiOpts := []AIServiceOption{}
if opts.CVService != "" {
aiOpts = append(aiOpts, WithCVService(opts.CVService))
}
if opts.LLMService != "" {
aiOpts = append(aiOpts, WithLLMService(opts.LLMService))
}
if opts.LLMConfig != nil {
aiOpts = append(aiOpts, WithLLMConfig(opts.LLMConfig))
}
return aiOpts
}
type AIServiceOption func(*AIServiceOptions)

View File

@@ -26,12 +26,16 @@ func NewXTDriver(driver IDriver, opts ...option.AIServiceOption) (*XTDriver, err
services := option.NewAIServiceOptions(opts...)
var err error
if services.CVService != "" {
driverExt.CVService, err = ai.NewCVService(services.CVService)
if err != nil {
log.Error().Err(err).Msg("init vedem image service failed")
return nil, err
}
// default to vedem CV service
if services.CVService == "" {
log.Warn().Msg("no CV service config provided, use default vedem")
services.CVService = option.CVServiceTypeVEDEM
}
driverExt.CVService, err = ai.NewCVService(services.CVService)
if err != nil {
log.Error().Err(err).Msg("init vedem image service failed")
return nil, err
}
// Handle LLM service initialization
@@ -47,6 +51,8 @@ func NewXTDriver(driver IDriver, opts ...option.AIServiceOption) (*XTDriver, err
if err != nil {
return nil, errors.Wrap(err, "init llm service failed")
}
} else {
log.Warn().Msg("no LLM service config provided")
}
// Register uixt MCP tools to LLM service if it exists