feat: 实现 AIQuery 功能并支持 OutputSchema

- 新增 AIQuery 方法到 StepMobile,支持使用自然语言从屏幕中提取信息
- 实现 AIQuery 在 driver_ext_ai.go 中的完整功能,包括屏幕截图和 LLM 查询
- 添加 OutputSchema 支持,允许用户定义自定义输出格式进行结构化查询
- 新增 ToolAIQuery MCP 工具,完整集成到 MCP 服务器中
- 在 ActionOptions 中添加 OutputSchema 字段和 WithOutputSchema 选项函数
- 添加 ACTION_Query 的配置支持和字段映射
- 完善测试覆盖:
  * 添加 TestAIQuery 单元测试,包含多种 OutputSchema 使用场景
  * 添加 TestToolAIQuery MCP 工具测试
  * 定义 GameInfo、UIElementInfo 等结构体用于测试
- 更新文档:
  * 在 docs/uixt/ai.md 中添加完整的 AIQuery 使用指南
  * 包含基本用法、OutputSchema 示例、最佳实践等
- 支持复杂的嵌套结构体和数组类型的 OutputSchema
- 与现有 AIAction、AIAssert 功能保持一致的 API 设计
This commit is contained in:
lilong.129
2025-06-13 10:27:08 +08:00
parent fb0418fa95
commit f6e7e970f8
9 changed files with 502 additions and 11 deletions
+65
View File
@@ -130,6 +130,71 @@ func (t *ToolAIAction) ConvertActionToCallToolRequest(action option.MobileAction
return mcp.CallToolRequest{}, fmt.Errorf("invalid AI action params: %v", action.Params)
}
// ToolAIQuery implements the ai_query tool call.
type ToolAIQuery struct {
// Return data fields - these define the structure of data returned by this tool
Prompt string `json:"prompt" desc:"AI query prompt that was executed"`
Result string `json:"result" desc:"Query result content"`
}
func (t *ToolAIQuery) Name() option.ActionName {
return option.ACTION_Query
}
func (t *ToolAIQuery) Description() string {
return "Query information from screen using AI vision model with natural language prompts"
}
func (t *ToolAIQuery) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_Query)
}
func (t *ToolAIQuery) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
driverExt, err := setupXTDriver(ctx, request.Params.Arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(request.Params.Arguments)
if err != nil {
return nil, err
}
// Build action options from unified request
opts := unifiedReq.Options()
// AI query logic with options
result, err := driverExt.AIQuery(unifiedReq.Prompt, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("AI query failed: %s", err.Error())), nil
}
message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt)
returnData := ToolAIQuery{
Prompt: unifiedReq.Prompt,
Result: result,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolAIQuery) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
if prompt, ok := action.Params.(string); ok {
arguments := map[string]any{
"prompt": prompt,
}
// Extract options to arguments
extractActionOptionsToArguments(action.GetOptions(), arguments)
return buildMCPCallToolRequest(t.Name(), arguments), nil
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid AI query params: %v", action.Params)
}
// ToolFinished implements the finished tool call.
type ToolFinished struct {
// Return data fields - these define the structure of data returned by this tool