refactor: merge ActionSummary and Thought fields to eliminate duplication

- Remove redundant ActionSummary field from PlanningResult struct
- Update parsers to use unified Thought field instead of duplicate fields
- Modify chat interface to display Thought instead of ActionSummary
- Update planner logging to use thought instead of summary
- Adjust prompt templates to use thought field consistently
- Switch test LLM service from UI-TARS to DoubaoVL
- Add default parameter handling for sleep tool
This commit is contained in:
lilong.129
2025-06-05 14:19:09 +08:00
parent 0864f74021
commit 0add3231ff
8 changed files with 24 additions and 26 deletions
+1 -1
View File
@@ -1 +1 @@
v5.0.0-beta-2506051328 v5.0.0-beta-2506051419
+1 -1
View File
@@ -136,7 +136,7 @@ func (c *Chat) runPrompt(ctx context.Context, prompt string) error {
return c.handleToolCalls(ctx, toolCalls) return c.handleToolCalls(ctx, toolCalls)
} }
c.renderContent("Assistant", result.ActionSummary) c.renderContent("Assistant", result.Thought)
return nil return nil
} }
+2 -3
View File
@@ -53,7 +53,7 @@ func (p *JSONContentParser) Parse(content string, size types.Size) (*PlanningRes
// Define a temporary struct to parse the expected JSON format // Define a temporary struct to parse the expected JSON format
var jsonResponse struct { var jsonResponse struct {
Actions []Action `json:"actions"` Actions []Action `json:"actions"`
Summary string `json:"summary"` Thought string `json:"thought"`
Error string `json:"error"` Error string `json:"error"`
} }
@@ -96,8 +96,7 @@ func (p *JSONContentParser) Parse(content string, size types.Size) (*PlanningRes
return &PlanningResult{ return &PlanningResult{
ToolCalls: toolCalls, ToolCalls: toolCalls,
ActionSummary: jsonResponse.Summary, Thought: jsonResponse.Thought,
Thought: jsonResponse.Summary,
Content: content, Content: content,
}, nil }, nil
} }
-1
View File
@@ -53,7 +53,6 @@ func (p *UITARSContentParser) Parse(content string, size types.Size) (*PlanningR
return &PlanningResult{ return &PlanningResult{
ToolCalls: toolCalls, ToolCalls: toolCalls,
ActionSummary: thought,
Thought: thought, Thought: thought,
Content: content, Content: content,
}, nil }, nil
+3 -4
View File
@@ -28,7 +28,6 @@ type PlanningOptions struct {
// PlanningResult represents the result of planning // PlanningResult represents the result of planning
type PlanningResult struct { type PlanningResult struct {
ToolCalls []schema.ToolCall `json:"tool_calls"` ToolCalls []schema.ToolCall `json:"tool_calls"`
ActionSummary string `json:"summary"`
Thought string `json:"thought"` Thought string `json:"thought"`
Content string `json:"content"` // original content from model Content string `json:"content"` // original content from model
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
@@ -126,7 +125,7 @@ func (p *Planner) Call(ctx context.Context, opts *PlanningOptions) (*PlanningRes
// history will be appended with tool calls execution result // history will be appended with tool calls execution result
result := &PlanningResult{ result := &PlanningResult{
ToolCalls: message.ToolCalls, ToolCalls: message.ToolCalls,
ActionSummary: message.Content, Thought: message.Content,
} }
return result, nil return result, nil
} }
@@ -135,7 +134,7 @@ func (p *Planner) Call(ctx context.Context, opts *PlanningOptions) (*PlanningRes
result, err := p.parser.Parse(message.Content, opts.Size) result, err := p.parser.Parse(message.Content, opts.Size)
if err != nil { if err != nil {
result = &PlanningResult{ result = &PlanningResult{
ActionSummary: message.Content, Thought: message.Content,
Error: err.Error(), Error: err.Error(),
} }
log.Debug().Str("reason", err.Error()).Msg("parse content to actions failed") log.Debug().Str("reason", err.Error()).Msg("parse content to actions failed")
@@ -147,7 +146,7 @@ func (p *Planner) Call(ctx context.Context, opts *PlanningOptions) (*PlanningRes
}) })
log.Info(). log.Info().
Interface("summary", result.ActionSummary). Interface("thought", result.Thought).
Interface("tool_calls", result.ToolCalls). Interface("tool_calls", result.ToolCalls).
Msg("get VLM planning result") Msg("get VLM planning result")
return result, nil return result, nil
+3 -3
View File
@@ -98,7 +98,7 @@ Supporting actions:
Field description: Field description:
* The ` + "`start_box`" + ` and ` + "`end_box`" + ` fields represent the bounding box coordinates of the target element in 1000x1000 relative coordinate system. * The ` + "`start_box`" + ` and ` + "`end_box`" + ` fields represent the bounding box coordinates of the target element in 1000x1000 relative coordinate system.
* Use Chinese in log and summary fields. * Use Chinese in log and thought fields.
Return in JSON format: Return in JSON format:
{ {
@@ -108,7 +108,7 @@ Return in JSON format:
"action_inputs": { ... } "action_inputs": { ... }
} }
], ],
"summary": "string", // Log what the next action you can do according to the screenshot and the instruction. Use Chinese. "thought": "string", // Log what the next action you can do according to the screenshot and the instruction. Use Chinese.
"error": "string" | null, // Error messages about unexpected situations, if any. Use Chinese. "error": "string" | null, // Error messages about unexpected situations, if any. Use Chinese.
} }
@@ -123,7 +123,7 @@ For example, when the instruction is "点击第二个帖子的作者头像", by
} }
} }
], ],
"summary": "点击第二个帖子的作者头像", "thought": "点击第二个帖子的作者头像",
"error": null "error": null
} }
+1 -1
View File
@@ -25,7 +25,7 @@ func setupADBDriverExt(t *testing.T) *XTDriver {
require.Nil(t, err) require.Nil(t, err)
driverExt, err := NewXTDriver(driver, driverExt, err := NewXTDriver(driver,
option.WithCVService(option.CVServiceTypeVEDEM), option.WithCVService(option.CVServiceTypeVEDEM),
option.WithLLMService(option.LLMServiceTypeUITARS), option.WithLLMService(option.LLMServiceTypeDoubaoVL),
) )
require.Nil(t, err) require.Nil(t, err)
return driverExt return driverExt
+2 -1
View File
@@ -34,7 +34,8 @@ func (t *ToolSleep) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
seconds, ok := request.Params.Arguments["seconds"] seconds, ok := request.Params.Arguments["seconds"]
if !ok { if !ok {
return nil, fmt.Errorf("seconds parameter is required") log.Warn().Msg("seconds parameter is required, using default value 5.0 seconds")
seconds = 5.0
} }
// Sleep action logic // Sleep action logic