feat: optimize UI-TARS parser with coordinate conversion and action mapping

- Add action mapping for UI-TARS parser to convert action names to option.ActionName
- Implement bounding box to center point coordinate conversion for better accuracy
- Update coordinate normalization to handle coordinates > 1000 properly
- Enhance test cases to verify coordinate scaling and center point conversion
- Improve action argument processing with proper coordinate transformation
- Add comprehensive test coverage for coordinate conversion edge cases

Key improvements:
- Bounding box [x1,y1,x2,y2] now converts to center point [cx,cy] for actions
- Coordinate scaling properly handles different screen resolutions
- Action names are mapped through doubao_1_5_ui_tars_action_mapping
- Enhanced error handling for invalid coordinate formats
This commit is contained in:
lilong.129
2025-06-04 22:39:17 +08:00
parent 1df529ecaa
commit c204542f1f
10 changed files with 386 additions and 411 deletions

View File

@@ -21,18 +21,21 @@ func NewLLMContentParser(modelType option.LLMServiceType) LLMContentParser {
switch modelType {
case option.LLMServiceTypeUITARS:
return &UITARSContentParser{
systemPrompt: doubao_1_5_ui_tars_planning_prompt,
systemPrompt: doubao_1_5_ui_tars_planning_prompt,
actionMapping: doubao_1_5_ui_tars_action_mapping,
}
default:
return &JSONContentParser{
systemPrompt: defaultPlanningResponseJsonFormat,
systemPrompt: defaultPlanningResponseJsonFormat,
actionMapping: map[string]option.ActionName{},
}
}
}
// JSONContentParser parses the response as JSON string format
type JSONContentParser struct {
systemPrompt string
systemPrompt string
actionMapping map[string]option.ActionName
}
func (p *JSONContentParser) SystemPrompt() string {
@@ -83,7 +86,7 @@ func (p *JSONContentParser) Parse(content string, size types.Size) (*PlanningRes
}
// Convert actions to tool calls using function from parser_ui_tars.go
toolCalls := convertActionsToToolCalls(normalizedActions)
toolCalls := convertActionsToToolCalls(normalizedActions, p.actionMapping)
return &PlanningResult{
ToolCalls: toolCalls,