feat: optimize UI-TARS parser with coordinate conversion and action mapping

- Add action mapping for UI-TARS parser to convert action names to option.ActionName
- Implement bounding box to center point coordinate conversion for better accuracy
- Update coordinate normalization to handle coordinates > 1000 properly
- Enhance test cases to verify coordinate scaling and center point conversion
- Improve action argument processing with proper coordinate transformation
- Add comprehensive test coverage for coordinate conversion edge cases

Key improvements:
- Bounding box [x1,y1,x2,y2] now converts to center point [cx,cy] for actions
- Coordinate scaling properly handles different screen resolutions
- Action names are mapped through doubao_1_5_ui_tars_action_mapping
- Enhanced error handling for invalid coordinate formats
This commit is contained in:
lilong.129
2025-06-04 23:16:14 +08:00
parent 1df529ecaa
commit c204542f1f
10 changed files with 386 additions and 411 deletions
+5 -14
View File
@@ -2,7 +2,6 @@ package ai
import (
"context"
"fmt"
"time"
"github.com/cloudwego/eino-ext/components/model/openai"
@@ -140,20 +139,12 @@ func (p *Planner) Call(ctx context.Context, opts *PlanningOptions) (*PlanningRes
Error: err.Error(),
}
log.Debug().Str("reason", err.Error()).Msg("parse content to actions failed")
// append assistant message
p.history.Append(&schema.Message{
Role: schema.Assistant,
Content: message.Content,
})
} else {
// append assistant message with tool calls
p.history.Append(&schema.Message{
Role: schema.Tool,
Content: result.Content,
ToolCalls: result.ToolCalls,
ToolCallID: fmt.Sprintf("%d", time.Now().Unix()),
})
}
// append assistant message (since we're parsing content, not using native function calling)
p.history.Append(&schema.Message{
Role: schema.Assistant,
Content: message.Content,
})
log.Info().
Interface("summary", result.ActionSummary).