feat: optimize UI-TARS parser with coordinate conversion and action mapping

- Add action mapping for UI-TARS parser to convert action names to option.ActionName
- Implement bounding box to center point coordinate conversion for better accuracy
- Update coordinate normalization to handle coordinates > 1000 properly
- Enhance test cases to verify coordinate scaling and center point conversion
- Improve action argument processing with proper coordinate transformation
- Add comprehensive test coverage for coordinate conversion edge cases

Key improvements:
- Bounding box [x1,y1,x2,y2] now converts to center point [cx,cy] for actions
- Coordinate scaling properly handles different screen resolutions
- Action names are mapped through doubao_1_5_ui_tars_action_mapping
- Enhanced error handling for invalid coordinate formats
This commit is contained in:
lilong.129
2025-06-04 23:16:14 +08:00
parent 1df529ecaa
commit c204542f1f
10 changed files with 386 additions and 411 deletions
+4 -4
View File
@@ -29,7 +29,7 @@ func TestVLMPlanning(t *testing.T) {
userInstruction += "\n\n请基于以上游戏规则,给出下一步可点击的两个图标坐标"
modelConfig, err := GetModelConfig(option.LLMServiceTypeDoubaoVL)
modelConfig, err := GetModelConfig(option.LLMServiceTypeUITARS)
require.NoError(t, err)
planner, err := NewPlanner(context.Background(), modelConfig)
@@ -157,9 +157,9 @@ func TestHandleSwitch(t *testing.T) {
imageFile string
actionType string
}{
{"testdata/deepseek_think_off.png", "click"}, // 关闭状态,需要点击开启
{"testdata/deepseek_think_on.png", "click"}, // 关闭状态,需要点击开启
{"testdata/deepseek_network_on.png", "finished"}, // 开启状态,无需操作
{"testdata/deepseek_think_off.png", "uixt__tap_xy"}, // 关闭状态,需要点击开启
{"testdata/deepseek_think_on.png", "uixt__tap_xy"}, // 关闭状态,需要点击开启
{"testdata/deepseek_network_on.png", "uixt__finished"}, // 开启状态,无需操作
}
for _, tc := range testCases {