refactor: merge ai parser

2026-09-07 08:27:35 +08:00 · 2025-05-24 00:25:44 +08:00
parent 19ddcb40cc
commit 81c854f963
7 changed files with 929 additions and 256 deletions
@@ -14,9 +14,6 @@ import (
 	"github.com/rs/zerolog/log"
 )

-// reference:
-// https://github.com/bytedance/UI-TARS/blob/main/codes/ui_tars/action_parser.py
-
 const (
 	DefaultFactor = 1000
 )
@@ -32,35 +29,31 @@ func (p *UITARSContentParser) SystemPrompt() string {

 // ParseActionToStructureOutput parses the model output text into structured actions.
 func (p *UITARSContentParser) Parse(content string, size types.Size) (*PlanningResult, error) {
-	text := strings.TrimSpace(content)
+	content = strings.TrimSpace(content)

-	// Extract thought/reflection
-	thought := p.extractThought(text)
+	// Extract thought string
+	thought := p.extractThought(content)

-	// Normalize text first
-	normalizedText := p.normalizeCoordinates(text)
-
-	// Get action string from normalized text
-	actionStr, err := p.extractActionString(normalizedText)
+	// Extract action string
+	actionStr, err := p.extractActionString(content)
 	if err != nil {
 		return nil, err
 	}

-	// Parse actions directly
+	// Parse and process actions
 	actions, err := p.parseActionString(actionStr, size)
 	if err != nil {
 		return nil, err
 	}

 	// Convert actions to tool calls
-	toolCalls := p.convertActionsToToolCalls(actions)
+	toolCalls := convertActionsToToolCalls(actions)

 	return &PlanningResult{
 		ToolCalls:     toolCalls,
-		Actions:       actions,
 		ActionSummary: thought,
 		Thought:       thought,
-		Text:          normalizedText,
+		Content:       content,
 	}, nil
 }

@@ -85,8 +78,31 @@ func (p *UITARSContentParser) extractActionString(text string) (string, error) {
 	return "", fmt.Errorf("no Action: found")
 }

-// normalizeCoordinates normalizes the text by converting points to coordinates and replacing keywords
-func (p *UITARSContentParser) normalizeCoordinates(text string) string {
+// parseActionString parse and process actions
+func (p *UITARSContentParser) parseActionString(actionStr string, size types.Size) ([]Action, error) {
+	// Parse action type and raw arguments
+	actionType, rawArgs, err := parseActionTypeAndArguments(actionStr)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process and normalize arguments
+	processedArgs, err := processActionArguments(rawArgs, size)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create final action
+	action := Action{
+		ActionType:   actionType,
+		ActionInputs: processedArgs,
+	}
+
+	return []Action{action}, nil
+}
+
+// normalizeCoordinatesFormat standardizes coordinate format in text (without pixel conversion)
+func normalizeCoordinatesFormat(text string) string {
 	// Convert point tags to coordinate format
 	if strings.Contains(text, "<point>") {
 		// support <point>x1 y1 x2 y2</point> or <point>x y</point>
@@ -127,28 +143,32 @@ func (p *UITARSContentParser) normalizeCoordinates(text string) string {
 		})
 	}

-	// Legacy parameter name replacements (keep for backward compatibility)
-	text = strings.ReplaceAll(text, "start_point=", "start_box=")
-	text = strings.ReplaceAll(text, "end_point=", "end_box=")
-	text = strings.ReplaceAll(text, "point=", "start_box=")
-
 	return text
 }

-// parseActionString parses the action string directly
-func (p *UITARSContentParser) parseActionString(actionStr string, size types.Size) ([]Action, error) {
-	actions := make([]Action, 0, 1)
+// convertRelativeToAbsolute converts relative coordinates to absolute pixel coordinates
+func convertRelativeToAbsolute(relativeCoord float64, isXCoord bool, size types.Size) float64 {
+	if isXCoord {
+		return math.Round((relativeCoord/DefaultFactor*float64(size.Width))*10) / 10
+	}
+	return math.Round((relativeCoord/DefaultFactor*float64(size.Height))*10) / 10
+}

+// parseActionTypeAndArguments extracts function name and raw parameter map from action string
+// Input: "click(start_box='100,200,150,250')" or "click(start_point='100,200,150,250')"
+// Output: actionType="click", rawArgs={"start_box": "100,200,150,250"}
+func parseActionTypeAndArguments(actionStr string) (actionType string, rawArgs map[string]interface{}, err error) {
 	// Parse action type and parameters
 	actionParts := strings.SplitN(actionStr, "(", 2)
 	if len(actionParts) < 2 {
-		return nil, fmt.Errorf("not a function call")
+		return "", nil, fmt.Errorf("not a function call")
 	}

-	funcName := strings.TrimSpace(actionParts[0])
+	actionType = strings.TrimSpace(actionParts[0])
 	paramsText := strings.TrimSuffix(strings.TrimSpace(actionParts[1]), ")")

-	args := make(map[string]string)
+	// Parse string parameters to map
+	rawArgs = make(map[string]interface{})
 	if paramsText != "" {
 		// Use regex to extract key=value pairs, handling quoted values properly
 		re := regexp.MustCompile(`(\w+)\s*=\s*['"]([^'"]*?)['"]`)
@@ -157,76 +177,188 @@ func (p *UITARSContentParser) parseActionString(actionStr string, size types.Siz
 			if len(match) >= 3 {
 				key := strings.TrimSpace(match[1])
 				value := strings.TrimSpace(match[2])
-				args[key] = value
+
+				// Apply parameter name mapping (legacy compatibility)
+				key = normalizeParameterName(key)
+				rawArgs[key] = value
 			}
 		}
 	}

-	actionInputs, err := p.parseActionInputs(args, size)
-	if err != nil {
-		return nil, err
-	}
-
-	actions = append(actions, Action{
-		ActionType:   funcName,
-		ActionInputs: actionInputs,
-	})
-
-	return actions, nil
+	return actionType, rawArgs, nil
 }

-// parseActionInputs parses action parameters and converts coordinates
-func (p *UITARSContentParser) parseActionInputs(args map[string]string, size types.Size) (map[string]any, error) {
-	actionInputs := make(map[string]any)
-	imageWidth := size.Width
-	imageHeight := size.Height
+// normalizeParameterName applies legacy parameter name mappings
+func normalizeParameterName(paramName string) string {
+	switch paramName {
+	case "start_point":
+		return "start_box"
+	case "end_point":
+		return "end_box"
+	case "point":
+		return "start_box"
+	default:
+		return paramName
+	}
+}

-	for paramName, param := range args {
-		if param == "" {
-			continue
-		}
-		param = strings.TrimSpace(param)
+// processActionArguments processes raw arguments based on action type and parameter types
+// Input: rawArgs={"start_box": "100,200,150,250"}
+// Output: processedArgs={"start_box": [120.5, 240.1, 180.7, 300.2]} (converted to pixels)
+func processActionArguments(rawArgs map[string]interface{}, size types.Size) (map[string]interface{}, error) {
+	processedArgs := make(map[string]interface{})

-		// Convert box coordinates
-		if strings.Contains(paramName, "box") || strings.Contains(paramName, "point") {
-			// Extract numbers from the parameter value using regex
-			re := regexp.MustCompile(`\d+`)
-			numbers := re.FindAllString(param, -1)
-			if len(numbers) >= 2 {
-				coords := make([]float64, len(numbers))
-				for i, numStr := range numbers {
-					num, err := strconv.ParseFloat(numStr, 64)
-					if err != nil {
-						return nil, fmt.Errorf("invalid coordinate: %s", numStr)
-					}
-					// Convert relative coordinates to absolute coordinates
-					if i%2 == 0 { // x coordinates
-						coords[i] = math.Round((num/DefaultFactor*float64(imageWidth))*10) / 10
-					} else { // y coordinates
-						coords[i] = math.Round((num/DefaultFactor*float64(imageHeight))*10) / 10
-					}
-				}
-				actionInputs[paramName] = coords
-			} else {
-				actionInputs[paramName] = param
-			}
-		} else {
-			// Handle other parameter types (content, key, direction, etc.)
-			if paramName == "content" {
-				// Handle escape characters
-				param = strings.ReplaceAll(param, "\\n", "\n")
-				param = strings.ReplaceAll(param, "\\\"", "\"")
-				param = strings.ReplaceAll(param, "\\'", "'")
-			}
-			actionInputs[paramName] = param
+	// Process each argument based on its type and context
+	for paramName, paramValue := range rawArgs {
+		processed, err := processArgument(paramName, paramValue, size)
+		if err != nil {
+			return nil, fmt.Errorf("failed to process argument %s: %w", paramName, err)
 		}
+		processedArgs[paramName] = processed
 	}

-	return actionInputs, nil
+	return processedArgs, nil
+}
+
+// Process a single argument based on its name and value
+func processArgument(paramName string, paramValue interface{}, size types.Size) (interface{}, error) {
+	// Handle coordinate parameters
+	if isCoordinateParameter(paramName) {
+		return normalizeActionCoordinates(paramValue, size)
+	}
+
+	// Handle other parameter types (content, key, direction, etc.)
+	return normalizeStringParam(paramName, paramValue), nil
+}
+
+// Check if a parameter is a coordinate parameter
+func isCoordinateParameter(paramName string) bool {
+	return strings.Contains(paramName, "box") || strings.Contains(paramName, "point")
+}
+
+// normalizeActionCoordinates normalizes coordinates from various formats to actual pixel coordinates
+func normalizeActionCoordinates(coordData interface{}, size types.Size) ([]float64, error) {
+	switch v := coordData.(type) {
+	case []interface{}:
+		// Handle JSON array format: [x1, y1, x2, y2] or [x1, y1]
+		if len(v) < 2 {
+			return nil, fmt.Errorf("coordinate array must have at least 2 elements, got %d", len(v))
+		}
+
+		coords := make([]float64, len(v))
+		for i, val := range v {
+			switch num := val.(type) {
+			case float64:
+				// Convert relative coordinates to absolute coordinates using DefaultFactor
+				if i%2 == 0 { // x coordinates
+					coords[i] = convertRelativeToAbsolute(num, true, size)
+				} else { // y coordinates
+					coords[i] = convertRelativeToAbsolute(num, false, size)
+				}
+			case int:
+				numFloat := float64(num)
+				// Convert relative coordinates to absolute coordinates using DefaultFactor
+				if i%2 == 0 { // x coordinates
+					coords[i] = convertRelativeToAbsolute(numFloat, true, size)
+				} else { // y coordinates
+					coords[i] = convertRelativeToAbsolute(numFloat, false, size)
+				}
+			default:
+				return nil, fmt.Errorf("coordinate value must be a number, got %T", val)
+			}
+		}
+		return coords, nil
+
+	case []float64:
+		// Handle already parsed float64 slice
+		coords := make([]float64, len(v))
+		for i, val := range v {
+			if i%2 == 0 { // x coordinates
+				coords[i] = convertRelativeToAbsolute(val, true, size)
+			} else { // y coordinates
+				coords[i] = convertRelativeToAbsolute(val, false, size)
+			}
+		}
+		return coords, nil
+
+	case string:
+		// Handle string format (from UI-TARS or string coordinates)
+		return normalizeStringCoordinates(v, size)
+
+	default:
+		return nil, fmt.Errorf("unsupported coordinate format: %T", coordData)
+	}
+}
+
+// normalizeStringParam normalizes string parameters, handling escape characters for content
+func normalizeStringParam(paramName string, paramValue interface{}) interface{} {
+	if paramValue == nil {
+		return paramValue
+	}
+
+	// Convert to string if possible
+	param, ok := paramValue.(string)
+	if !ok {
+		return paramValue // Return as-is if not a string
+	}
+
+	param = strings.TrimSpace(param)
+	if param == "" {
+		return param
+	}
+
+	// Handle escape characters for content parameter
+	if paramName == "content" {
+		param = strings.ReplaceAll(param, "\\n", "\n")
+		param = strings.ReplaceAll(param, "\\\"", "\"")
+		param = strings.ReplaceAll(param, "\\'", "'")
+	}
+
+	return param
+}
+
+// normalizeStringCoordinates normalizes coordinates from string format
+func normalizeStringCoordinates(coordStr string, size types.Size) ([]float64, error) {
+	// check empty string
+	if coordStr == "" {
+		return nil, fmt.Errorf("empty coordinate string")
+	}
+
+	// Apply coordinate format normalization using the shared function
+	normalizedStr := normalizeCoordinatesFormat(coordStr)
+
+	// Extract numbers from the normalized string using regex
+	re := regexp.MustCompile(`\d+`)
+	numbers := re.FindAllString(normalizedStr, -1)
+	if len(numbers) >= 2 {
+		coords := make([]float64, len(numbers))
+		for i, numStr := range numbers {
+			num, err := strconv.ParseFloat(numStr, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid coordinate: %s", numStr)
+			}
+			// Convert relative coordinates to absolute coordinates
+			if i%2 == 0 { // x coordinates
+				coords[i] = convertRelativeToAbsolute(num, true, size)
+			} else { // y coordinates
+				coords[i] = convertRelativeToAbsolute(num, false, size)
+			}
+		}
+		return coords, nil
+	}
+
+	return nil, fmt.Errorf("invalid coordinate string format: %s", coordStr)
+}
+
+// Action represents a parsed action with its context.
+type Action struct {
+	ActionType   string         `json:"action_type"`
+	ActionInputs map[string]any `json:"action_inputs"`
 }

 // convertActionsToToolCalls converts actions to tool calls
-func (p *UITARSContentParser) convertActionsToToolCalls(actions []Action) []schema.ToolCall {
+// This is a shared function used by both JSONContentParser and UITARSContentParser
+func convertActionsToToolCalls(actions []Action) []schema.ToolCall {
 	toolCalls := make([]schema.ToolCall, 0, len(actions))
 	for _, action := range actions {
 		jsonArgs, err := json.Marshal(action.ActionInputs)
@@ -245,45 +377,3 @@ func (p *UITARSContentParser) convertActionsToToolCalls(actions []Action) []sche
 	}
 	return toolCalls
 }
-
-// Action represents a parsed action with its context.
-type Action struct {
-	ActionType   string         `json:"action_type"`
-	ActionInputs map[string]any `json:"action_inputs"`
-}
-
-// ParseAction parses an action string into function name and arguments.
-func ParseAction(actionStr string) (*ParsedAction, error) {
-	// Parse action type and parameters
-	actionParts := strings.SplitN(actionStr, "(", 2)
-	if len(actionParts) < 2 {
-		return nil, fmt.Errorf("not a function call")
-	}
-
-	funcName := strings.TrimSpace(actionParts[0])
-	paramsText := strings.TrimSuffix(strings.TrimSpace(actionParts[1]), ")")
-
-	args := make(map[string]string)
-	if paramsText != "" {
-		// Split parameters by comma and parse key=value pairs
-		for _, param := range strings.Split(paramsText, ",") {
-			param = strings.TrimSpace(param)
-			if strings.Contains(param, "=") {
-				parts := strings.SplitN(param, "=", 2)
-				key := strings.TrimSpace(parts[0])
-				value := strings.TrimSpace(parts[1])
-				// Remove surrounding quotes
-				value = strings.Trim(value, "'\"")
-				args[key] = value
-			}
-		}
-	}
-
-	return &ParsedAction{Function: funcName, Args: args}, nil
-}
-
-// ParsedAction represents the result of parsing an action string.
-type ParsedAction struct {
-	Function string
-	Args     map[string]string
-}