refactor: replace ui-tars parser with https://github.com/bytedance/UI-TARS/blob/main/codes/ui_tars/action_parser.py

uixt/ai/parser_default.go

@@ -0,0 +1,157 @@
+package ai
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/httprunner/httprunner/v5/internal/json"
+	"github.com/httprunner/httprunner/v5/uixt/option"
+	"github.com/httprunner/httprunner/v5/uixt/types"
+	"github.com/pkg/errors"
+)
+
+// LLMContentParser parses the content from the LLM response
+// parser is corresponding to the model type and system prompt
+type LLMContentParser interface {
+	SystemPrompt() string
+	Parse(content string, size types.Size) (*PlanningResult, error)
+}
+
+func NewLLMContentParser(modelType option.LLMServiceType) LLMContentParser {
+	switch modelType {
+	case option.LLMServiceTypeUITARS:
+		return &UITARSContentParser{
+			systemPrompt: uiTarsPlanningPrompt,
+		}
+	default:
+		return &JSONContentParser{
+			systemPrompt: defaultPlanningResponseJsonFormat,
+		}
+	}
+}
+
+// JSONContentParser parses the response as JSON string format
+type JSONContentParser struct {
+	systemPrompt string
+}
+
+func (p *JSONContentParser) SystemPrompt() string {
+	return p.systemPrompt
+}
+
+func (p *JSONContentParser) Parse(content string, size types.Size) (*PlanningResult, error) {
+	content = strings.TrimSpace(content)
+	if strings.HasPrefix(content, "```json") && strings.HasSuffix(content, "```") {
+		content = strings.TrimPrefix(content, "```json")
+		content = strings.TrimSuffix(content, "```")
+	}
+	content = strings.TrimSpace(content)
+
+	var response PlanningResult
+	if err := json.Unmarshal([]byte(content), &response); err != nil {
+		return nil, fmt.Errorf("failed to parse VLM response: %v", err)
+	}
+
+	if response.Error != "" {
+		return nil, errors.New(response.Error)
+	}
+
+	if len(response.Actions) == 0 {
+		return nil, errors.New("no actions returned from VLM")
+	}
+
+	// normalize actions
+	var normalizedActions []Action
+	for i := range response.Actions {
+		// create a new variable, avoid implicit memory aliasing in for loop.
+		action := response.Actions[i]
+		if err := normalizeAction(&action); err != nil {
+			return nil, errors.Wrap(err, "failed to normalize action")
+		}
+		normalizedActions = append(normalizedActions, action)
+	}
+
+	return &PlanningResult{
+		Actions:       normalizedActions,
+		ActionSummary: response.ActionSummary,
+	}, nil
+}
+
+// normalizeAction normalizes the coordinates in the action
+func normalizeAction(action *Action) error {
+	switch action.ActionType {
+	case "click", "drag":
+		// handle click and drag action coordinates
+		if startBox, ok := action.ActionInputs["startBox"].(string); ok {
+			normalized, err := normalizeCoordinates(startBox)
+			if err != nil {
+				return fmt.Errorf("failed to normalize startBox: %w", err)
+			}
+			action.ActionInputs["startBox"] = normalized
+		}
+
+		if endBox, ok := action.ActionInputs["endBox"].(string); ok {
+			normalized, err := normalizeCoordinates(endBox)
+			if err != nil {
+				return fmt.Errorf("failed to normalize endBox: %w", err)
+			}
+			action.ActionInputs["endBox"] = normalized
+		}
+	}
+
+	return nil
+}
+
+// normalizeCoordinates normalizes the coordinates based on the factor
+func normalizeCoordinates(coordStr string) (coords []float64, err error) {
+	// check empty string
+	if coordStr == "" {
+		return nil, fmt.Errorf("empty coordinate string")
+	}
+
+	// handle BBox format: <bbox>x1 y1 x2 y2</bbox>
+	bboxRegex := regexp.MustCompile(`<bbox>(\d+\s+\d+\s+\d+\s+\d+)</bbox>`)
+	bboxMatches := bboxRegex.FindStringSubmatch(coordStr)
+	if len(bboxMatches) > 1 {
+		// Extract space-separated values from inside the bbox tags
+		bboxContent := bboxMatches[1]
+		// Split by whitespace
+		parts := strings.Fields(bboxContent)
+		if len(parts) == 4 {
+			coords = make([]float64, 4)
+			for i, part := range parts {
+				val, e := strconv.ParseFloat(part, 64)
+				if e != nil {
+					return nil, fmt.Errorf("failed to parse coordinate value '%s': %w", part, e)
+				}
+				coords[i] = val
+			}
+			// 将 val 转换为 [x,y] 坐标
+			x := (coords[0] + coords[2]) / 2
+			y := (coords[1] + coords[3]) / 2
+			return []float64{x, y}, nil
+		}
+	}
+
+	// handle coordinate string, e.g. "[100, 200]", "(100, 200)"
+	if strings.Contains(coordStr, ",") {
+		// remove possible brackets and split coordinates
+		coordStr = strings.Trim(coordStr, "[]() \t")
+
+		// try parsing JSON array
+		jsonStr := coordStr
+		if !strings.HasPrefix(jsonStr, "[") {
+			jsonStr = "[" + coordStr + "]"
+		}
+
+		err = json.Unmarshal([]byte(jsonStr), &coords)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse coordinate string: %w", err)
+		}
+		return coords, nil
+	}
+
+	return nil, fmt.Errorf("invalid coordinate string format: %s", coordStr)
+}