refactor: unify action execution interface and merge AI action handling

2026-09-04 23:16:57 +08:00 · 2025-06-07 23:59:07 +08:00
parent fcf3009c67
commit ec4f1eb68a
8 changed files with 199 additions and 65 deletions
@@ -3,13 +3,12 @@ package uixt
 import (
 	"context"
 	"encoding/base64"
-	"fmt"
-	"path/filepath"
+	"strings"
+	"time"

 	"github.com/cloudwego/eino/schema"
 	"github.com/httprunner/httprunner/v5/code"
 	"github.com/httprunner/httprunner/v5/internal/builtin"
-	"github.com/httprunner/httprunner/v5/internal/config"
 	"github.com/httprunner/httprunner/v5/internal/json"
 	"github.com/httprunner/httprunner/v5/uixt/ai"
 	"github.com/httprunner/httprunner/v5/uixt/option"
@@ -18,10 +17,11 @@ import (
 	"github.com/rs/zerolog/log"
 )

-func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...option.ActionOption) error {
+func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...option.ActionOption) ([]*SubActionResult, error) {
 	options := option.NewActionOptions(opts...)
 	log.Info().Int("max_retry_times", options.MaxRetryTimes).Msg("StartToGoal")

+	var allSubActions []*SubActionResult
 	var attempt int
 	for {
 		attempt++
@@ -31,7 +31,7 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op
 		select {
 		case <-ctx.Done():
 			log.Warn().Msg("interrupted in StartToGoal")
-			return errors.Wrap(code.InterruptError, "StartToGoal interrupted")
+			return allSubActions, errors.Wrap(code.InterruptError, "StartToGoal interrupted")
 		default:
 		}

@@ -49,37 +49,44 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op
 					Msg("LLM service request failed, retrying...")
 				continue
 			}
-			return err
+			return allSubActions, err
 		}

 		// Check if task is finished BEFORE executing actions
 		if dExt.isTaskFinished(result) {
 			log.Info().Msg("task finished, stopping StartToGoal")
-			return nil
+			return allSubActions, nil
 		}

-		// Execute actions only if task is not finished
-		if err := dExt.executeActions(ctx, result.ToolCalls); err != nil {
-			return err
+		// Invoke tool calls
+		subActions, err := dExt.invokeToolCalls(ctx, result.Thought, result.ToolCalls)
+		allSubActions = append(allSubActions, subActions...)
+		if err != nil {
+			return allSubActions, err
 		}

 		if options.MaxRetryTimes > 1 && attempt >= options.MaxRetryTimes {
-			return errors.New("reached max retry times")
+			return allSubActions, errors.New("reached max retry times")
 		}
 	}
 }

-func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) error {
+func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) ([]*SubActionResult, error) {
 	log.Info().Str("prompt", prompt).Msg("performing AI action")

 	// plan next action
 	result, err := dExt.PlanNextAction(ctx, prompt, opts...)
 	if err != nil {
-		return err
+		return nil, err
 	}

-	// execute actions
-	return dExt.executeActions(ctx, result.ToolCalls)
+	// Invoke tool calls
+	subActionResults, err := dExt.invokeToolCalls(ctx, result.Thought, result.ToolCalls)
+	if err != nil {
+		return subActionResults, err
+	}
+
+	return subActionResults, nil
 }

 func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ...option.ActionOption) (*ai.PlanningResult, error) {
@@ -87,36 +94,28 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ..
 		return nil, errors.New("LLM service is not initialized")
 	}

-	compressedBufSource, err := getScreenShotBuffer(dExt.IDriver)
+	// Parse action options to get ResetHistory setting
+	options := option.NewActionOptions(opts...)
+	resetHistory := options.ResetHistory
+
+	// Use GetScreenResult to handle screenshot capture, save, and session tracking
+	screenResult, err := dExt.GetScreenResult(
+		option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")),
+	)
 	if err != nil {
 		return nil, err
 	}

-	// convert buffer to base64 string
+	// convert buffer to base64 string for LLM
 	screenShotBase64 := "data:image/jpeg;base64," +
-		base64.StdEncoding.EncodeToString(compressedBufSource.Bytes())
-
-	// save screenshot to file
-	imagePath := filepath.Join(
-		config.GetConfig().ScreenShotsPath,
-		fmt.Sprintf("%s.jpeg", builtin.GenNameWithTimestamp("%d_screenshot")),
-	)
-	go func() {
-		err := saveScreenShot(compressedBufSource, imagePath)
-		if err != nil {
-			log.Error().Err(err).Msg("save screenshot file failed")
-		}
-	}()
+		base64.StdEncoding.EncodeToString(screenResult.bufSource.Bytes())

+	// get window size
 	size, err := dExt.IDriver.WindowSize()
 	if err != nil {
 		return nil, errors.Wrap(code.DeviceGetInfoError, err.Error())
 	}

-	// Parse action options to get ResetHistory setting
-	options := option.NewActionOptions(opts...)
-	resetHistory := options.ResetHistory
-
 	planningOpts := &ai.PlanningOptions{
 		UserInstruction: prompt,
 		Message: &schema.Message{
@@ -160,23 +159,40 @@ func (dExt *XTDriver) isTaskFinished(result *ai.PlanningResult) bool {
 	return false
 }

-// executeActions executes the planned actions
-func (dExt *XTDriver) executeActions(ctx context.Context, toolCalls []schema.ToolCall) error {
+// invokeToolCalls invokes the tool calls and returns sub-action results
+func (dExt *XTDriver) invokeToolCalls(ctx context.Context, thought string, toolCalls []schema.ToolCall) ([]*SubActionResult, error) {
+	var subActionResults []*SubActionResult
+
 	for _, action := range toolCalls {
 		// Check for context cancellation before each action
 		select {
 		case <-ctx.Done():
-			log.Warn().Msg("interrupted in executeActions")
-			return errors.Wrap(code.InterruptError, "executeActions interrupted")
+			log.Warn().Msg("interrupted in invokeToolCalls")
+			return subActionResults, errors.Wrap(code.InterruptError, "invokeToolCalls interrupted")
 		default:
 		}

-		// call eino tool
+		subActionStartTime := time.Now()
+
+		// Extract action name (remove "uixt__" prefix)
+		actionName := strings.TrimPrefix(action.Function.Name, "uixt__")
+
+		// Parse arguments
 		arguments := make(map[string]interface{})
 		err := json.Unmarshal([]byte(action.Function.Arguments), &arguments)
 		if err != nil {
-			return err
+			return subActionResults, err
 		}
+
+		// Create sub-action result
+		subActionResult := &SubActionResult{
+			ActionName: actionName,
+			Arguments:  arguments,
+			StartTime:  subActionStartTime.Unix(),
+			Thought:    thought,
+		}
+
+		// Execute the action
 		req := mcp.CallToolRequest{
 			Params: struct {
 				Name      string         `json:"name"`
@@ -191,12 +207,42 @@ func (dExt *XTDriver) executeActions(ctx context.Context, toolCalls []schema.Too
 		}

 		_, err = dExt.client.CallTool(ctx, req)
+		subActionResult.Elapsed = time.Since(subActionStartTime).Milliseconds()
 		if err != nil {
-			return err
+			subActionResult.Error = err
+			subActionResults = append(subActionResults, subActionResult)
+			return subActionResults, err
 		}
+
+		// Collect sub-action specific attachments and reset session data
+		subActionData := dExt.GetData(true) // reset after getting data
+
+		// Add requests if any
+		if requests, ok := subActionData["requests"].([]*DriverRequests); ok && len(requests) > 0 {
+			subActionResult.Requests = requests
+		}
+
+		// Add screen_results if any
+		if screenResults, ok := subActionData["screen_results"].([]*ScreenResult); ok && len(screenResults) > 0 {
+			subActionResult.ScreenResults = screenResults
+		}
+
+		subActionResults = append(subActionResults, subActionResult)
 	}

-	return nil
+	return subActionResults, nil
+}
+
+// SubActionResult represents a sub-action within a start_to_goal action
+type SubActionResult struct {
+	ActionName    string            `json:"action_name"`              // name of the sub-action (e.g., "tap", "input")
+	Arguments     interface{}       `json:"arguments,omitempty"`      // arguments passed to the sub-action
+	StartTime     int64             `json:"start_time"`               // sub-action start time
+	Elapsed       int64             `json:"elapsed_ms"`               // sub-action elapsed time(ms)
+	Error         error             `json:"error,omitempty"`          // sub-action execution result
+	Thought       string            `json:"thought,omitempty"`        // sub-action thought
+	Requests      []*DriverRequests `json:"requests,omitempty"`       // store sub-action specific requests
+	ScreenResults []*ScreenResult   `json:"screen_results,omitempty"` // store sub-action specific screen_results
 }

 func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (string, error) {