feat: 支持History储存和StartToGoal

uixt/ai/asserter.go

@@ -24,9 +24,10 @@ type IAsserter interface {
 
 // AssertOptions represents the input options for assertion
 type AssertOptions struct {
-	Assertion  string     `json:"assertion"`  // The assertion text to verify
+	Assertion    string     `json:"assertion"`     // The assertion text to verify
-	Screenshot string     `json:"screenshot"` // Base64 encoded screenshot
+	Screenshot   string     `json:"screenshot"`    // Base64 encoded screenshot
-	Size       types.Size `json:"size"`       // Screen dimensions
+	Size         types.Size `json:"size"`          // Screen dimensions
+	ResetHistory bool       `json:"reset_history"` // whether to reset conversation history before assertion
 }
 
 // AssertionResult represents the response from an AI assertion

uixt/ai/wings_service.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -26,6 +27,7 @@ type WingsService struct {
 	bizId     string
 	accessKey string
 	secretKey string
+	history   []History // Conversation history for Wings API
 }
 
 // NewWingsService creates a new Wings service instance
@@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
 		return nil, errors.Wrap(err, "validate planning parameters failed")
 	}
 
+	// Reset history if requested
+	if opts.ResetHistory {
+		w.resetHistory()
+	}
+
 	// Extract screenshot from message
 	screenshot, err := w.extractScreenshotFromMessage(opts.Message)
 	if err != nil {
@@ -70,25 +77,13 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
 
 	// Prepare Wings API request
 	apiRequest := WingsActionRequest{
-		Historys: []interface{}{
+		Historys:   w.history,
-			map[string]interface{}{
+		DeviceInfo: deviceInfo,
-				"observation":   "",
+		StepText:   opts.UserInstruction,
-				"thought":       "",
+		BizId:      w.bizId,
-				"summary":       "",
+		TextCase:   fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s。\n断言: 当前在“张杰演唱会”搜索页。\n停止操作。\n注意事项：\n", opts.UserInstruction),
-				"step_text":     "我获取了设备'设备a', 设备id是'xxx'；",
+		StepType:   "automation",
-				"device_id":     "default-device",
+		DeviceID:   deviceInfo.DeviceID,
-				"action_type":   "material",
-				"action_result": "",
-			},
-		}, // empty as specified
-		DeviceInfos: []WingsDeviceInfo{
-			deviceInfo,
-		},
-		StepText: opts.UserInstruction,
-		BizId:    w.bizId,
-		TextCase: fmt.Sprintf("整体描述：\n前置条件：\n获取 1 台设备 A。\n操作步骤：\n%s。\n停止操作。\n注意事项：\n", opts.UserInstruction),
-		StepType: "automation",
-		DeviceID: deviceInfo.DeviceID,
 		Base: WingsBase{
 			LogID: generateWingsUUID(),
 		},
@@ -117,15 +112,36 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
 		}, err
 	}
 
-	// Convert Wings API response to tool calls
+	// Update history with response data
-	toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams)
+	newHistoryEntry := History{
-	if err != nil {
+		Observation:   response.ThoughtChain.Observation,
-		return &PlanningResult{
+		Thought:       response.ThoughtChain.Thought,
-			Thought:   response.ThoughtChain.Thought,
+		Summary:       response.ThoughtChain.Summary,
-			Error:     err.Error(),
+		StepText:      response.StepText,
-			ModelName: "wings-api",
+		StepTextTrans: response.StepTextTrans,
-		}, errors.Wrap(err, "convert Wings response to tool calls failed")
+		OriStepIndex:  parseOriStepIndex(response.OriStepIndex),
+		DeviceID:      deviceInfo.DeviceID,
+		ActionType:    response.StepType,
+		ActionResult:  "", // Always empty as requested
+		DeviceInfo:    &deviceInfo,
+		ActionParams:  response.ActionParams,
 	}
+	w.history = append(w.history, newHistoryEntry)
+	var toolCalls []schema.ToolCall
+	if response.StepType != "FINISH" {
+		// Convert Wings API response to tool calls
+		toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams)
+		if err != nil {
+			return &PlanningResult{
+				Thought:   response.ThoughtChain.Thought,
+				Error:     err.Error(),
+				ModelName: "wings-api",
+			}, errors.Wrap(err, "convert Wings response to tool calls failed")
+		}
+	}
+
+	// No need to update ActionResult as per user request
+	// ActionResult should always be empty
 
 	log.Info().
 		Str("thought", response.ThoughtChain.Thought).
@@ -148,6 +164,11 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
 		return nil, errors.Wrap(err, "validate assertion parameters failed")
 	}
 
+	// Reset history if requested
+	if opts.ResetHistory {
+		w.resetHistory()
+	}
+
 	// Clean screenshot data URL prefix
 	cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot)
 
@@ -156,25 +177,13 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
 
 	// Prepare Wings API request for assertion
 	apiRequest := WingsActionRequest{
-		Historys: []interface{}{
+		Historys:   w.history,
-			map[string]interface{}{
+		DeviceInfo: deviceInfo,
-				"observation":   "",
+		StepText:   opts.Assertion,
-				"thought":       "",
+		BizId:      w.bizId,
-				"summary":       "",
+		TextCase:   fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion),
-				"step_text":     "我获取了设备'设备a', 设备id是'xxx'；",
+		StepType:   "assert", // Different from automation
-				"device_id":     "",
+		DeviceID:   deviceInfo.DeviceID,
-				"action_type":   "material",
-				"action_result": "",
-			},
-		}, // empty as specified
-		DeviceInfos: []WingsDeviceInfo{
-			deviceInfo,
-		},
-		StepText: opts.Assertion,
-		BizId:    w.bizId,
-		TextCase: "整体描述：\n前置条件：\n获取 1 台设备 A。\n获取 1 个[万粉创作者]账号a。\n获取 2 个[普通]账号 b、c。\n账号 a 和账号 b 互相关注。\n账号 a 和账号 c 互相关注。\n账号 a 给账号 b 设置备注为 “11131b”。\n账号 a 给账号 c 设置备注为 “11131c”。\n账号 a 创建一个粉丝群 m。\n 账号 a 修改粉丝群 m 名称为“11131群”。\n 账号 a 邀请账号 b 加入粉丝群 m。\n账号 a 邀请账号 c 加入粉丝群 m。\n账号 a 给群聊 m 发送一条文字消息。\n设备 A 打开抖音 app。\n设备 A 登录账号 a。\n设备 A 退出抖音 app。\n操作步骤：\n账号a打开抖音app。\n点击“消息”。\n点击“11131群”cell。\n点击“聊天信息页入口”按钮。\n点击“分享公开群”按钮。\n点击文字“群口令”。\n断言：屏幕中存在文字“口令复制成功”。\n停止操作。\n注意事项：\n",
-		StepType: "assert", // Different from automation
-		DeviceID: deviceInfo.DeviceID,
 		Base: WingsBase{
 			LogID: generateWingsUUID(),
 		},
@@ -204,6 +213,22 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
 		}, err
 	}
 
+	// Update history with response data
+	newHistoryEntry := History{
+		Observation:   response.ThoughtChain.Observation,
+		Thought:       response.ThoughtChain.Thought,
+		Summary:       response.ThoughtChain.Summary,
+		StepText:      response.StepText,
+		StepTextTrans: response.StepTextTrans,
+		OriStepIndex:  parseOriStepIndex(response.OriStepIndex),
+		DeviceID:      deviceInfo.DeviceID,
+		ActionType:    response.StepType,
+		ActionResult:  "", // Always empty as requested
+		DeviceInfo:    &deviceInfo,
+		ActionParams:  response.ActionParams,
+	}
+	w.history = append(w.history, newHistoryEntry)
+
 	// Parse assertion result from action_params
 	passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain)
 	if err != nil {
@@ -214,6 +239,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
 		}, errors.Wrap(err, "parse assertion result failed")
 	}
 
+	// No need to update ActionResult as per user request
+	// ActionResult should always be empty
+
 	log.Info().
 		Bool("passed", passed).
 		Str("thought", assertionThought).
@@ -248,14 +276,14 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error {
 
 // Wings API data structures
 type WingsActionRequest struct {
-	Historys    []interface{}     `json:"historys"`
+	Historys   []History       `json:"historys"`
-	DeviceInfos []WingsDeviceInfo `json:"device_infos"`
+	DeviceInfo WingsDeviceInfo `json:"device_info"`
-	StepText    string            `json:"step_text"`
+	StepText   string          `json:"step_text"`
-	BizId       string            `json:"biz_id"`
+	BizId      string          `json:"biz_id"`
-	TextCase    string            `json:"text_case"`
+	TextCase   string          `json:"text_case"`
-	StepType    string            `json:"step_type"`
+	StepType   string          `json:"step_type"`
-	DeviceID    string            `json:"device_id"`
+	DeviceID   string          `json:"device_id"`
-	Base        WingsBase         `json:"Base"`
+	Base       WingsBase       `json:"Base"`
 }
 
 type WingsDeviceInfo struct {
@@ -273,10 +301,14 @@ type WingsBase struct {
 }
 
 type WingsActionResponse struct {
-	StepType     string            `json:"step_type"`
+	AgentType     string            `json:"agent_type" thrift:"agent_type,1,required"`
-	ActionParams string            `json:"action_params"`
+	StepText      string            `json:"step_text" thrift:"step_text,2,required"`
-	ThoughtChain WingsThoughtChain `json:"thought_chain"`
+	StepTextTrans string            `json:"step_text_trans" thrift:"step_text_trans,3,required"`
-	BaseResp     WingsBaseResp     `json:"BaseResp"`
+	OriStepIndex  string            `json:"ori_step_index" thrift:"ori_step_index,4,required"`
+	StepType      string            `json:"step_type" thrift:"step_type,5,required"`
+	ActionParams  string            `json:"action_params" thrift:"action_params,6,required"`
+	ThoughtChain  WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"`
+	BaseResp      WingsBaseResp     `json:"BaseResp" thrift:"BaseResp,255,optional"`
 }
 
 type WingsThoughtChain struct {
@@ -296,6 +328,21 @@ type WingsExtra struct {
 	LogID    string `json:"_log_id"`
 }
 
+// History structure for request and response
+type History struct {
+	Observation   string           `json:"observation" thrift:"observation,1,required"`           // 思考结果
+	Thought       string           `json:"thought" thrift:"thought,2,required"`                   // 思考结果
+	Summary       string           `json:"summary" thrift:"summary,3,required"`                   // 思考结果
+	StepText      string           `json:"step_text" thrift:"step_text,4"`                        // 操作的指令
+	DeviceID      string           `json:"device_id" thrift:"device_id,5"`                        // 操作的设备id
+	ActionType    string           `json:"action_type" thrift:"action_type,7"`                    // 最终决策的agent类型
+	ActionResult  string           `json:"action_result" thrift:"action_result,8"`                // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果
+	DeviceInfo    *WingsDeviceInfo `json:"device_info,omitempty" thrift:"device_info,9"`          // 操作设备的信息
+	ActionParams  string           `json:"action_params,omitempty" thrift:"action_params,10"`     // 历史操作解析结果(断言，自动化，物料构造)
+	StepTextTrans string           `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本)
+	OriStepIndex  int64            `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"`   // 原本的执行序列（扩展前、目标导向原始文本步骤）
+}
+
 // Action parameter structures
 type WingsActionParams struct {
 	Type    string      `json:"Type"`
@@ -335,11 +382,29 @@ type WingsTextParams struct {
 
 // Helper methods
 
+// resetHistory resets the conversation history
+func (w *WingsService) resetHistory() {
+	w.history = []History{}
+}
+
 // generateWingsUUID generates a random UUID for LogID
 func generateWingsUUID() string {
 	return uuid.New().String()
 }
 
+// parseOriStepIndex converts string to int64 with fallback to 0
+func parseOriStepIndex(index string) int64 {
+	if index == "" {
+		return 0
+	}
+
+	val, err := strconv.ParseInt(index, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return val
+}
+
 // extractScreenshotFromMessage extracts base64 screenshot from message
 func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (string, error) {
 	if message == nil || len(message.MultiContent) == 0 {
@@ -454,7 +519,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
 
 // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach
 func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) {
-	if actionParamsStr == "" {
+	if actionParamsStr == "" || actionParamsStr == "FINISH" {
 		return []schema.ToolCall{}, nil
 	}
 

uixt/driver_ext_ai_test.go

@@ -292,31 +292,14 @@ func TestDriverExt_AIAction(t *testing.T) {
 func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) {
 	driver := setupDriverExt(t)
 
-	prompt := "点击搜索按钮"
+	prompt := "[目标导向]搜索“张杰演唱会”进入搜索页后停止"
 
 	// Test both methods with the same prompt
-	aiResult, aiErr := driver.AIAction(context.Background(), prompt)
+	aiResult, aiErr := driver.StartToGoal(context.Background(), prompt)
 
 	// Both should execute without critical errors (may have different implementations)
 	t.Logf("AIAction error: %v", aiErr)
-
+	t.Logf("AIAction result: %v", aiResult)
-	// If both succeed, compare results
-	if aiResult != nil {
-		assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'")
-
-		// Both should have timing information
-		assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time")
-
-		// Both should have screenshot information
-		assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path")
-
-		// Compare model names
-		if aiResult.PlanningResult != nil {
-			t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName)
-
-			assert.Equal(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should use wings-api")
-		}
-	}
 }
 
 // TestDriverExt_AIAction_ErrorHandling tests AIAction error handling