diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index f34e5513..87300ad9 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -24,9 +24,10 @@ type IAsserter interface { // AssertOptions represents the input options for assertion type AssertOptions struct { - Assertion string `json:"assertion"` // The assertion text to verify - Screenshot string `json:"screenshot"` // Base64 encoded screenshot - Size types.Size `json:"size"` // Screen dimensions + Assertion string `json:"assertion"` // The assertion text to verify + Screenshot string `json:"screenshot"` // Base64 encoded screenshot + Size types.Size `json:"size"` // Screen dimensions + ResetHistory bool `json:"reset_history"` // whether to reset conversation history before assertion } // AssertionResult represents the response from an AI assertion diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index d83ed2d8..77503d6b 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "strconv" "strings" "time" @@ -26,6 +27,7 @@ type WingsService struct { bizId string accessKey string secretKey string + history []History // Conversation history for Wings API } // NewWingsService creates a new Wings service instance @@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni return nil, errors.Wrap(err, "validate planning parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Extract screenshot from message screenshot, err := w.extractScreenshotFromMessage(opts.Message) if err != nil { @@ -70,25 +77,13 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: []interface{}{ - map[string]interface{}{ - "observation": "", - "thought": "", - "summary": "", - "step_text": "我获取了设备'设备a', 设备id是'xxx'；", - "device_id": "default-device", - "action_type": "material", - "action_result": "", - }, - }, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.UserInstruction, - BizId: w.bizId, - TextCase: fmt.Sprintf("整体描述：\n前置条件：\n获取 1 台设备 A。\n操作步骤：\n%s。\n停止操作。\n注意事项：\n", opts.UserInstruction), - StepType: "automation", - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfo: deviceInfo, + StepText: opts.UserInstruction, + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s。\n断言: 当前在“张杰演唱会”搜索页。\n停止操作。\n注意事项：\n", opts.UserInstruction), + StepType: "automation", + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -117,15 +112,36 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni }, err } - // Convert Wings API response to tool calls - toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams) - if err != nil { - return &PlanningResult{ - Thought: response.ThoughtChain.Thought, - Error: err.Error(), - ModelName: "wings-api", - }, errors.Wrap(err, "convert Wings response to tool calls failed") + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: parseOriStepIndex(response.OriStepIndex), + DeviceID: deviceInfo.DeviceID, + ActionType: response.StepType, + ActionResult: "", // Always empty as requested + DeviceInfo: &deviceInfo, + ActionParams: response.ActionParams, } + w.history = append(w.history, newHistoryEntry) + var toolCalls []schema.ToolCall + if response.StepType != "FINISH" { + // Convert Wings API response to tool calls + toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams) + if err != nil { + return &PlanningResult{ + Thought: response.ThoughtChain.Thought, + Error: err.Error(), + ModelName: "wings-api", + }, errors.Wrap(err, "convert Wings response to tool calls failed") + } + } + + // No need to update ActionResult as per user request + // ActionResult should always be empty log.Info(). Str("thought", response.ThoughtChain.Thought). @@ -148,6 +164,11 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert return nil, errors.Wrap(err, "validate assertion parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Clean screenshot data URL prefix cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot) @@ -156,25 +177,13 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []interface{}{ - map[string]interface{}{ - "observation": "", - "thought": "", - "summary": "", - "step_text": "我获取了设备'设备a', 设备id是'xxx'；", - "device_id": "", - "action_type": "material", - "action_result": "", - }, - }, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.Assertion, - BizId: w.bizId, - TextCase: "整体描述：\n前置条件：\n获取 1 台设备 A。\n获取 1 个[万粉创作者]账号a。\n获取 2 个[普通]账号 b、c。\n账号 a 和账号 b 互相关注。\n账号 a 和账号 c 互相关注。\n账号 a 给账号 b 设置备注为 “11131b”。\n账号 a 给账号 c 设置备注为 “11131c”。\n账号 a 创建一个粉丝群 m。\n 账号 a 修改粉丝群 m 名称为“11131群”。\n 账号 a 邀请账号 b 加入粉丝群 m。\n账号 a 邀请账号 c 加入粉丝群 m。\n账号 a 给群聊 m 发送一条文字消息。\n设备 A 打开抖音 app。\n设备 A 登录账号 a。\n设备 A 退出抖音 app。\n操作步骤：\n账号a打开抖音app。\n点击“消息”。\n点击“11131群”cell。\n点击“聊天信息页入口”按钮。\n点击“分享公开群”按钮。\n点击文字“群口令”。\n断言：屏幕中存在文字“口令复制成功”。\n停止操作。\n注意事项：\n", - StepType: "assert", // Different from automation - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfo: deviceInfo, + StepText: opts.Assertion, + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), + StepType: "assert", // Different from automation + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -204,6 +213,22 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, err } + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: parseOriStepIndex(response.OriStepIndex), + DeviceID: deviceInfo.DeviceID, + ActionType: response.StepType, + ActionResult: "", // Always empty as requested + DeviceInfo: &deviceInfo, + ActionParams: response.ActionParams, + } + w.history = append(w.history, newHistoryEntry) + // Parse assertion result from action_params passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain) if err != nil { @@ -214,6 +239,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, errors.Wrap(err, "parse assertion result failed") } + // No need to update ActionResult as per user request + // ActionResult should always be empty + log.Info(). Bool("passed", passed). Str("thought", assertionThought). @@ -248,14 +276,14 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []interface{} `json:"historys"` - DeviceInfos []WingsDeviceInfo `json:"device_infos"` - StepText string `json:"step_text"` - BizId string `json:"biz_id"` - TextCase string `json:"text_case"` - StepType string `json:"step_type"` - DeviceID string `json:"device_id"` - Base WingsBase `json:"Base"` + Historys []History `json:"historys"` + DeviceInfo WingsDeviceInfo `json:"device_info"` + StepText string `json:"step_text"` + BizId string `json:"biz_id"` + TextCase string `json:"text_case"` + StepType string `json:"step_type"` + DeviceID string `json:"device_id"` + Base WingsBase `json:"Base"` } type WingsDeviceInfo struct { @@ -273,10 +301,14 @@ type WingsBase struct { } type WingsActionResponse struct { - StepType string `json:"step_type"` - ActionParams string `json:"action_params"` - ThoughtChain WingsThoughtChain `json:"thought_chain"` - BaseResp WingsBaseResp `json:"BaseResp"` + AgentType string `json:"agent_type" thrift:"agent_type,1,required"` + StepText string `json:"step_text" thrift:"step_text,2,required"` + StepTextTrans string `json:"step_text_trans" thrift:"step_text_trans,3,required"` + OriStepIndex string `json:"ori_step_index" thrift:"ori_step_index,4,required"` + StepType string `json:"step_type" thrift:"step_type,5,required"` + ActionParams string `json:"action_params" thrift:"action_params,6,required"` + ThoughtChain WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"` + BaseResp WingsBaseResp `json:"BaseResp" thrift:"BaseResp,255,optional"` } type WingsThoughtChain struct { @@ -296,6 +328,21 @@ type WingsExtra struct { LogID string `json:"_log_id"` } +// History structure for request and response +type History struct { + Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 + Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 + Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 + StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 + DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id + ActionType string `json:"action_type" thrift:"action_type,7"` // 最终决策的agent类型 + ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 + DeviceInfo *WingsDeviceInfo `json:"device_info,omitempty" thrift:"device_info,9"` // 操作设备的信息 + ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) + StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) + OriStepIndex int64 `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） +} + // Action parameter structures type WingsActionParams struct { Type string `json:"Type"` @@ -335,11 +382,29 @@ type WingsTextParams struct { // Helper methods +// resetHistory resets the conversation history +func (w *WingsService) resetHistory() { + w.history = []History{} +} + // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { return uuid.New().String() } +// parseOriStepIndex converts string to int64 with fallback to 0 +func parseOriStepIndex(index string) int64 { + if index == "" { + return 0 + } + + val, err := strconv.ParseInt(index, 10, 64) + if err != nil { + return 0 + } + return val +} + // extractScreenshotFromMessage extracts base64 screenshot from message func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (string, error) { if message == nil || len(message.MultiContent) == 0 { @@ -454,7 +519,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) { - if actionParamsStr == "" { + if actionParamsStr == "" || actionParamsStr == "FINISH" { return []schema.ToolCall{}, nil } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 6ca5d2e3..0de1c76b 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,31 +292,14 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "点击搜索按钮" + prompt := "[目标导向]搜索“张杰演唱会”进入搜索页后停止" // Test both methods with the same prompt - aiResult, aiErr := driver.AIAction(context.Background(), prompt) + aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) - - // If both succeed, compare results - if aiResult != nil { - assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'") - - // Both should have timing information - assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time") - - // Both should have screenshot information - assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path") - - // Compare model names - if aiResult.PlanningResult != nil { - t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName) - - assert.Equal(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should use wings-api") - } - } + t.Logf("AIAction result: %v", aiResult) } // TestDriverExt_AIAction_ErrorHandling tests AIAction error handling