From 44b4da80918e7291cc739a171f110aeb432227c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 16:30:16 +0800 Subject: [PATCH 01/32] =?UTF-8?q?feat:=20=E5=90=88=E5=B9=B6=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- uixt/ai/wings_service.go | 28 ++++++++++++++++++++++++---- uixt/android_test.go | 5 ----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index e5858edd..db846138 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250724 +v5.0.0-250728 diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 35fa77db..d83ed2d8 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -70,13 +70,23 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified + Historys: []interface{}{ + map[string]interface{}{ + "observation": "", + "thought": "", + "summary": "", + "step_text": "我获取了设备'设备a', 设备id是'xxx'；", + "device_id": "default-device", + "action_type": "material", + "action_result": "", + }, + }, // empty as specified DeviceInfos: []WingsDeviceInfo{ deviceInfo, }, StepText: opts.UserInstruction, BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n获取 1 台设备 A。\n操作步骤：\n%s。\n停止操作。\n注意事项：\n", opts.UserInstruction), StepType: "automation", DeviceID: deviceInfo.DeviceID, Base: WingsBase{ @@ -146,13 +156,23 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified + Historys: []interface{}{ + map[string]interface{}{ + "observation": "", + "thought": "", + "summary": "", + "step_text": "我获取了设备'设备a', 设备id是'xxx'；", + "device_id": "", + "action_type": "material", + "action_result": "", + }, + }, // empty as specified DeviceInfos: []WingsDeviceInfo{ deviceInfo, }, StepText: opts.Assertion, BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", + TextCase: "整体描述：\n前置条件：\n获取 1 台设备 A。\n获取 1 个[万粉创作者]账号a。\n获取 2 个[普通]账号 b、c。\n账号 a 和账号 b 互相关注。\n账号 a 和账号 c 互相关注。\n账号 a 给账号 b 设置备注为 “11131b”。\n账号 a 给账号 c 设置备注为 “11131c”。\n账号 a 创建一个粉丝群 m。\n 账号 a 修改粉丝群 m 名称为“11131群”。\n 账号 a 邀请账号 b 加入粉丝群 m。\n账号 a 邀请账号 c 加入粉丝群 m。\n账号 a 给群聊 m 发送一条文字消息。\n设备 A 打开抖音 app。\n设备 A 登录账号 a。\n设备 A 退出抖音 app。\n操作步骤：\n账号a打开抖音app。\n点击“消息”。\n点击“11131群”cell。\n点击“聊天信息页入口”按钮。\n点击“分享公开群”按钮。\n点击文字“群口令”。\n断言：屏幕中存在文字“口令复制成功”。\n停止操作。\n注意事项：\n", StepType: "assert", // Different from automation DeviceID: deviceInfo.DeviceID, Base: WingsBase{ diff --git a/uixt/android_test.go b/uixt/android_test.go index b8a085be..c7cb883d 100644 --- a/uixt/android_test.go +++ b/uixt/android_test.go @@ -21,11 +21,6 @@ func setupADBDriverExt(t *testing.T) *XTDriver { Serial: "", // Let it auto-detect the device serial AIOptions: []option.AIServiceOption{ option.WithCVService(option.CVServiceTypeVEDEM), - option.WithLLMConfig( - option.NewLLMServiceConfig(option.DOUBAO_1_5_UI_TARS_250328). - WithPlannerModel(option.WINGS_SERVICE). - WithAsserterModel(option.WINGS_SERVICE), - ), }, } From f51780025716be289bc08eee74b9b11a69439c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 19:39:29 +0800 Subject: [PATCH 02/32] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81History?= =?UTF-8?q?=E5=82=A8=E5=AD=98=E5=92=8CStartToGoal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/asserter.go | 7 +- uixt/ai/wings_service.go | 183 +++++++++++++++++++++++++------------ uixt/driver_ext_ai_test.go | 23 +---- 3 files changed, 131 insertions(+), 82 deletions(-) diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index f34e5513..87300ad9 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -24,9 +24,10 @@ type IAsserter interface { // AssertOptions represents the input options for assertion type AssertOptions struct { - Assertion string `json:"assertion"` // The assertion text to verify - Screenshot string `json:"screenshot"` // Base64 encoded screenshot - Size types.Size `json:"size"` // Screen dimensions + Assertion string `json:"assertion"` // The assertion text to verify + Screenshot string `json:"screenshot"` // Base64 encoded screenshot + Size types.Size `json:"size"` // Screen dimensions + ResetHistory bool `json:"reset_history"` // whether to reset conversation history before assertion } // AssertionResult represents the response from an AI assertion diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index d83ed2d8..77503d6b 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "strconv" "strings" "time" @@ -26,6 +27,7 @@ type WingsService struct { bizId string accessKey string secretKey string + history []History // Conversation history for Wings API } // NewWingsService creates a new Wings service instance @@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni return nil, errors.Wrap(err, "validate planning parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Extract screenshot from message screenshot, err := w.extractScreenshotFromMessage(opts.Message) if err != nil { @@ -70,25 +77,13 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: []interface{}{ - map[string]interface{}{ - "observation": "", - "thought": "", - "summary": "", - "step_text": "我获取了设备'设备a', 设备id是'xxx'；", - "device_id": "default-device", - "action_type": "material", - "action_result": "", - }, - }, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.UserInstruction, - BizId: w.bizId, - TextCase: fmt.Sprintf("整体描述：\n前置条件：\n获取 1 台设备 A。\n操作步骤：\n%s。\n停止操作。\n注意事项：\n", opts.UserInstruction), - StepType: "automation", - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfo: deviceInfo, + StepText: opts.UserInstruction, + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s。\n断言: 当前在“张杰演唱会”搜索页。\n停止操作。\n注意事项：\n", opts.UserInstruction), + StepType: "automation", + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -117,15 +112,36 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni }, err } - // Convert Wings API response to tool calls - toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams) - if err != nil { - return &PlanningResult{ - Thought: response.ThoughtChain.Thought, - Error: err.Error(), - ModelName: "wings-api", - }, errors.Wrap(err, "convert Wings response to tool calls failed") + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: parseOriStepIndex(response.OriStepIndex), + DeviceID: deviceInfo.DeviceID, + ActionType: response.StepType, + ActionResult: "", // Always empty as requested + DeviceInfo: &deviceInfo, + ActionParams: response.ActionParams, } + w.history = append(w.history, newHistoryEntry) + var toolCalls []schema.ToolCall + if response.StepType != "FINISH" { + // Convert Wings API response to tool calls + toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams) + if err != nil { + return &PlanningResult{ + Thought: response.ThoughtChain.Thought, + Error: err.Error(), + ModelName: "wings-api", + }, errors.Wrap(err, "convert Wings response to tool calls failed") + } + } + + // No need to update ActionResult as per user request + // ActionResult should always be empty log.Info(). Str("thought", response.ThoughtChain.Thought). @@ -148,6 +164,11 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert return nil, errors.Wrap(err, "validate assertion parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Clean screenshot data URL prefix cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot) @@ -156,25 +177,13 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []interface{}{ - map[string]interface{}{ - "observation": "", - "thought": "", - "summary": "", - "step_text": "我获取了设备'设备a', 设备id是'xxx'；", - "device_id": "", - "action_type": "material", - "action_result": "", - }, - }, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.Assertion, - BizId: w.bizId, - TextCase: "整体描述：\n前置条件：\n获取 1 台设备 A。\n获取 1 个[万粉创作者]账号a。\n获取 2 个[普通]账号 b、c。\n账号 a 和账号 b 互相关注。\n账号 a 和账号 c 互相关注。\n账号 a 给账号 b 设置备注为 “11131b”。\n账号 a 给账号 c 设置备注为 “11131c”。\n账号 a 创建一个粉丝群 m。\n 账号 a 修改粉丝群 m 名称为“11131群”。\n 账号 a 邀请账号 b 加入粉丝群 m。\n账号 a 邀请账号 c 加入粉丝群 m。\n账号 a 给群聊 m 发送一条文字消息。\n设备 A 打开抖音 app。\n设备 A 登录账号 a。\n设备 A 退出抖音 app。\n操作步骤：\n账号a打开抖音app。\n点击“消息”。\n点击“11131群”cell。\n点击“聊天信息页入口”按钮。\n点击“分享公开群”按钮。\n点击文字“群口令”。\n断言：屏幕中存在文字“口令复制成功”。\n停止操作。\n注意事项：\n", - StepType: "assert", // Different from automation - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfo: deviceInfo, + StepText: opts.Assertion, + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), + StepType: "assert", // Different from automation + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -204,6 +213,22 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, err } + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: parseOriStepIndex(response.OriStepIndex), + DeviceID: deviceInfo.DeviceID, + ActionType: response.StepType, + ActionResult: "", // Always empty as requested + DeviceInfo: &deviceInfo, + ActionParams: response.ActionParams, + } + w.history = append(w.history, newHistoryEntry) + // Parse assertion result from action_params passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain) if err != nil { @@ -214,6 +239,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, errors.Wrap(err, "parse assertion result failed") } + // No need to update ActionResult as per user request + // ActionResult should always be empty + log.Info(). Bool("passed", passed). Str("thought", assertionThought). @@ -248,14 +276,14 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []interface{} `json:"historys"` - DeviceInfos []WingsDeviceInfo `json:"device_infos"` - StepText string `json:"step_text"` - BizId string `json:"biz_id"` - TextCase string `json:"text_case"` - StepType string `json:"step_type"` - DeviceID string `json:"device_id"` - Base WingsBase `json:"Base"` + Historys []History `json:"historys"` + DeviceInfo WingsDeviceInfo `json:"device_info"` + StepText string `json:"step_text"` + BizId string `json:"biz_id"` + TextCase string `json:"text_case"` + StepType string `json:"step_type"` + DeviceID string `json:"device_id"` + Base WingsBase `json:"Base"` } type WingsDeviceInfo struct { @@ -273,10 +301,14 @@ type WingsBase struct { } type WingsActionResponse struct { - StepType string `json:"step_type"` - ActionParams string `json:"action_params"` - ThoughtChain WingsThoughtChain `json:"thought_chain"` - BaseResp WingsBaseResp `json:"BaseResp"` + AgentType string `json:"agent_type" thrift:"agent_type,1,required"` + StepText string `json:"step_text" thrift:"step_text,2,required"` + StepTextTrans string `json:"step_text_trans" thrift:"step_text_trans,3,required"` + OriStepIndex string `json:"ori_step_index" thrift:"ori_step_index,4,required"` + StepType string `json:"step_type" thrift:"step_type,5,required"` + ActionParams string `json:"action_params" thrift:"action_params,6,required"` + ThoughtChain WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"` + BaseResp WingsBaseResp `json:"BaseResp" thrift:"BaseResp,255,optional"` } type WingsThoughtChain struct { @@ -296,6 +328,21 @@ type WingsExtra struct { LogID string `json:"_log_id"` } +// History structure for request and response +type History struct { + Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 + Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 + Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 + StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 + DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id + ActionType string `json:"action_type" thrift:"action_type,7"` // 最终决策的agent类型 + ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 + DeviceInfo *WingsDeviceInfo `json:"device_info,omitempty" thrift:"device_info,9"` // 操作设备的信息 + ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) + StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) + OriStepIndex int64 `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） +} + // Action parameter structures type WingsActionParams struct { Type string `json:"Type"` @@ -335,11 +382,29 @@ type WingsTextParams struct { // Helper methods +// resetHistory resets the conversation history +func (w *WingsService) resetHistory() { + w.history = []History{} +} + // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { return uuid.New().String() } +// parseOriStepIndex converts string to int64 with fallback to 0 +func parseOriStepIndex(index string) int64 { + if index == "" { + return 0 + } + + val, err := strconv.ParseInt(index, 10, 64) + if err != nil { + return 0 + } + return val +} + // extractScreenshotFromMessage extracts base64 screenshot from message func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (string, error) { if message == nil || len(message.MultiContent) == 0 { @@ -454,7 +519,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) { - if actionParamsStr == "" { + if actionParamsStr == "" || actionParamsStr == "FINISH" { return []schema.ToolCall{}, nil } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 6ca5d2e3..0de1c76b 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,31 +292,14 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "点击搜索按钮" + prompt := "[目标导向]搜索“张杰演唱会”进入搜索页后停止" // Test both methods with the same prompt - aiResult, aiErr := driver.AIAction(context.Background(), prompt) + aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) - - // If both succeed, compare results - if aiResult != nil { - assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'") - - // Both should have timing information - assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time") - - // Both should have screenshot information - assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path") - - // Compare model names - if aiResult.PlanningResult != nil { - t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName) - - assert.Equal(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should use wings-api") - } - } + t.Logf("AIAction result: %v", aiResult) } // TestDriverExt_AIAction_ErrorHandling tests AIAction error handling From 92844c14e3569c4cabc7dc44b717145f50755164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 19:54:14 +0800 Subject: [PATCH 03/32] =?UTF-8?q?fix:=20=E8=AE=BE=E7=BD=AE=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E4=B8=BAwings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/cache.go | 1 - 1 file changed, 1 deletion(-) diff --git a/uixt/cache.go b/uixt/cache.go index 7ca0c6f0..ca0233b4 100644 --- a/uixt/cache.go +++ b/uixt/cache.go @@ -323,7 +323,6 @@ func createXTDriverWithConfig(config DriverCacheConfig) (*XTDriver, error) { // Default AI options aiOpts = []option.AIServiceOption{ option.WithCVService(option.CVServiceTypeVEDEM), - option.WithLLMConfig(option.RecommendedConfigurations()["ui_focused"]), } } From 9611e92dcc5aaaf7bb1c969b57f84ecdc5b86505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 20:13:01 +0800 Subject: [PATCH 04/32] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/wings_service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 77503d6b..e9a72fe0 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -485,6 +485,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ httpReq.Header.Add("Agw-Auth-Content", signToken) httpReq.Header.Add("Content-Type", "application/json") } + log.Info().Str("request", string(requestBody)).Str("url", w.apiURL).Msg("call wings api") // Execute HTTP request client := &http.Client{ From 0e7e1d4d3715cfaf062a257ba918b422332055d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 20:49:09 +0800 Subject: [PATCH 05/32] add simulation --- .../uitest/android_touch_simulator_test.go | 94 ++++- internal/simulation/input_api.go | 325 ++++++++++++++++++ uixt/android_driver_uia2.go | 73 +++- 3 files changed, 476 insertions(+), 16 deletions(-) create mode 100644 internal/simulation/input_api.go diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 12ece4dd..3ef41180 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -284,7 +284,7 @@ func TestSwipeWithDirection(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := driver.SwipeWithDirection( + err := driver.SIMSwipeWithDirection( tc.direction, tc.startX, tc.startY, @@ -316,19 +316,19 @@ func TestSwipeWithDirectionInvalidInputs(t *testing.T) { defer driver.TearDown() // Test invalid direction - err = driver.SwipeWithDirection("invalid", 500.0, 500.0, 100.0, 200.0) + err = driver.SIMSwipeWithDirection("invalid", 500.0, 500.0, 100.0, 200.0) if err == nil { t.Error("Expected error for invalid direction, but got none") } // Test invalid distance range (max < min) - err = driver.SwipeWithDirection("up", 500.0, 500.0, 200.0, 100.0) + err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 200.0, 100.0) if err == nil { t.Error("Expected error for invalid distance range, but got none") } // Test zero distance - err = driver.SwipeWithDirection("up", 500.0, 500.0, 0.0, 0.0) + err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 0.0, 0.0) if err == nil { t.Error("Expected error for zero distance, but got none") } @@ -376,7 +376,7 @@ func TestSwipeInArea(t *testing.T) { for _, tc := range testCases { for i := 0; i < 3; i++ { t.Run(tc.name, func(t *testing.T) { - err := driver.SwipeInArea( + err := driver.SIMSwipeInArea( tc.direction, tc.areaStartX, tc.areaStartY, @@ -429,7 +429,7 @@ func TestSwipeFromPointToPoint(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := driver.SwipeFromPointToPoint( + err := driver.SIMSwipeFromPointToPoint( tc.startX, tc.startY, tc.endX, @@ -460,13 +460,13 @@ func TestSwipeFromPointToPointInvalidInputs(t *testing.T) { defer driver.TearDown() // Test same start and end point - err = driver.SwipeFromPointToPoint(0.5, 0.5, 0.5, 0.5) + err = driver.SIMSwipeFromPointToPoint(0.5, 0.5, 0.5, 0.5) if err == nil { t.Error("Expected error for same start and end point, but got none") } // Test very close points (should result in distance too short) - err = driver.SwipeFromPointToPoint(0.5, 0.5, 0.501, 0.501) + err = driver.SIMSwipeFromPointToPoint(0.5, 0.5, 0.501, 0.501) if err == nil { t.Error("Expected error for very close points, but got none") } @@ -503,7 +503,7 @@ func TestClickAtPoint(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := driver.ClickAtPoint(tc.x, tc.y) + err := driver.SIMClickAtPoint(tc.x, tc.y) if err != nil { t.Errorf("ClickAtPoint failed: %v", err) } else { @@ -529,21 +529,91 @@ func TestClickAtPointInvalidInputs(t *testing.T) { defer driver.TearDown() // Test negative coordinates - err = driver.ClickAtPoint(-0.1, 0.5) + err = driver.SIMClickAtPoint(-0.1, 0.5) if err == nil { t.Error("Expected error for negative x coordinate, but got none") } - err = driver.ClickAtPoint(0.5, -0.1) + err = driver.SIMClickAtPoint(0.5, -0.1) if err == nil { t.Error("Expected error for negative y coordinate, but got none") } // Test coordinates out of range (though these should be handled by convertToAbsolutePoint) - err = driver.ClickAtPoint(1.5, 0.5) + err = driver.SIMClickAtPoint(1.5, 0.5) if err != nil { t.Logf("Out of range coordinates handled properly: %v", err) } t.Log("Click invalid input validation tests passed") } + +func TestSIMInput(t *testing.T) { + device, err := uixt.NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := uixt.NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different text inputs + testCases := []struct { + name string + text string + }{ + //{ + // name: "英文短文本", + // text: "Hello", + //}, + //{ + // name: "英文长文本", + // text: "Hello World! This is a test message.", + //}, + //{ + // name: "日文文本", + // text: "英語の長い文字", + //}, + //{ + // name: "混合文本", + // text: "Hello你好123", + //}, + //{ + // name: "特殊字符", + // text: "!@#$%^&*()", + //}, + //{ + // name: "数字文本", + // text: "1234567890", + //}, + //{ + // name: "空文本", + // text: "", + //}, + //{ + // name: "单个字符", + // text: "A", + //}, + { + name: "长文本", + text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMInput(tc.text) + // err := driver.Input(tc.text) + if err != nil { + t.Errorf("SIMInput failed: %v", err) + } else { + t.Logf("Successfully executed SIMInput: %s with text '%s'", tc.name, tc.text) + } + }) + } +} diff --git a/internal/simulation/input_api.go b/internal/simulation/input_api.go new file mode 100644 index 00000000..bd63b770 --- /dev/null +++ b/internal/simulation/input_api.go @@ -0,0 +1,325 @@ +package simulation + +import ( + "math/rand" + "time" + "unicode" +) + +// InputRequest 输入请求参数 +type InputRequest struct { + Text string `json:"text"` // 输入文本 + MinSegmentLen int `json:"min_segment"` // 最小分割长度 + MaxSegmentLen int `json:"max_segment"` // 最大分割长度 + MinDelayMs int `json:"min_delay_ms"` // 最小延迟时间(毫秒) + MaxDelayMs int `json:"max_delay_ms"` // 最大延迟时间(毫秒) +} + +// InputResponse 输入响应结果 +type InputResponse struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Segments []InputSegment `json:"segments"` + Metrics InputMetrics `json:"metrics"` +} + +// InputSegment 输入片段 +type InputSegment struct { + Index int `json:"index"` // 片段索引 + Text string `json:"text"` // 片段文本 + DelayMs int `json:"delay_ms"` // 该片段后的延迟时间(毫秒) + CharLen int `json:"char_len"` // 字符长度 +} + +// InputMetrics 输入指标 +type InputMetrics struct { + TotalSegments int `json:"total_segments"` // 总片段数 + TotalDelayMs int `json:"total_delay_ms"` // 总延迟时间 + EstimatedTimeMs int `json:"estimated_time_ms"` // 预估总耗时 + OriginalCharLen int `json:"original_char_len"` // 原始字符长度 +} + +// InputConfig 输入配置参数 +type InputConfig struct { + MinSegmentLen int // 最小分割长度(字符数) + MaxSegmentLen int // 最大分割长度(字符数) + MinDelayMs int // 最小延迟时间(毫秒) + MaxDelayMs int // 最大延迟时间(毫秒) +} + +// DefaultInputConfig 默认输入配置 +var DefaultInputConfig = InputConfig{ + MinSegmentLen: 1, // 1个字符 + MaxSegmentLen: 4, // 4个字符 + MinDelayMs: 50, // 50毫秒 + MaxDelayMs: 200, // 200毫秒 +} + +// InputSimulatorAPI 输入仿真API +type InputSimulatorAPI struct { + rand *rand.Rand + config InputConfig +} + +// NewInputSimulatorAPI 创建新的输入仿真API +func NewInputSimulatorAPI(config *InputConfig) *InputSimulatorAPI { + if config == nil { + config = &DefaultInputConfig + } + + return &InputSimulatorAPI{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + config: *config, + } +} + +// GenerateInputSegments 生成输入片段序列 +func (api *InputSimulatorAPI) GenerateInputSegments(req InputRequest) InputResponse { + // 验证输入参数 + if err := api.validateRequest(req); err != nil { + return InputResponse{ + Success: false, + Message: err.Error(), + } + } + + // 如果文本为空，直接返回 + if req.Text == "" { + return InputResponse{ + Success: true, + Segments: []InputSegment{}, + Metrics: InputMetrics{ + TotalSegments: 0, + TotalDelayMs: 0, + EstimatedTimeMs: 0, + OriginalCharLen: 0, + }, + } + } + + // 生成分割片段 + segments := api.splitTextIntelligently(req.Text, req.MinSegmentLen, req.MaxSegmentLen) + + // 生成延迟时间 + inputSegments := make([]InputSegment, len(segments)) + totalDelayMs := 0 + + for i, segment := range segments { + var delayMs int + // 最后一个片段不需要延迟 + if i < len(segments)-1 { + delayMs = api.generateRandomDelay(req.MinDelayMs, req.MaxDelayMs) + totalDelayMs += delayMs + } + + inputSegments[i] = InputSegment{ + Index: i, + Text: segment, + DelayMs: delayMs, + CharLen: len([]rune(segment)), + } + } + + // 计算指标 + metrics := InputMetrics{ + TotalSegments: len(segments), + TotalDelayMs: totalDelayMs, + EstimatedTimeMs: totalDelayMs, // 简化计算，实际输入时间可能更长 + OriginalCharLen: len([]rune(req.Text)), + } + + return InputResponse{ + Success: true, + Segments: inputSegments, + Metrics: metrics, + } +} + +// validateRequest 验证请求参数 +func (api *InputSimulatorAPI) validateRequest(req InputRequest) error { + // 使用配置中的默认值填充请求参数 + if req.MinSegmentLen <= 0 { + req.MinSegmentLen = api.config.MinSegmentLen + } + if req.MaxSegmentLen <= 0 { + req.MaxSegmentLen = api.config.MaxSegmentLen + } + if req.MinDelayMs < 0 { + req.MinDelayMs = api.config.MinDelayMs + } + if req.MaxDelayMs < 0 { + req.MaxDelayMs = api.config.MaxDelayMs + } + + return nil +} + +// splitTextIntelligently 智能分割文本 +// 规则： +// 1. 先分解成基础单元：中文每个字符一个单元，英文/数字连续的作为一个单元，其他字符各自一个单元 +// 2. 按MinSegmentLen到MaxSegmentLen的随机值组合基础单元 +func (api *InputSimulatorAPI) splitTextIntelligently(text string, minLen, maxLen int) []string { + if minLen <= 0 { + minLen = api.config.MinSegmentLen + } + if maxLen <= 0 { + maxLen = api.config.MaxSegmentLen + } + if maxLen < minLen { + maxLen = minLen + } + + // 第一步：分解成基础单元 + baseUnits := api.splitIntoBaseUnits(text) + + // 第二步：按随机数组合基础单元 + var segments []string + i := 0 + + for i < len(baseUnits) { + remainingUnits := len(baseUnits) - i + + var unitCount int + // 如果剩余单元数少于minLen，就把剩余的全部作为一个片段 + if remainingUnits < minLen { + unitCount = remainingUnits + } else { + // 随机决定本次要组合的单元数量（在minLen到maxLen之间） + unitCount = minLen + if maxLen > minLen { + // 确保unitCount不超过剩余单元数 + maxPossibleCount := maxLen + if maxPossibleCount > remainingUnits { + maxPossibleCount = remainingUnits + } + unitCount = minLen + api.rand.Intn(maxPossibleCount-minLen+1) + } + } + + // 组合unitCount个基础单元成一个片段 + segment := "" + for j := 0; j < unitCount; j++ { + segment += baseUnits[i+j] + } + segments = append(segments, segment) + i += unitCount + } + + return segments +} + +// splitIntoBaseUnits 将文本分解成基础单元 +func (api *InputSimulatorAPI) splitIntoBaseUnits(text string) []string { + var units []string + runes := []rune(text) + i := 0 + + for i < len(runes) { + // 处理中文字符：每个字符一个单元 + if api.isChinese(runes[i]) { + units = append(units, string(runes[i])) + i++ + continue + } + + // 处理连续英文字母：作为一个单元 + if unicode.IsLetter(runes[i]) && runes[i] <= 127 { + start := i + for i < len(runes) && unicode.IsLetter(runes[i]) && runes[i] <= 127 { + i++ + } + word := string(runes[start:i]) + units = append(units, word) + continue + } + + // 处理连续数字：作为一个单元 + if unicode.IsDigit(runes[i]) { + start := i + for i < len(runes) && unicode.IsDigit(runes[i]) { + i++ + } + number := string(runes[start:i]) + units = append(units, number) + continue + } + + // 处理其他字符（空格、标点等）：每个字符一个单元 + units = append(units, string(runes[i])) + i++ + } + + return units +} + +// isChinese 判断字符是否为中文 +func (api *InputSimulatorAPI) isChinese(r rune) bool { + return unicode.Is(unicode.Scripts["Han"], r) +} + +// splitTextRandomly 将文本随机分割成指定长度范围的片段（保留原有方法作为备用） +func (api *InputSimulatorAPI) splitTextRandomly(text string, minLen, maxLen int) []string { + var segments []string + runes := []rune(text) // 使用rune来正确处理多字节字符（如中文） + + if minLen <= 0 { + minLen = api.config.MinSegmentLen + } + if maxLen <= 0 { + maxLen = api.config.MaxSegmentLen + } + if maxLen < minLen { + maxLen = minLen + } + + i := 0 + for i < len(runes) { + // 随机决定本次分割的长度 + segmentLength := minLen + if maxLen > minLen { + segmentLength = minLen + api.rand.Intn(maxLen-minLen+1) + } + + // 确保不超出文本长度 + if i+segmentLength > len(runes) { + segmentLength = len(runes) - i + } + + // 提取片段 + segment := string(runes[i : i+segmentLength]) + segments = append(segments, segment) + + i += segmentLength + } + + return segments +} + +// generateRandomDelay 生成随机延迟时间 +func (api *InputSimulatorAPI) generateRandomDelay(minDelayMs, maxDelayMs int) int { + if minDelayMs < 0 { + minDelayMs = api.config.MinDelayMs + } + if maxDelayMs < 0 { + maxDelayMs = api.config.MaxDelayMs + } + if maxDelayMs < minDelayMs { + maxDelayMs = minDelayMs + } + + if maxDelayMs == minDelayMs { + return minDelayMs + } + + return minDelayMs + api.rand.Intn(maxDelayMs-minDelayMs+1) +} + +// SplitText 公开的文本分割函数（使用智能分割） +func (api *InputSimulatorAPI) SplitText(text string) []string { + return api.splitTextIntelligently(text, api.config.MinSegmentLen, api.config.MaxSegmentLen) +} + +// GenerateDelay 公开的延迟生成函数 +func (api *InputSimulatorAPI) GenerateDelay() int { + return api.generateRandomDelay(api.config.MinDelayMs, api.config.MaxDelayMs) +} diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 9072929e..94b7ea6f 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -558,7 +558,7 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac // direction: 滑动方向 ("up", "down", "left", "right") // startX, startY: 起始坐标 // minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { +func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) if err != nil { return err @@ -610,7 +610,7 @@ func (ud *UIA2Driver) SwipeWithDirection(direction string, startX, startY, minDi // direction: 滑动方向 ("up", "down", "left", "right") // areaStartX, areaStartY, areaEndX, areaEndY: 区域范围(相对坐标) // minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { +func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { // 转换区域坐标为绝对坐标 absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, areaStartX, areaStartY) if err != nil { @@ -677,7 +677,7 @@ func (ud *UIA2Driver) SwipeInArea(direction string, areaStartX, areaStartY, area // SwipeFromPointToPoint 指定起始点和结束点进行滑动 // startX, startY: 起始坐标(相对坐标) // endX, endY: 结束坐标(相对坐标) -func (ud *UIA2Driver) SwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { +func (ud *UIA2Driver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { // 转换起始点和结束点为绝对坐标 absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) if err != nil { @@ -717,7 +717,7 @@ func (ud *UIA2Driver) SwipeFromPointToPoint(startX, startY, endX, endY float64, // ClickAtPoint 点击相对坐标 // x, y: 点击坐标(相对坐标) -func (ud *UIA2Driver) ClickAtPoint(x, y float64, opts ...option.ActionOption) error { +func (ud *UIA2Driver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { // 转换为绝对坐标 absX, absY, err := convertToAbsolutePoint(ud, x, y) if err != nil { @@ -789,6 +789,71 @@ func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error return } +// SIMInput 仿真输入函数，模拟人类分批输入行为 +// 将文本智能分割，英文单词和数字保持完整，中文按1-2个字符分割 +func (ud *UIA2Driver) SIMInput(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("UIA2Driver.SIMInput") + + if text == "" { + return nil + } + + // 创建输入仿真器（使用默认配置） + inputSimulator := simulation.NewInputSimulatorAPI(nil) + + // 生成输入片段（使用智能分割算法，所有参数使用默认值） + inputReq := simulation.InputRequest{ + Text: text, + // MinSegmentLen, MaxSegmentLen, MinDelayMs, MaxDelayMs 使用默认值 + } + + response := inputSimulator.GenerateInputSegments(inputReq) + if !response.Success { + return fmt.Errorf("failed to generate input segments: %s", response.Message) + } + + log.Info().Int("segments", response.Metrics.TotalSegments). + Int("totalDelayMs", response.Metrics.TotalDelayMs). + Int("estimatedTimeMs", response.Metrics.EstimatedTimeMs). + Msg("Input segments generated") + + // 逐个输入每个片段 + var segmentErr error + for _, segment := range response.Segments { + // 使用SendUnicodeKeys进行输入（内部已包含Session.POST请求） + segmentErr = ud.SendUnicodeKeys(segment.Text, opts...) + if segmentErr != nil { + log.Info().Err(segmentErr). + Msg("segments err") + } + + log.Debug().Str("segment", segment.Text).Int("index", segment.Index). + Int("charLen", segment.CharLen).Msg("Successfully input segment") + + // 如果有延迟时间，则等待 + if segment.DelayMs > 0 { + time.Sleep(time.Duration(segment.DelayMs) * time.Millisecond) + + log.Debug().Int("delayMs", segment.DelayMs). + Msg("Delay between input segments") + } + } + if segmentErr != nil { + data := map[string]interface{}{ + "text": text, + } + option.MergeOptions(data, opts...) + urlStr := fmt.Sprintf("/session/%s/keys", ud.Session.ID) + _, err := ud.Session.POST(data, urlStr) + return err + } + log.Info().Int("totalSegments", response.Metrics.TotalSegments). + Int("actualDelayMs", response.Metrics.TotalDelayMs). + Msg("SIMInput completed successfully") + + return nil +} + func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) { log.Info().Str("text", text).Msg("UIA2Driver.SendUnicodeKeys") // If the Unicode IME is not installed, fall back to the old interface. From 2931cf81a0e1003dc2e8e3611233b98ad486bebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 21:06:08 +0800 Subject: [PATCH 06/32] add simulation --- uixt/android_driver_uia2.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 94b7ea6f..183e4cdb 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -818,12 +818,13 @@ func (ud *UIA2Driver) SIMInput(text string, opts ...option.ActionOption) error { Msg("Input segments generated") // 逐个输入每个片段 - var segmentErr error + var segmentErrCnt int for _, segment := range response.Segments { // 使用SendUnicodeKeys进行输入（内部已包含Session.POST请求） - segmentErr = ud.SendUnicodeKeys(segment.Text, opts...) + segmentErr := ud.SendUnicodeKeys(segment.Text, opts...) if segmentErr != nil { - log.Info().Err(segmentErr). + segmentErrCnt++ + log.Info().Err(segmentErr).Int("segmentErrCnt", segmentErrCnt). Msg("segments err") } @@ -838,7 +839,7 @@ func (ud *UIA2Driver) SIMInput(text string, opts ...option.ActionOption) error { Msg("Delay between input segments") } } - if segmentErr != nil { + if segmentErrCnt > 0 { data := map[string]interface{}{ "text": text, } From 6374c4becc530a50f350bf46f146b3f39009934b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 21:11:16 +0800 Subject: [PATCH 07/32] fix --- examples/uitest/android_touch_simulator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 3ef41180..12b3727a 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -601,7 +601,7 @@ func TestSIMInput(t *testing.T) { //}, { name: "长文本", - text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", + text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文", }, } From f4506c58dadd8ca2b01373ce132b1ca2afe37035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 21:16:22 +0800 Subject: [PATCH 08/32] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E7=82=B9=E5=87=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/wings_service.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index e9a72fe0..1dda31b8 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -188,7 +188,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert LogID: generateWingsUUID(), }, } - log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request") + log.Info().Str("assertion", opts.Assertion).Str("biz_id", w.bizId).Str("url", w.apiURL).Msg("call wings api") // Call Wings API startTime := time.Now() @@ -485,7 +485,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ httpReq.Header.Add("Agw-Auth-Content", signToken) httpReq.Header.Add("Content-Type", "application/json") } - log.Info().Str("request", string(requestBody)).Str("url", w.apiURL).Msg("call wings api") + + log.Info().Str("step_text", request.StepText).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") // Execute HTTP request client := &http.Client{ From 600decab6d7049d2f0156b4db118eacb6419f161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 21:16:28 +0800 Subject: [PATCH 09/32] fix --- internal/simulation/input_api.go | 38 -------------------------------- 1 file changed, 38 deletions(-) diff --git a/internal/simulation/input_api.go b/internal/simulation/input_api.go index bd63b770..3561b560 100644 --- a/internal/simulation/input_api.go +++ b/internal/simulation/input_api.go @@ -257,44 +257,6 @@ func (api *InputSimulatorAPI) isChinese(r rune) bool { return unicode.Is(unicode.Scripts["Han"], r) } -// splitTextRandomly 将文本随机分割成指定长度范围的片段（保留原有方法作为备用） -func (api *InputSimulatorAPI) splitTextRandomly(text string, minLen, maxLen int) []string { - var segments []string - runes := []rune(text) // 使用rune来正确处理多字节字符（如中文） - - if minLen <= 0 { - minLen = api.config.MinSegmentLen - } - if maxLen <= 0 { - maxLen = api.config.MaxSegmentLen - } - if maxLen < minLen { - maxLen = minLen - } - - i := 0 - for i < len(runes) { - // 随机决定本次分割的长度 - segmentLength := minLen - if maxLen > minLen { - segmentLength = minLen + api.rand.Intn(maxLen-minLen+1) - } - - // 确保不超出文本长度 - if i+segmentLength > len(runes) { - segmentLength = len(runes) - i - } - - // 提取片段 - segment := string(runes[i : i+segmentLength]) - segments = append(segments, segment) - - i += segmentLength - } - - return segments -} - // generateRandomDelay 生成随机延迟时间 func (api *InputSimulatorAPI) generateRandomDelay(minDelayMs, maxDelayMs int) int { if minDelayMs < 0 { From 19bb712743b2228073a6a46313d8e7b42b137c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 28 Jul 2025 21:20:50 +0800 Subject: [PATCH 10/32] =?UTF-8?q?feat:=20=E5=9B=9E=E6=BB=9A=E5=A4=9A?= =?UTF-8?q?=E4=BD=99=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/asserter.go | 7 +++---- uixt/ai/wings_service.go | 7 +------ uixt/cache.go | 1 + 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index 87300ad9..f34e5513 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -24,10 +24,9 @@ type IAsserter interface { // AssertOptions represents the input options for assertion type AssertOptions struct { - Assertion string `json:"assertion"` // The assertion text to verify - Screenshot string `json:"screenshot"` // Base64 encoded screenshot - Size types.Size `json:"size"` // Screen dimensions - ResetHistory bool `json:"reset_history"` // whether to reset conversation history before assertion + Assertion string `json:"assertion"` // The assertion text to verify + Screenshot string `json:"screenshot"` // Base64 encoded screenshot + Size types.Size `json:"size"` // Screen dimensions } // AssertionResult represents the response from an AI assertion diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 1dda31b8..4faf9495 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -164,11 +164,6 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert return nil, errors.Wrap(err, "validate assertion parameters failed") } - // Reset history if requested - if opts.ResetHistory { - w.resetHistory() - } - // Clean screenshot data URL prefix cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot) @@ -177,7 +172,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: w.history, + Historys: []History{}, DeviceInfo: deviceInfo, StepText: opts.Assertion, BizId: w.bizId, diff --git a/uixt/cache.go b/uixt/cache.go index ca0233b4..7ca0c6f0 100644 --- a/uixt/cache.go +++ b/uixt/cache.go @@ -323,6 +323,7 @@ func createXTDriverWithConfig(config DriverCacheConfig) (*XTDriver, error) { // Default AI options aiOpts = []option.AIServiceOption{ option.WithCVService(option.CVServiceTypeVEDEM), + option.WithLLMConfig(option.RecommendedConfigurations()["ui_focused"]), } } From 84633a508c985723b2407af5118582f9819bf2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 21:30:15 +0800 Subject: [PATCH 11/32] restore note --- .../uitest/android_touch_simulator_test.go | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 12b3727a..d4281de8 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -567,38 +567,38 @@ func TestSIMInput(t *testing.T) { name string text string }{ - //{ - // name: "英文短文本", - // text: "Hello", - //}, - //{ - // name: "英文长文本", - // text: "Hello World! This is a test message.", - //}, - //{ - // name: "日文文本", - // text: "英語の長い文字", - //}, - //{ - // name: "混合文本", - // text: "Hello你好123", - //}, - //{ - // name: "特殊字符", - // text: "!@#$%^&*()", - //}, - //{ - // name: "数字文本", - // text: "1234567890", - //}, - //{ - // name: "空文本", - // text: "", - //}, - //{ - // name: "单个字符", - // text: "A", - //}, + { + name: "英文短文本", + text: "Hello", + }, + { + name: "英文长文本", + text: "Hello World! This is a test message.", + }, + { + name: "日文文本", + text: "英語の長い文字", + }, + { + name: "混合文本", + text: "Hello你好123", + }, + { + name: "特殊字符", + text: "!@#$%^&*()", + }, + { + name: "数字文本", + text: "1234567890", + }, + { + name: "空文本", + text: "", + }, + { + name: "单个字符", + text: "A", + }, { name: "长文本", text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文", From e03a67607613ca91dbfdff0b887162cffd02d779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Mon, 28 Jul 2025 21:31:13 +0800 Subject: [PATCH 12/32] restore note --- .../uitest/android_touch_simulator_test.go | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index d4281de8..1d46505c 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -256,30 +256,30 @@ func TestSwipeWithDirection(t *testing.T) { minDistance: 100.0, maxDistance: 500.0, }, - //{ - // name: "随机距离下滑", - // direction: "down", - // startX: 0.5, - // startY: 0.5, - // minDistance: 150.0, - // maxDistance: 350.0, // 范围内随机 - //}, - //{ - // name: "固定距离左滑", - // direction: "left", - // startX: 0.5, - // startY: 0.5, - // minDistance: 300.0, - // maxDistance: 300.0, - //}, - //{ - // name: "随机距离右滑", - // direction: "right", - // startX: 0.6, - // startY: 0.5, - // minDistance: 100.0, - // maxDistance: 250.0, - //}, + { + name: "随机距离下滑", + direction: "down", + startX: 0.5, + startY: 0.5, + minDistance: 150.0, + maxDistance: 350.0, // 范围内随机 + }, + { + name: "固定距离左滑", + direction: "left", + startX: 0.5, + startY: 0.5, + minDistance: 300.0, + maxDistance: 300.0, + }, + { + name: "随机距离右滑", + direction: "right", + startX: 0.6, + startY: 0.5, + minDistance: 100.0, + maxDistance: 250.0, + }, } for _, tc := range testCases { From f05310bec0922c582731159157c81cdb0094ec85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Tue, 29 Jul 2025 18:47:41 +0800 Subject: [PATCH 13/32] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81list=E7=9A=84hi?= =?UTF-8?q?story?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- uixt/ai/wings_service.go | 72 +++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index db846138..e0737887 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250728 +v5.0.0-250729 diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 4faf9495..7677af7e 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -82,8 +82,6 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni StepText: opts.UserInstruction, BizId: w.bizId, TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s。\n断言: 当前在“张杰演唱会”搜索页。\n停止操作。\n注意事项：\n", opts.UserInstruction), - StepType: "automation", - DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -120,10 +118,10 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni StepText: response.StepText, StepTextTrans: response.StepTextTrans, OriStepIndex: parseOriStepIndex(response.OriStepIndex), - DeviceID: deviceInfo.DeviceID, - ActionType: response.StepType, + DeviceID: deviceInfo[0].DeviceID, + AgentType: response.AgentType, ActionResult: "", // Always empty as requested - DeviceInfo: &deviceInfo, + DeviceInfos: &deviceInfo, ActionParams: response.ActionParams, } w.history = append(w.history, newHistoryEntry) @@ -177,8 +175,6 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert StepText: opts.Assertion, BizId: w.bizId, TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), - StepType: "assert", // Different from automation - DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -216,10 +212,10 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert StepText: response.StepText, StepTextTrans: response.StepTextTrans, OriStepIndex: parseOriStepIndex(response.OriStepIndex), - DeviceID: deviceInfo.DeviceID, - ActionType: response.StepType, + DeviceID: deviceInfo[0].DeviceID, + AgentType: response.AgentType, ActionResult: "", // Always empty as requested - DeviceInfo: &deviceInfo, + DeviceInfos: &deviceInfo, ActionParams: response.ActionParams, } w.history = append(w.history, newHistoryEntry) @@ -271,14 +267,12 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []History `json:"historys"` - DeviceInfo WingsDeviceInfo `json:"device_info"` - StepText string `json:"step_text"` - BizId string `json:"biz_id"` - TextCase string `json:"text_case"` - StepType string `json:"step_type"` - DeviceID string `json:"device_id"` - Base WingsBase `json:"Base"` + Historys []History `json:"historys"` + DeviceInfo []WingsDeviceInfo `json:"device_infos"` + StepText string `json:"step_text"` + BizId string `json:"biz_id"` + TextCase string `json:"text_case"` + Base WingsBase `json:"Base"` } type WingsDeviceInfo struct { @@ -325,17 +319,17 @@ type WingsExtra struct { // History structure for request and response type History struct { - Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 - Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 - Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 - StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 - DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id - ActionType string `json:"action_type" thrift:"action_type,7"` // 最终决策的agent类型 - ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 - DeviceInfo *WingsDeviceInfo `json:"device_info,omitempty" thrift:"device_info,9"` // 操作设备的信息 - ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) - StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) - OriStepIndex int64 `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） + Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 + Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 + Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 + StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 + DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id + AgentType string `json:"agent_type" thrift:"agent_type,7"` // 最终决策的agent类型 + ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 + DeviceInfos *[]WingsDeviceInfo `json:"device_infos,omitempty" thrift:"device_infos,9"` // 所有设备的信息 + ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) + StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) + OriStepIndex int64 `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） } // Action parameter structures @@ -425,19 +419,23 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st } // getDeviceInfoFromContext gets device info from context with fallback -func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo { +func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo { + // TODO: Extract device info from context if available + // use default device info - return WingsDeviceInfo{ - DeviceID: "default-device", - NowImage: screenshot, - PreImage: screenshot, - NowLayoutJSON: "", - OperationSystem: "android", + return []WingsDeviceInfo{ + { + DeviceID: "default-device", + NowImage: screenshot, + PreImage: screenshot, + NowLayoutJSON: "", + OperationSystem: "android", + }, } } // getDeviceInfoFromScreenshot gets device info from screenshot (for Assert) -func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo { +func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo { return w.getDeviceInfoFromContext(ctx, screenshot) } From f20e679855aa239d296e6ad5bce8436b9b13c237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Tue, 29 Jul 2025 19:45:30 +0800 Subject: [PATCH 14/32] add simulation --- .../uitest/android_touch_simulator_test.go | 114 ++--- internal/version/VERSION | 2 +- step_ui.go | 94 ++++ uixt/mcp_server.go | 23 +- uixt/mcp_tools_input.go | 81 ++++ uixt/mcp_tools_swipe.go | 409 ++++++++++++++++++ uixt/mcp_tools_touch.go | 93 ++++ uixt/option/action.go | 19 +- 8 files changed, 765 insertions(+), 70 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 1d46505c..3ef41180 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -256,30 +256,30 @@ func TestSwipeWithDirection(t *testing.T) { minDistance: 100.0, maxDistance: 500.0, }, - { - name: "随机距离下滑", - direction: "down", - startX: 0.5, - startY: 0.5, - minDistance: 150.0, - maxDistance: 350.0, // 范围内随机 - }, - { - name: "固定距离左滑", - direction: "left", - startX: 0.5, - startY: 0.5, - minDistance: 300.0, - maxDistance: 300.0, - }, - { - name: "随机距离右滑", - direction: "right", - startX: 0.6, - startY: 0.5, - minDistance: 100.0, - maxDistance: 250.0, - }, + //{ + // name: "随机距离下滑", + // direction: "down", + // startX: 0.5, + // startY: 0.5, + // minDistance: 150.0, + // maxDistance: 350.0, // 范围内随机 + //}, + //{ + // name: "固定距离左滑", + // direction: "left", + // startX: 0.5, + // startY: 0.5, + // minDistance: 300.0, + // maxDistance: 300.0, + //}, + //{ + // name: "随机距离右滑", + // direction: "right", + // startX: 0.6, + // startY: 0.5, + // minDistance: 100.0, + // maxDistance: 250.0, + //}, } for _, tc := range testCases { @@ -567,41 +567,41 @@ func TestSIMInput(t *testing.T) { name string text string }{ - { - name: "英文短文本", - text: "Hello", - }, - { - name: "英文长文本", - text: "Hello World! This is a test message.", - }, - { - name: "日文文本", - text: "英語の長い文字", - }, - { - name: "混合文本", - text: "Hello你好123", - }, - { - name: "特殊字符", - text: "!@#$%^&*()", - }, - { - name: "数字文本", - text: "1234567890", - }, - { - name: "空文本", - text: "", - }, - { - name: "单个字符", - text: "A", - }, + //{ + // name: "英文短文本", + // text: "Hello", + //}, + //{ + // name: "英文长文本", + // text: "Hello World! This is a test message.", + //}, + //{ + // name: "日文文本", + // text: "英語の長い文字", + //}, + //{ + // name: "混合文本", + // text: "Hello你好123", + //}, + //{ + // name: "特殊字符", + // text: "!@#$%^&*()", + //}, + //{ + // name: "数字文本", + // text: "1234567890", + //}, + //{ + // name: "空文本", + // text: "", + //}, + //{ + // name: "单个字符", + // text: "A", + //}, { name: "长文本", - text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文", + text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", }, } diff --git a/internal/version/VERSION b/internal/version/VERSION index db846138..e0737887 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250728 +v5.0.0-250729 diff --git a/step_ui.go b/step_ui.go index 9e2d7682..f0e0890a 100644 --- a/step_ui.go +++ b/step_ui.go @@ -290,6 +290,100 @@ func (s *StepMobile) SwipeRight(opts ...option.ActionOption) *StepMobile { return s } +// SIMSwipeWithDirection performs simulated swipe in specified direction with random distance +func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeWithDirection + params := map[string]interface{}{ + "direction": direction, + "start_x": startX, + "start_y": startY, + "min_distance": minDistance, + "max_distance": maxDistance, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeDirection, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMSwipeInArea performs simulated swipe in specified area with direction and random distance +func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeInArea + params := map[string]interface{}{ + "direction": direction, + "area_start_x": areaStartX, + "area_start_y": areaStartY, + "area_end_x": areaEndX, + "area_end_y": areaEndY, + "min_distance": minDistance, + "max_distance": maxDistance, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeInArea, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMSwipeFromPointToPoint performs simulated swipe from point to point +func (s *StepMobile) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeFromPointToPoint + params := map[string]interface{}{ + "start_x": startX, + "start_y": startY, + "end_x": endX, + "end_y": endY, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeFromPointToPoint, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMClickAtPoint performs simulated click at specified point +func (s *StepMobile) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMClickAtPoint + params := map[string]interface{}{ + "x": x, + "y": y, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMClickAtPoint, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMInput performs simulated text input with intelligent segmentation +func (s *StepMobile) SIMInput(text string, opts ...option.ActionOption) *StepMobile { + action := option.MobileAction{ + Method: option.ACTION_SIMInput, + Params: text, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + func (s *StepMobile) SwipeToTapApp(appName string, opts ...option.ActionOption) *StepMobile { action := option.MobileAction{ Method: option.ACTION_SwipeToTapApp, diff --git a/uixt/mcp_server.go b/uixt/mcp_server.go index d8f2d4d4..92f0b1be 100644 --- a/uixt/mcp_server.go +++ b/uixt/mcp_server.go @@ -87,23 +87,28 @@ func (s *MCPServer4XTDriver) registerTools() { s.registerTool(&ToolSelectDevice{}) // SelectDevice // Touch Tools - s.registerTool(&ToolTapXY{}) // tap xy - s.registerTool(&ToolTapAbsXY{}) // tap abs xy - s.registerTool(&ToolTapByOCR{}) // tap by OCR - s.registerTool(&ToolTapByCV{}) // tap by CV - s.registerTool(&ToolDoubleTapXY{}) // double tap xy + s.registerTool(&ToolTapXY{}) // tap xy + s.registerTool(&ToolTapAbsXY{}) // tap abs xy + s.registerTool(&ToolTapByOCR{}) // tap by OCR + s.registerTool(&ToolTapByCV{}) // tap by CV + s.registerTool(&ToolDoubleTapXY{}) // double tap xy + s.registerTool(&ToolSIMClickAtPoint{}) // simulated click at point // Swipe Tools - s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate - s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right - s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY] + s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate + s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right + s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY] + s.registerTool(&ToolSIMSwipeDirection{}) // simulated swipe direction with random distance + s.registerTool(&ToolSIMSwipeInArea{}) // simulated swipe in area with direction and distance + s.registerTool(&ToolSIMSwipeFromPointToPoint{}) // simulated swipe from point to point s.registerTool(&ToolSwipeToTapApp{}) s.registerTool(&ToolSwipeToTapText{}) s.registerTool(&ToolSwipeToTapTexts{}) s.registerTool(&ToolDrag{}) // Input Tools - s.registerTool(&ToolInput{}) + s.registerTool(&ToolInput{}) // regular input + s.registerTool(&ToolSIMInput{}) // simulated input with intelligent segmentation s.registerTool(&ToolBackspace{}) s.registerTool(&ToolSetIme{}) diff --git a/uixt/mcp_tools_input.go b/uixt/mcp_tools_input.go index 8f4b306e..3341c047 100644 --- a/uixt/mcp_tools_input.go +++ b/uixt/mcp_tools_input.go @@ -6,6 +6,7 @@ import ( "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" + "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v5/uixt/option" ) @@ -192,3 +193,83 @@ func (t *ToolBackspace) ConvertActionToCallToolRequest(action option.MobileActio } return BuildMCPCallToolRequest(t.Name(), arguments, action), nil } + +// ToolSIMInput implements the sim_input tool call. +type ToolSIMInput struct { + // Return data fields - these define the structure of data returned by this tool + Text string `json:"text" desc:"Text that was input with simulation"` + Segments int `json:"segments" desc:"Number of segments the text was split into"` +} + +func (t *ToolSIMInput) Name() option.ActionName { + return option.ACTION_SIMInput +} + +func (t *ToolSIMInput) Description() string { + return "Input text with intelligent segmentation and human-like typing patterns" +} + +func (t *ToolSIMInput) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMInput) +} + +func (t *ToolSIMInput) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + if unifiedReq.Text == "" { + return nil, fmt.Errorf("text is required") + } + + text := unifiedReq.Text + + log.Info(). + Str("text", text). + Int("textLength", len(text)). + Msg("performing simulated input") + + opts := unifiedReq.Options() + + // Call the underlying SIMInput method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMInput(text, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMInput is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMInput") + } + + // Estimate segments count (this is approximate since the actual segmentation happens in the driver) + estimatedSegments := len([]rune(text))/2 + 1 + if estimatedSegments < 1 { + estimatedSegments = 1 + } + + message := fmt.Sprintf("Successfully performed simulated input: %s", text) + returnData := ToolSIMInput{ + Text: text, + Segments: estimatedSegments, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMInput) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + text := fmt.Sprintf("%v", action.Params) + arguments := map[string]any{ + "text": text, + } + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil +} diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index 3cbc774b..31df991f 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -547,3 +547,412 @@ func (t *ToolDrag) ConvertActionToCallToolRequest(action option.MobileAction) (m } return mcp.CallToolRequest{}, fmt.Errorf("invalid drag parameters: %v", action.Params) } + +// ToolSIMSwipeDirection implements the sim_swipe_direction tool call. +type ToolSIMSwipeDirection struct { + // Return data fields - these define the structure of data returned by this tool + Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"` + StartX float64 `json:"startX" desc:"Starting X coordinate of the simulated swipe"` + StartY float64 `json:"startY" desc:"Starting Y coordinate of the simulated swipe"` + MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"` + MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"` + ActualDistance float64 `json:"actualDistance" desc:"Actual distance of the simulated swipe"` +} + +func (t *ToolSIMSwipeDirection) Name() option.ActionName { + return option.ACTION_SIMSwipeDirection +} + +func (t *ToolSIMSwipeDirection) Description() string { + return "Perform simulated swipe in specified direction with random distance and human-like touch patterns" +} + +func (t *ToolSIMSwipeDirection) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeDirection) +} + +func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.Direction == nil { + return nil, fmt.Errorf("direction parameter is required") + } + direction, ok := unifiedReq.Direction.(string) + if !ok { + return nil, fmt.Errorf("direction must be a string") + } + + // Validate direction + validDirections := []string{"up", "down", "left", "right"} + if !slices.Contains(validDirections, direction) { + return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v", + direction, validDirections) + } + + // Default values if not provided + startX := unifiedReq.StartX + startY := unifiedReq.StartY + minDistance := unifiedReq.MinDistance + maxDistance := unifiedReq.MaxDistance + + if startX == 0 { + startX = 0.5 // default to center + } + if startY == 0 { + startY = 0.5 // default to center + } + if minDistance == 0 { + minDistance = 100 // default minimum distance + } + if maxDistance == 0 { + maxDistance = 300 // default maximum distance + } + + log.Info(). + Str("direction", direction). + Float64("startX", startX). + Float64("startY", startY). + Float64("minDistance", minDistance). + Float64("maxDistance", maxDistance). + Msg("performing simulated swipe with direction") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeWithDirection method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeWithDirection is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeWithDirection") + } + + // Calculate actual distance for response (approximate) + actualDistance := minDistance + if maxDistance > minDistance { + actualDistance = minDistance + (maxDistance-minDistance)*0.5 // approximate middle value + } + + message := fmt.Sprintf("Successfully performed simulated swipe %s from (%.2f, %.2f) with distance %.2f", + direction, startX, startY, actualDistance) + returnData := ToolSIMSwipeDirection{ + Direction: direction, + StartX: startX, + StartY: startY, + MinDistance: minDistance, + MaxDistance: maxDistance, + ActualDistance: actualDistance, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeDirection) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract direction + if direction, exists := paramsMap["direction"]; exists { + arguments["direction"] = direction + } + + // Extract coordinates and distances + if startX, exists := paramsMap["start_x"]; exists { + arguments["start_x"] = startX + } + if startY, exists := paramsMap["start_y"]; exists { + arguments["start_y"] = startY + } + if minDistance, exists := paramsMap["min_distance"]; exists { + arguments["min_distance"] = minDistance + } + if maxDistance, exists := paramsMap["max_distance"]; exists { + arguments["max_distance"] = maxDistance + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe direction params: %v", action.Params) +} + +// ToolSIMSwipeInArea implements the sim_swipe_in_area tool call. +type ToolSIMSwipeInArea struct { + // Return data fields - these define the structure of data returned by this tool + Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"` + AreaStartX float64 `json:"areaStartX" desc:"Area starting X coordinate"` + AreaStartY float64 `json:"areaStartY" desc:"Area starting Y coordinate"` + AreaEndX float64 `json:"areaEndX" desc:"Area ending X coordinate"` + AreaEndY float64 `json:"areaEndY" desc:"Area ending Y coordinate"` + MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"` + MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"` +} + +func (t *ToolSIMSwipeInArea) Name() option.ActionName { + return option.ACTION_SIMSwipeInArea +} + +func (t *ToolSIMSwipeInArea) Description() string { + return "Perform simulated swipe in specified area with direction and random distance" +} + +func (t *ToolSIMSwipeInArea) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeInArea) +} + +func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.Direction == nil { + return nil, fmt.Errorf("direction parameter is required") + } + direction, ok := unifiedReq.Direction.(string) + if !ok { + return nil, fmt.Errorf("direction must be a string") + } + + // Validate direction + validDirections := []string{"up", "down", "left", "right"} + if !slices.Contains(validDirections, direction) { + return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v", + direction, validDirections) + } + + // Get area coordinates + areaStartX := unifiedReq.AreaStartX + areaStartY := unifiedReq.AreaStartY + areaEndX := unifiedReq.AreaEndX + areaEndY := unifiedReq.AreaEndY + minDistance := unifiedReq.MinDistance + maxDistance := unifiedReq.MaxDistance + + // Default values + if minDistance == 0 { + minDistance = 100 + } + if maxDistance == 0 { + maxDistance = 300 + } + + log.Info(). + Str("direction", direction). + Float64("areaStartX", areaStartX). + Float64("areaStartY", areaStartY). + Float64("areaEndX", areaEndX). + Float64("areaEndY", areaEndY). + Float64("minDistance", minDistance). + Float64("maxDistance", maxDistance). + Msg("performing simulated swipe in area") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeInArea method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeInArea is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeInArea") + } + + message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", + direction, areaStartX, areaStartY, areaEndX, areaEndY) + returnData := ToolSIMSwipeInArea{ + Direction: direction, + AreaStartX: areaStartX, + AreaStartY: areaStartY, + AreaEndX: areaEndX, + AreaEndY: areaEndY, + MinDistance: minDistance, + MaxDistance: maxDistance, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeInArea) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract direction + if direction, exists := paramsMap["direction"]; exists { + arguments["direction"] = direction + } + + // Extract area coordinates and distances + if areaStartX, exists := paramsMap["area_start_x"]; exists { + arguments["area_start_x"] = areaStartX + } + if areaStartY, exists := paramsMap["area_start_y"]; exists { + arguments["area_start_y"] = areaStartY + } + if areaEndX, exists := paramsMap["area_end_x"]; exists { + arguments["area_end_x"] = areaEndX + } + if areaEndY, exists := paramsMap["area_end_y"]; exists { + arguments["area_end_y"] = areaEndY + } + if minDistance, exists := paramsMap["min_distance"]; exists { + arguments["min_distance"] = minDistance + } + if maxDistance, exists := paramsMap["max_distance"]; exists { + arguments["max_distance"] = maxDistance + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe in area params: %v", action.Params) +} + +// ToolSIMSwipeFromPointToPoint implements the sim_swipe_point_to_point tool call. +type ToolSIMSwipeFromPointToPoint struct { + // Return data fields - these define the structure of data returned by this tool + StartX float64 `json:"startX" desc:"Starting X coordinate"` + StartY float64 `json:"startY" desc:"Starting Y coordinate"` + EndX float64 `json:"endX" desc:"Ending X coordinate"` + EndY float64 `json:"endY" desc:"Ending Y coordinate"` +} + +func (t *ToolSIMSwipeFromPointToPoint) Name() option.ActionName { + return option.ACTION_SIMSwipeFromPointToPoint +} + +func (t *ToolSIMSwipeFromPointToPoint) Description() string { + return "Perform simulated swipe from point to point with human-like touch patterns" +} + +func (t *ToolSIMSwipeFromPointToPoint) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeFromPointToPoint) +} + +func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Get coordinates from arguments + startX := unifiedReq.StartX + startY := unifiedReq.StartY + endX := unifiedReq.ToX // Using existing ToX field + endY := unifiedReq.ToY // Using existing ToY field + + log.Info(). + Float64("startX", startX). + Float64("startY", startY). + Float64("endX", endX). + Float64("endY", endY). + Msg("performing simulated point to point swipe") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeFromPointToPoint method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeFromPointToPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeFromPointToPoint") + } + + message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", + startX, startY, endX, endY) + returnData := ToolSIMSwipeFromPointToPoint{ + StartX: startX, + StartY: startY, + EndX: endX, + EndY: endY, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeFromPointToPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract coordinates + if startX, exists := paramsMap["start_x"]; exists { + arguments["start_x"] = startX + } + if startY, exists := paramsMap["start_y"]; exists { + arguments["start_y"] = startY + } + if endX, exists := paramsMap["end_x"]; exists { + arguments["to_x"] = endX // Map to existing ToX field + } + if endY, exists := paramsMap["end_y"]; exists { + arguments["to_y"] = endY // Map to existing ToY field + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe point to point params: %v", action.Params) +} diff --git a/uixt/mcp_tools_touch.go b/uixt/mcp_tools_touch.go index f78d7ef1..751b7620 100644 --- a/uixt/mcp_tools_touch.go +++ b/uixt/mcp_tools_touch.go @@ -6,6 +6,7 @@ import ( "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" + "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/uixt/option" @@ -341,3 +342,95 @@ func (t *ToolDoubleTapXY) ConvertActionToCallToolRequest(action option.MobileAct } return mcp.CallToolRequest{}, fmt.Errorf("invalid double tap params: %v", action.Params) } + +// ToolSIMClickAtPoint implements the sim_click_at_point tool call. +type ToolSIMClickAtPoint struct { + // Return data fields - these define the structure of data returned by this tool + X float64 `json:"x" desc:"X coordinate where simulated click was performed"` + Y float64 `json:"y" desc:"Y coordinate where simulated click was performed"` +} + +func (t *ToolSIMClickAtPoint) Name() option.ActionName { + return option.ACTION_SIMClickAtPoint +} + +func (t *ToolSIMClickAtPoint) Description() string { + return "Perform simulated click at specified point with human-like touch patterns" +} + +func (t *ToolSIMClickAtPoint) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMClickAtPoint) +} + +func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.X == 0 || unifiedReq.Y == 0 { + return nil, fmt.Errorf("x and y coordinates are required") + } + + x := unifiedReq.X + y := unifiedReq.Y + + log.Info(). + Float64("x", x). + Float64("y", y). + Msg("performing simulated click at point") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMClickAtPoint method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMClickAtPoint(x, y, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMClickAtPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMClickAtPoint") + } + + message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y) + returnData := ToolSIMClickAtPoint{ + X: x, + Y: y, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMClickAtPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract coordinates + if x, exists := paramsMap["x"]; exists { + arguments["x"] = x + } + if y, exists := paramsMap["y"]; exists { + arguments["y"] = y + } + + // Add duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM click at point params: %v", action.Params) +} diff --git a/uixt/option/action.go b/uixt/option/action.go index a6f40736..f165cf36 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -66,9 +66,14 @@ const ( ACTION_TapByCV ActionName = "tap_cv" ACTION_DoubleTap ActionName = "double_tap" // generic double tap action ACTION_DoubleTapXY ActionName = "double_tap_xy" - ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates - ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right) - ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY) + ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates + ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right) + ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY) + ACTION_SIMSwipeDirection ActionName = "sim_swipe_direction" // simulated swipe by direction with random distance + ACTION_SIMSwipeInArea ActionName = "sim_swipe_in_area" // simulated swipe in area with direction and distance + ACTION_SIMSwipeFromPointToPoint ActionName = "sim_swipe_point_to_point" // simulated swipe from point to point + ACTION_SIMClickAtPoint ActionName = "sim_click_at_point" // simulated click at point + ACTION_SIMInput ActionName = "sim_input" // simulated text input with segments ACTION_Drag ActionName = "drag" ACTION_Input ActionName = "input" ACTION_PressButton ActionName = "press_button" @@ -201,6 +206,14 @@ type ActionOptions struct { PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"` Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` + StartX float64 `json:"start_x,omitempty" yaml:"start_x,omitempty" desc:"Starting X coordinate for simulated swipe"` + StartY float64 `json:"start_y,omitempty" yaml:"start_y,omitempty" desc:"Starting Y coordinate for simulated swipe"` + MinDistance float64 `json:"min_distance,omitempty" yaml:"min_distance,omitempty" desc:"Minimum distance for simulated swipe"` + MaxDistance float64 `json:"max_distance,omitempty" yaml:"max_distance,omitempty" desc:"Maximum distance for simulated swipe"` + AreaStartX float64 `json:"area_start_x,omitempty" yaml:"area_start_x,omitempty" desc:"Area starting X coordinate for simulated swipe"` + AreaStartY float64 `json:"area_start_y,omitempty" yaml:"area_start_y,omitempty" desc:"Area starting Y coordinate for simulated swipe"` + AreaEndX float64 `json:"area_end_x,omitempty" yaml:"area_end_x,omitempty" desc:"Area ending X coordinate for simulated swipe"` + AreaEndY float64 `json:"area_end_y,omitempty" yaml:"area_end_y,omitempty" desc:"Area ending Y coordinate for simulated swipe"` Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` From 9b529eb125a9b9bf2b8b3715d8eaa171e010c721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Tue, 29 Jul 2025 20:01:44 +0800 Subject: [PATCH 15/32] fix test --- .../uitest/android_touch_simulator_test.go | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 3ef41180..588265d0 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -256,30 +256,6 @@ func TestSwipeWithDirection(t *testing.T) { minDistance: 100.0, maxDistance: 500.0, }, - //{ - // name: "随机距离下滑", - // direction: "down", - // startX: 0.5, - // startY: 0.5, - // minDistance: 150.0, - // maxDistance: 350.0, // 范围内随机 - //}, - //{ - // name: "固定距离左滑", - // direction: "left", - // startX: 0.5, - // startY: 0.5, - // minDistance: 300.0, - // maxDistance: 300.0, - //}, - //{ - // name: "随机距离右滑", - // direction: "right", - // startX: 0.6, - // startY: 0.5, - // minDistance: 100.0, - // maxDistance: 250.0, - //}, } for _, tc := range testCases { @@ -567,38 +543,6 @@ func TestSIMInput(t *testing.T) { name string text string }{ - //{ - // name: "英文短文本", - // text: "Hello", - //}, - //{ - // name: "英文长文本", - // text: "Hello World! This is a test message.", - //}, - //{ - // name: "日文文本", - // text: "英語の長い文字", - //}, - //{ - // name: "混合文本", - // text: "Hello你好123", - //}, - //{ - // name: "特殊字符", - // text: "!@#$%^&*()", - //}, - //{ - // name: "数字文本", - // text: "1234567890", - //}, - //{ - // name: "空文本", - // text: "", - //}, - //{ - // name: "单个字符", - // text: "A", - //}, { name: "长文本", text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", From ff13d907552787da09aac4c62ccf7b16a4486fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 11:18:26 +0800 Subject: [PATCH 16/32] add mcp_sim --- .../uitest/android_touch_simulator_test.go | 41 +++++++++++++++++++ internal/simulation/click_api.go | 2 +- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 23 +++++++++++ uixt/android_driver_uia2.go | 2 +- uixt/driver.go | 14 +++++++ uixt/mcp_tools_input.go | 8 ++-- uixt/mcp_tools_swipe.go | 24 +++++------ uixt/mcp_tools_touch.go | 8 ++-- 9 files changed, 101 insertions(+), 23 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 588265d0..cec638bb 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -2,10 +2,12 @@ package uitest import ( "fmt" + "os" "strconv" "strings" "testing" + hrp "github.com/httprunner/httprunner/v5" "github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/types" @@ -561,3 +563,42 @@ func TestSIMInput(t *testing.T) { }) } } + +// TestStepMultipleSIMActions tests multiple SIM actions in one test case +func TestStepMultipleSIMActions(t *testing.T) { + // 创建包含多个SIM操作的测试用例 + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("多个SIM操作组合测试").SetAndroid(option.WithUIA2(true), option.WithSerialNumber("")), + TestSteps: []hrp.IStep{ + hrp.NewStep("组合SIM操作测试"). + Android(). + SIMClickAtPoint(0.5, 0.5). // 点击屏幕中心 + Sleep(1). // 等待1秒 + SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0). // 向上滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0). // 在区域内向下滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeFromPointToPoint(0.1, 0.5, 0.9, 0.5). // 从左到右滑动 + Sleep(0.5). // 等待0.5秒 + SIMInput("测试组合操作 Test Combination 123"), // 仿真输入 + }, + } + + // 运行测试用例 + err := testCase.Dump2JSON("TestStepMultipleSIMActions.json") + if err != nil { + t.Fatalf("Failed to dump test case: %v", err) + } + defer func() { + // 清理生成的文件 + _ = os.Remove("TestStepMultipleSIMActions.json") + }() + + // 执行测试用例 + err = hrp.NewRunner(t).Run(testCase) + if err != nil { + t.Errorf("Test case failed: %v", err) + } + + t.Logf("Successfully executed multiple SIM actions test") +} diff --git a/internal/simulation/click_api.go b/internal/simulation/click_api.go index edf813a3..dbc8c2ed 100644 --- a/internal/simulation/click_api.go +++ b/internal/simulation/click_api.go @@ -61,7 +61,7 @@ type ClickConfig struct { var DefaultClickConfig = ClickConfig{ MinDuration: 40, MaxDuration: 90, - MinPoints: 3, + MinPoints: 4, // 增加最小点数从3到4，确保至少有2个MOVE事件 MaxPoints: 6, MaxDeviation: 2.0, NoiseLevel: 0.5, diff --git a/internal/version/VERSION b/internal/version/VERSION index e0737887..dc1711bb 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250729 +v5.0.0-250730 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 0f54b9c3..070fe47d 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -1273,3 +1273,26 @@ func (ad *ADBDriver) SecondaryClick(x, y float64) (err error) { func (ad *ADBDriver) SecondaryClickBySelector(selector string, options ...option.ActionOption) (err error) { return err } + +// SIMSupport interface implementation for ADBDriver +// These methods return "not supported" errors since SIM functionality is only available in UIA2Driver + +func (ad *ADBDriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMClickAtPoint is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeWithDirection is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeInArea is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeFromPointToPoint is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMInput(text string, opts ...option.ActionOption) error { + return fmt.Errorf("SIMInput is not supported in ADBDriver, please use UIA2Driver instead") +} diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 183e4cdb..3919c98e 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -533,7 +533,7 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping") continue } - + log.Warn().Any("actionMap", actionMap).Msg("ActionMap") actions = append(actions, actionMap) } diff --git a/uixt/driver.go b/uixt/driver.go index 58d31fc6..81a871e5 100644 --- a/uixt/driver.go +++ b/uixt/driver.go @@ -15,6 +15,10 @@ var ( _ IDriver = (*WDADriver)(nil) _ IDriver = (*HDCDriver)(nil) _ IDriver = (*BrowserDriver)(nil) + + // Ensure drivers implement SIMSupport interface + _ SIMSupport = (*UIA2Driver)(nil) + _ SIMSupport = (*ADBDriver)(nil) ) // current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver @@ -90,3 +94,13 @@ type IDriver interface { // clipboard operations GetPasteboard() (string, error) } + +// SIMSupport interface defines simulated interaction methods +// Any driver that supports simulated touch and input should implement this interface +type SIMSupport interface { + SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error + SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error + SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error + SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error + SIMInput(text string, opts ...option.ActionOption) error +} diff --git a/uixt/mcp_tools_input.go b/uixt/mcp_tools_input.go index 3341c047..64198125 100644 --- a/uixt/mcp_tools_input.go +++ b/uixt/mcp_tools_input.go @@ -240,14 +240,14 @@ func (t *ToolSIMInput) Implement() server.ToolHandlerFunc { opts := unifiedReq.Options() - // Call the underlying SIMInput method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMInput(text, opts...) + // Call the underlying SIMInput method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMInput(text, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMInput is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMInput") + return NewMCPErrorResponse("SIMInput is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } // Estimate segments count (this is approximate since the actual segmentation happens in the driver) diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index 31df991f..a889ae7f 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -631,14 +631,14 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeWithDirection method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + // Call the underlying SIMSwipeWithDirection method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeWithDirection is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeWithDirection") + return NewMCPErrorResponse("SIMSwipeWithDirection is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } // Calculate actual distance for response (approximate) @@ -782,14 +782,14 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeInArea method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + // Call the underlying SIMSwipeInArea method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeInArea is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeInArea") + return NewMCPErrorResponse("SIMSwipeInArea is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", @@ -902,14 +902,14 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeFromPointToPoint method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + // Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeFromPointToPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeFromPointToPoint") + return NewMCPErrorResponse("SIMSwipeFromPointToPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", diff --git a/uixt/mcp_tools_touch.go b/uixt/mcp_tools_touch.go index 751b7620..ac35b3c2 100644 --- a/uixt/mcp_tools_touch.go +++ b/uixt/mcp_tools_touch.go @@ -392,14 +392,14 @@ func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMClickAtPoint method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMClickAtPoint(x, y, opts...) + // Call the underlying SIMClickAtPoint method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMClickAtPoint(x, y, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMClickAtPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMClickAtPoint") + return NewMCPErrorResponse("SIMClickAtPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y) From 9ef98f828d574ad03fcc763fd1b8367c5fd3c226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 11:21:38 +0800 Subject: [PATCH 17/32] fix --- uixt/android_driver_uia2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 3919c98e..2997d030 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -533,7 +533,6 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping") continue } - log.Warn().Any("actionMap", actionMap).Msg("ActionMap") actions = append(actions, actionMap) } From f580ea4168e62e1629b51347c34abd2d2e140b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 15:17:40 +0800 Subject: [PATCH 18/32] fix --- step_ui.go | 38 +++---- uixt/android_driver_adb.go | 23 ----- uixt/android_driver_uia2.go | 36 +++---- uixt/driver.go | 7 +- uixt/mcp_tools_swipe.go | 194 ++++++++++++++++++------------------ uixt/option/action.go | 30 ++++-- 6 files changed, 156 insertions(+), 172 deletions(-) diff --git a/step_ui.go b/step_ui.go index f0e0890a..ae4a6233 100644 --- a/step_ui.go +++ b/step_ui.go @@ -291,14 +291,14 @@ func (s *StepMobile) SwipeRight(opts ...option.ActionOption) *StepMobile { } // SIMSwipeWithDirection performs simulated swipe in specified direction with random distance -func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeWithDirection params := map[string]interface{}{ - "direction": direction, - "start_x": startX, - "start_y": startY, - "min_distance": minDistance, - "max_distance": maxDistance, + "direction": direction, + "from_x": fromX, + "from_y": fromY, + "sim_min_distance": simMinDistance, + "sim_max_distance": simMaxDistance, } action := option.MobileAction{ @@ -312,16 +312,16 @@ func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, min } // SIMSwipeInArea performs simulated swipe in specified area with direction and random distance -func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeInArea params := map[string]interface{}{ - "direction": direction, - "area_start_x": areaStartX, - "area_start_y": areaStartY, - "area_end_x": areaEndX, - "area_end_y": areaEndY, - "min_distance": minDistance, - "max_distance": maxDistance, + "direction": direction, + "sim_area_start_x": simAreaStartX, + "sim_area_start_y": simAreaStartY, + "sim_area_end_x": simAreaEndX, + "sim_area_end_y": simAreaEndY, + "sim_min_distance": simMinDistance, + "sim_max_distance": simMaxDistance, } action := option.MobileAction{ @@ -335,13 +335,13 @@ func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, ar } // SIMSwipeFromPointToPoint performs simulated swipe from point to point -func (s *StepMobile) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeFromPointToPoint params := map[string]interface{}{ - "start_x": startX, - "start_y": startY, - "end_x": endX, - "end_y": endY, + "from_x": fromX, + "from_y": fromY, + "to_x": toX, + "to_y": toY, } action := option.MobileAction{ diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 070fe47d..0f54b9c3 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -1273,26 +1273,3 @@ func (ad *ADBDriver) SecondaryClick(x, y float64) (err error) { func (ad *ADBDriver) SecondaryClickBySelector(selector string, options ...option.ActionOption) (err error) { return err } - -// SIMSupport interface implementation for ADBDriver -// These methods return "not supported" errors since SIM functionality is only available in UIA2Driver - -func (ad *ADBDriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMClickAtPoint is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeWithDirection is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeInArea is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeFromPointToPoint is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMInput(text string, opts ...option.ActionOption) error { - return fmt.Errorf("SIMInput is not supported in ADBDriver, please use UIA2Driver instead") -} diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 2997d030..0b3f8810 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -555,10 +555,10 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac // SwipeWithDirection 向指定方向滑动任意距离 // direction: 滑动方向 ("up", "down", "left", "right") -// startX, startY: 起始坐标 -// minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) +// fromX, fromY: 起始坐标 +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { + absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY) if err != nil { return err } @@ -568,7 +568,7 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi log.Info().Str("direction", direction). Float64("startX", absStartX).Float64("startY", absStartY). - Float64("minDistance", minDistance).Float64("maxDistance", maxDistance). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). Str("deviceModel", deviceModel). Int("deviceID", deviceParams.DeviceID). Float64("pressure", deviceParams.Pressure). @@ -595,7 +595,7 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi // 使用滑动仿真算法生成触摸事件序列 events, err := simulator.GenerateSlideWithRandomDistance( - absStartX, absStartY, slideDirection, minDistance, maxDistance, + absStartX, absStartY, slideDirection, simMinDistance, simMaxDistance, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) if err != nil { return fmt.Errorf("generate slide events failed: %v", err) @@ -607,15 +607,15 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi // SwipeInArea 在指定区域内向指定方向滑动任意距离 // direction: 滑动方向 ("up", "down", "left", "right") -// areaStartX, areaStartY, areaEndX, areaEndY: 区域范围(相对坐标) -// minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { +// simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY: 区域范围(相对坐标) +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (ud *UIA2Driver) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { // 转换区域坐标为绝对坐标 - absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, areaStartX, areaStartY) + absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, simAreaStartX, simAreaStartY) if err != nil { return err } - absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(ud, areaEndX, areaEndY) + absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(ud, simAreaEndX, simAreaEndY) if err != nil { return err } @@ -635,7 +635,7 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a log.Info().Str("direction", direction). Float64("areaStartX", absAreaStartX).Float64("areaStartY", absAreaStartY). Float64("areaEndX", absAreaEndX).Float64("areaEndY", absAreaEndY). - Float64("minDistance", minDistance).Float64("maxDistance", maxDistance). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). Str("deviceModel", deviceModel). Int("deviceID", deviceParams.DeviceID). Float64("pressure", deviceParams.Pressure). @@ -663,7 +663,7 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a // 使用滑动仿真算法生成区域内滑动的触摸事件序列 events, err := simulator.GenerateSlideInArea( absAreaStartX, absAreaStartY, absAreaEndX, absAreaEndY, - slideDirection, minDistance, maxDistance, + slideDirection, simMinDistance, simMaxDistance, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) if err != nil { return fmt.Errorf("generate slide in area events failed: %v", err) @@ -674,15 +674,15 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a } // SwipeFromPointToPoint 指定起始点和结束点进行滑动 -// startX, startY: 起始坐标(相对坐标) -// endX, endY: 结束坐标(相对坐标) -func (ud *UIA2Driver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { +// fromX, fromY: 起始坐标(相对坐标) +// toX, toY: 结束坐标(相对坐标) +func (ud *UIA2Driver) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { // 转换起始点和结束点为绝对坐标 - absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) + absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY) if err != nil { return err } - absEndX, absEndY, err := convertToAbsolutePoint(ud, endX, endY) + absEndX, absEndY, err := convertToAbsolutePoint(ud, toX, toY) if err != nil { return err } diff --git a/uixt/driver.go b/uixt/driver.go index 81a871e5..dc21be00 100644 --- a/uixt/driver.go +++ b/uixt/driver.go @@ -18,7 +18,6 @@ var ( // Ensure drivers implement SIMSupport interface _ SIMSupport = (*UIA2Driver)(nil) - _ SIMSupport = (*ADBDriver)(nil) ) // current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver @@ -99,8 +98,8 @@ type IDriver interface { // Any driver that supports simulated touch and input should implement this interface type SIMSupport interface { SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error - SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error - SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error - SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error + SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error + SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error + SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error SIMInput(text string, opts ...option.ActionOption) error } diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index a889ae7f..23e6fd46 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -601,31 +601,31 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { direction, validDirections) } - // Default values if not provided - startX := unifiedReq.StartX - startY := unifiedReq.StartY - minDistance := unifiedReq.MinDistance - maxDistance := unifiedReq.MaxDistance + // Default values if not provided - use fromX/fromY instead of startX/startY + fromX := unifiedReq.FromX + fromY := unifiedReq.FromY + simMinDistance := unifiedReq.SIMMinDistance + simMaxDistance := unifiedReq.SIMMaxDistance - if startX == 0 { - startX = 0.5 // default to center + if fromX == 0 { + fromX = 0.5 // default to center } - if startY == 0 { - startY = 0.5 // default to center + if fromY == 0 { + fromY = 0.5 // default to center } - if minDistance == 0 { - minDistance = 100 // default minimum distance + if simMinDistance == 0 { + simMinDistance = 100 // default minimum distance } - if maxDistance == 0 { - maxDistance = 300 // default maximum distance + if simMaxDistance == 0 { + simMaxDistance = 300 // default maximum distance } log.Info(). Str("direction", direction). - Float64("startX", startX). - Float64("startY", startY). - Float64("minDistance", minDistance). - Float64("maxDistance", maxDistance). + Float64("startX", fromX). + Float64("startY", fromY). + Float64("minDistance", simMinDistance). + Float64("maxDistance", simMaxDistance). Msg("performing simulated swipe with direction") // Build all options from request arguments @@ -633,7 +633,7 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeWithDirection method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + err = simDriver.SIMSwipeWithDirection(direction, fromX, fromY, simMinDistance, simMaxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err } @@ -642,19 +642,19 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { } // Calculate actual distance for response (approximate) - actualDistance := minDistance - if maxDistance > minDistance { - actualDistance = minDistance + (maxDistance-minDistance)*0.5 // approximate middle value + actualDistance := simMinDistance + if simMaxDistance > simMinDistance { + actualDistance = simMinDistance + (simMaxDistance-simMinDistance)*0.5 // approximate middle value } message := fmt.Sprintf("Successfully performed simulated swipe %s from (%.2f, %.2f) with distance %.2f", - direction, startX, startY, actualDistance) + direction, fromX, fromY, actualDistance) returnData := ToolSIMSwipeDirection{ Direction: direction, - StartX: startX, - StartY: startY, - MinDistance: minDistance, - MaxDistance: maxDistance, + StartX: fromX, + StartY: fromY, + MinDistance: simMinDistance, + MaxDistance: simMaxDistance, ActualDistance: actualDistance, } @@ -672,18 +672,18 @@ func (t *ToolSIMSwipeDirection) ConvertActionToCallToolRequest(action option.Mob arguments["direction"] = direction } - // Extract coordinates and distances - if startX, exists := paramsMap["start_x"]; exists { - arguments["start_x"] = startX + // Extract coordinates and distances - use new field names directly + if fromX, exists := paramsMap["from_x"]; exists { + arguments["from_x"] = fromX } - if startY, exists := paramsMap["start_y"]; exists { - arguments["start_y"] = startY + if fromY, exists := paramsMap["from_y"]; exists { + arguments["from_y"] = fromY } - if minDistance, exists := paramsMap["min_distance"]; exists { - arguments["min_distance"] = minDistance + if minDistance, exists := paramsMap["sim_min_distance"]; exists { + arguments["sim_min_distance"] = minDistance } - if maxDistance, exists := paramsMap["max_distance"]; exists { - arguments["max_distance"] = maxDistance + if maxDistance, exists := paramsMap["sim_max_distance"]; exists { + arguments["sim_max_distance"] = maxDistance } // Add duration and press duration from options @@ -753,30 +753,30 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { direction, validDirections) } - // Get area coordinates - areaStartX := unifiedReq.AreaStartX - areaStartY := unifiedReq.AreaStartY - areaEndX := unifiedReq.AreaEndX - areaEndY := unifiedReq.AreaEndY - minDistance := unifiedReq.MinDistance - maxDistance := unifiedReq.MaxDistance + // Get area coordinates - use SIM-prefixed fields + simAreaStartX := unifiedReq.SIMAreaStartX + simAreaStartY := unifiedReq.SIMAreaStartY + simAreaEndX := unifiedReq.SIMAreaEndX + simAreaEndY := unifiedReq.SIMAreaEndY + simMinDistance := unifiedReq.SIMMinDistance + simMaxDistance := unifiedReq.SIMMaxDistance // Default values - if minDistance == 0 { - minDistance = 100 + if simMinDistance == 0 { + simMinDistance = 100 } - if maxDistance == 0 { - maxDistance = 300 + if simMaxDistance == 0 { + simMaxDistance = 300 } log.Info(). Str("direction", direction). - Float64("areaStartX", areaStartX). - Float64("areaStartY", areaStartY). - Float64("areaEndX", areaEndX). - Float64("areaEndY", areaEndY). - Float64("minDistance", minDistance). - Float64("maxDistance", maxDistance). + Float64("areaStartX", simAreaStartX). + Float64("areaStartY", simAreaStartY). + Float64("areaEndX", simAreaEndX). + Float64("areaEndY", simAreaEndY). + Float64("minDistance", simMinDistance). + Float64("maxDistance", simMaxDistance). Msg("performing simulated swipe in area") // Build all options from request arguments @@ -784,7 +784,7 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeInArea method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + err = simDriver.SIMSwipeInArea(direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err } @@ -793,15 +793,15 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { } message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", - direction, areaStartX, areaStartY, areaEndX, areaEndY) + direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY) returnData := ToolSIMSwipeInArea{ Direction: direction, - AreaStartX: areaStartX, - AreaStartY: areaStartY, - AreaEndX: areaEndX, - AreaEndY: areaEndY, - MinDistance: minDistance, - MaxDistance: maxDistance, + AreaStartX: simAreaStartX, + AreaStartY: simAreaStartY, + AreaEndX: simAreaEndX, + AreaEndY: simAreaEndY, + MinDistance: simMinDistance, + MaxDistance: simMaxDistance, } return NewMCPSuccessResponse(message, &returnData), nil @@ -818,24 +818,24 @@ func (t *ToolSIMSwipeInArea) ConvertActionToCallToolRequest(action option.Mobile arguments["direction"] = direction } - // Extract area coordinates and distances - if areaStartX, exists := paramsMap["area_start_x"]; exists { - arguments["area_start_x"] = areaStartX + // Extract area coordinates and distances - use SIM-prefixed field names + if areaStartX, exists := paramsMap["sim_area_start_x"]; exists { + arguments["sim_area_start_x"] = areaStartX } - if areaStartY, exists := paramsMap["area_start_y"]; exists { - arguments["area_start_y"] = areaStartY + if areaStartY, exists := paramsMap["sim_area_start_y"]; exists { + arguments["sim_area_start_y"] = areaStartY } - if areaEndX, exists := paramsMap["area_end_x"]; exists { - arguments["area_end_x"] = areaEndX + if areaEndX, exists := paramsMap["sim_area_end_x"]; exists { + arguments["sim_area_end_x"] = areaEndX } - if areaEndY, exists := paramsMap["area_end_y"]; exists { - arguments["area_end_y"] = areaEndY + if areaEndY, exists := paramsMap["sim_area_end_y"]; exists { + arguments["sim_area_end_y"] = areaEndY } - if minDistance, exists := paramsMap["min_distance"]; exists { - arguments["min_distance"] = minDistance + if minDistance, exists := paramsMap["sim_min_distance"]; exists { + arguments["sim_min_distance"] = minDistance } - if maxDistance, exists := paramsMap["max_distance"]; exists { - arguments["max_distance"] = maxDistance + if maxDistance, exists := paramsMap["sim_max_distance"]; exists { + arguments["sim_max_distance"] = maxDistance } // Add duration and press duration from options @@ -886,17 +886,17 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { return nil, err } - // Get coordinates from arguments - startX := unifiedReq.StartX - startY := unifiedReq.StartY - endX := unifiedReq.ToX // Using existing ToX field - endY := unifiedReq.ToY // Using existing ToY field + // Get coordinates from arguments - use fromX/fromY instead of startX/startY + fromX := unifiedReq.FromX + fromY := unifiedReq.FromY + toX := unifiedReq.ToX + toY := unifiedReq.ToY log.Info(). - Float64("startX", startX). - Float64("startY", startY). - Float64("endX", endX). - Float64("endY", endY). + Float64("startX", fromX). + Float64("startY", fromY). + Float64("endX", toX). + Float64("endY", toY). Msg("performing simulated point to point swipe") // Build all options from request arguments @@ -904,7 +904,7 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + err = simDriver.SIMSwipeFromPointToPoint(fromX, fromY, toX, toY, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err } @@ -913,12 +913,12 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { } message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", - startX, startY, endX, endY) + fromX, fromY, toX, toY) returnData := ToolSIMSwipeFromPointToPoint{ - StartX: startX, - StartY: startY, - EndX: endX, - EndY: endY, + StartX: fromX, + StartY: fromY, + EndX: toX, + EndY: toY, } return NewMCPSuccessResponse(message, &returnData), nil @@ -930,18 +930,18 @@ func (t *ToolSIMSwipeFromPointToPoint) ConvertActionToCallToolRequest(action opt if paramsMap, ok := action.Params.(map[string]interface{}); ok { arguments := map[string]any{} - // Extract coordinates - if startX, exists := paramsMap["start_x"]; exists { - arguments["start_x"] = startX + // Extract coordinates - use new field names directly + if fromX, exists := paramsMap["from_x"]; exists { + arguments["from_x"] = fromX } - if startY, exists := paramsMap["start_y"]; exists { - arguments["start_y"] = startY + if fromY, exists := paramsMap["from_y"]; exists { + arguments["from_y"] = fromY } - if endX, exists := paramsMap["end_x"]; exists { - arguments["to_x"] = endX // Map to existing ToX field + if toX, exists := paramsMap["to_x"]; exists { + arguments["to_x"] = toX } - if endY, exists := paramsMap["end_y"]; exists { - arguments["to_y"] = endY // Map to existing ToY field + if toY, exists := paramsMap["to_y"]; exists { + arguments["to_y"] = toY } // Add duration and press duration from options diff --git a/uixt/option/action.go b/uixt/option/action.go index f165cf36..7dfd872c 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -206,17 +206,18 @@ type ActionOptions struct { PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"` Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` - StartX float64 `json:"start_x,omitempty" yaml:"start_x,omitempty" desc:"Starting X coordinate for simulated swipe"` - StartY float64 `json:"start_y,omitempty" yaml:"start_y,omitempty" desc:"Starting Y coordinate for simulated swipe"` - MinDistance float64 `json:"min_distance,omitempty" yaml:"min_distance,omitempty" desc:"Minimum distance for simulated swipe"` - MaxDistance float64 `json:"max_distance,omitempty" yaml:"max_distance,omitempty" desc:"Maximum distance for simulated swipe"` - AreaStartX float64 `json:"area_start_x,omitempty" yaml:"area_start_x,omitempty" desc:"Area starting X coordinate for simulated swipe"` - AreaStartY float64 `json:"area_start_y,omitempty" yaml:"area_start_y,omitempty" desc:"Area starting Y coordinate for simulated swipe"` - AreaEndX float64 `json:"area_end_x,omitempty" yaml:"area_end_x,omitempty" desc:"Area ending X coordinate for simulated swipe"` - AreaEndY float64 `json:"area_end_y,omitempty" yaml:"area_end_y,omitempty" desc:"Area ending Y coordinate for simulated swipe"` - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` - TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` - Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` + + // SIM specific options with SIM prefix + SIMMinDistance float64 `json:"sim_min_distance,omitempty" yaml:"sim_min_distance,omitempty" desc:"Minimum distance for SIM simulated actions"` + SIMMaxDistance float64 `json:"sim_max_distance,omitempty" yaml:"sim_max_distance,omitempty" desc:"Maximum distance for SIM simulated actions"` + SIMAreaStartX float64 `json:"sim_area_start_x,omitempty" yaml:"sim_area_start_x,omitempty" desc:"Area starting X coordinate for SIM simulated swipe"` + SIMAreaStartY float64 `json:"sim_area_start_y,omitempty" yaml:"sim_area_start_y,omitempty" desc:"Area starting Y coordinate for SIM simulated swipe"` + SIMAreaEndX float64 `json:"sim_area_end_x,omitempty" yaml:"sim_area_end_x,omitempty" desc:"Area ending X coordinate for SIM simulated swipe"` + SIMAreaEndY float64 `json:"sim_area_end_y,omitempty" yaml:"sim_area_end_y,omitempty" desc:"Area ending Y coordinate for SIM simulated swipe"` + + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` + TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` + Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` ScreenOptions @@ -662,6 +663,13 @@ func (o *ActionOptions) GetMCPOptions(actionType ActionName) []mcp.ToolOption { ACTION_Back: {"platform", "serial"}, ACTION_ListPackages: {"platform", "serial"}, ACTION_ClosePopups: {"platform", "serial"}, + + // SIM specific actions using fromX/fromY for startX/startY and SIM-prefixed fields + ACTION_SIMSwipeDirection: {"platform", "serial", "direction", "fromX", "fromY", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"}, + ACTION_SIMSwipeInArea: {"platform", "serial", "direction", "sim_area_start_x", "sim_area_start_y", "sim_area_end_x", "sim_area_end_y", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"}, + ACTION_SIMSwipeFromPointToPoint: {"platform", "serial", "fromX", "fromY", "toX", "toY", "duration", "pressDuration"}, + ACTION_SIMClickAtPoint: {"platform", "serial", "x", "y", "duration", "pressDuration"}, + ACTION_SIMInput: {"platform", "serial", "text", "frequency"}, } fields := fieldMappings[actionType] From 5189bc8d53ff8f099d82c287b7d85a1cff8adf67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 30 Jul 2025 15:54:03 +0800 Subject: [PATCH 19/32] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=8D=8F?= =?UTF-8?q?=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- uixt/ai/wings_service.go | 21 ++++++++++----------- uixt/driver_ext_ai_test.go | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index e0737887..dc1711bb 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250729 +v5.0.0-250730 diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 7677af7e..7b8f25c9 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -79,9 +79,9 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni apiRequest := WingsActionRequest{ Historys: w.history, DeviceInfo: deviceInfo, - StepText: opts.UserInstruction, + StepText: fmt.Sprintf("%s", opts.UserInstruction), BizId: w.bizId, - TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s。\n断言: 当前在“张杰演唱会”搜索页。\n停止操作。\n注意事项：\n", opts.UserInstruction), + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s\n停止操作。\n注意事项：\n", opts.UserInstruction), Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -101,7 +101,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &PlanningResult{ Thought: response.ThoughtChain.Thought, @@ -117,7 +117,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni Summary: response.ThoughtChain.Summary, StepText: response.StepText, StepTextTrans: response.StepTextTrans, - OriStepIndex: parseOriStepIndex(response.OriStepIndex), + OriStepIndex: response.OriStepIndex, DeviceID: deviceInfo[0].DeviceID, AgentType: response.AgentType, ActionResult: "", // Always empty as requested @@ -172,14 +172,13 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert apiRequest := WingsActionRequest{ Historys: []History{}, DeviceInfo: deviceInfo, - StepText: opts.Assertion, + StepText: fmt.Sprintf("断言:%s", opts.Assertion), BizId: w.bizId, TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), Base: WingsBase{ LogID: generateWingsUUID(), }, } - log.Info().Str("assertion", opts.Assertion).Str("biz_id", w.bizId).Str("url", w.apiURL).Msg("call wings api") // Call Wings API startTime := time.Now() @@ -195,7 +194,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &AssertionResult{ Pass: false, @@ -211,7 +210,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert Summary: response.ThoughtChain.Summary, StepText: response.StepText, StepTextTrans: response.StepTextTrans, - OriStepIndex: parseOriStepIndex(response.OriStepIndex), + OriStepIndex: response.OriStepIndex, DeviceID: deviceInfo[0].DeviceID, AgentType: response.AgentType, ActionResult: "", // Always empty as requested @@ -293,7 +292,7 @@ type WingsActionResponse struct { AgentType string `json:"agent_type" thrift:"agent_type,1,required"` StepText string `json:"step_text" thrift:"step_text,2,required"` StepTextTrans string `json:"step_text_trans" thrift:"step_text_trans,3,required"` - OriStepIndex string `json:"ori_step_index" thrift:"ori_step_index,4,required"` + OriStepIndex int `json:"ori_step_index" thrift:"ori_step_index,4,required"` StepType string `json:"step_type" thrift:"step_type,5,required"` ActionParams string `json:"action_params" thrift:"action_params,6,required"` ThoughtChain WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"` @@ -329,7 +328,7 @@ type History struct { DeviceInfos *[]WingsDeviceInfo `json:"device_infos,omitempty" thrift:"device_infos,9"` // 所有设备的信息 ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) - OriStepIndex int64 `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） + OriStepIndex int `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） } // Action parameter structures @@ -483,7 +482,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Execute HTTP request client := &http.Client{ - Timeout: 60 * time.Second, + Timeout: 120 * time.Second, } resp, err := client.Do(httpReq) diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 0de1c76b..decfbf7a 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,10 +292,10 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "[目标导向]搜索“张杰演唱会”进入搜索页后停止" + prompt := "点击返回按钮" // Test both methods with the same prompt - aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) + aiResult, aiErr := driver.AIAction(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) From b9cb39a3cf4e7b11781c1a0234567b49a122243c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 30 Jul 2025 16:56:29 +0800 Subject: [PATCH 20/32] =?UTF-8?q?feat:=20wings=E6=8E=A5=E5=8F=A3=E5=85=88?= =?UTF-8?q?=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/wings_service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 7b8f25c9..3fbe5f48 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -467,6 +467,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Set headers httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Accept", "application/json") + httpReq.Header.Add("x-use-ppe", "1") + httpReq.Header.Add("x-tt-env", "ppe_refactor_merge") // Add authentication headers if using external API if w.accessKey != "" && w.secretKey != "" { From c9d3175bc76a3b2c63fa04c7aef2759d3ea2ea7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 30 Jul 2025 19:30:07 +0800 Subject: [PATCH 21/32] =?UTF-8?q?feat:=20=E5=8A=A0=E9=95=BF=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- uixt/android_device.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index e5858edd..dc1711bb 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250724 +v5.0.0-250730 diff --git a/uixt/android_device.go b/uixt/android_device.go index efb243e8..413a171b 100644 --- a/uixt/android_device.go +++ b/uixt/android_device.go @@ -240,7 +240,7 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er return err } // 等待安装完成或超时 - timeout := 3 * time.Minute + timeout := 8 * time.Minute select { case err := <-done: return err From 8cbe3389f921fdea46f518cec99c65d68840dd34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 30 Jul 2025 19:33:12 +0800 Subject: [PATCH 22/32] =?UTF-8?q?feat:=20=E5=AE=89=E8=A3=85=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE8min=E8=B6=85=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/gadb/device.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/gadb/device.go b/pkg/gadb/device.go index d616a654..e6cd5aed 100644 --- a/pkg/gadb/device.go +++ b/pkg/gadb/device.go @@ -615,14 +615,22 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt tp transport filesize int64 ) + timeout := 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute) + defer cancel() + filesize, err = apk.Seek(0, io.SeekEnd) if err != nil { return nil, err } - if tp, err = d.createDeviceTransport(5 * time.Minute); err != nil { + if tp, err = d.createDeviceTransport(4 * time.Minute); err != nil { return nil, err } defer func() { _ = tp.Close() }() + go func() { + <-ctx.Done() + _ = tp.Close() + }() cmd := "abb_exec:package\x00install\x00-t" for _, arg := range args { cmd += "\x00" + arg @@ -641,6 +649,9 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt return nil, err } raw, err = tp.ReadBytesAll() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("installation timed out after %d minutes", timeout) + } return } From 4031290a704a36dffa633ab10a5308bb0e170a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 30 Jul 2025 21:36:25 +0800 Subject: [PATCH 23/32] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E9=BB=98?= =?UTF-8?q?=E8=AE=A4history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/wings_service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 3fbe5f48..0755952a 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -51,6 +51,7 @@ func NewWingsService() (ILLMService, error) { bizId: bizID, accessKey: accessKey, secretKey: secretKey, + history: []History{}, }, nil } From 69aa92caf222ba776ff469c011ce11220a5e38a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Thu, 31 Jul 2025 11:13:32 +0800 Subject: [PATCH 24/32] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9Elogid=E9=80=8F?= =?UTF-8?q?=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 ++++ go.sum | 10 ++++++++++ internal/version/VERSION | 2 +- uixt/ai/wings_service.go | 26 ++++++++------------------ uixt/driver_ext_ai_test.go | 4 ++-- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 3abfe4f7..392b2404 100644 --- a/go.mod +++ b/go.mod @@ -48,11 +48,15 @@ require ( ) require ( + code.byted.org/gopkg/ctxvalues v0.4.0 // indirect + code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830 // indirect + code.byted.org/gopkg/net2 v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/antchfx/xpath v1.3.3 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/catppuccin/go v0.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index a4404074..818a0440 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +code.byted.org/gopkg/ctxvalues v0.4.0 h1:31pwEthLdrjbn/vmq0AgehFHvH1QlMSBca+KFvfL2uY= +code.byted.org/gopkg/ctxvalues v0.4.0/go.mod h1:xaQkBQksiY6rtaDilHDy1yYvvOARMdOIPf1BvimjfGE= +code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830 h1:+Msg9hILmtkMi+MrWblG50517OAJiOJGCJNDroRjkDU= +code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830/go.mod h1:UWtjHynVwwLkHxYiarvtfXCkxiOMpEFKAu5n8pop9R4= +code.byted.org/gopkg/net2 v1.2.0 h1:Bxc8ixC/rOWNha1XXbkzHs+y60hWPNPtCuojCZbkg5o= +code.byted.org/gopkg/net2 v1.2.0/go.mod h1:R+TvSEn+fkrfLgVwQ7KtlssZ5bZ2Vcew9pyhL7LrlkU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -30,6 +36,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d h1:vibs+JRVU76xBvTK/EVfv8AwFr8+bZfUOgLwE6Kyq3A= +github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d/go.mod h1:birsdqRCbwnckJbdAvcSao+AzOyibVEoWB55MjpYpB8= github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8= github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= @@ -488,6 +496,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -507,6 +516,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/version/VERSION b/internal/version/VERSION index dc1711bb..043ab204 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250730 +v5.0.0-250731 diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 0755952a..fc28ba20 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -8,10 +8,10 @@ import ( "io" "net/http" "os" - "strconv" "strings" "time" + "code.byted.org/gopkg/logid" "github.com/cloudwego/eino/schema" "github.com/google/uuid" "github.com/pkg/errors" @@ -144,14 +144,17 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni log.Info(). Str("thought", response.ThoughtChain.Thought). + Str("action", response.AgentType). + Str("action_params", response.ActionParams). + Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)). Int("tool_calls_count", len(toolCalls)). Int64("elapsed_ms", elapsed). Msg("Wings API planning completed") return &PlanningResult{ ToolCalls: toolCalls, - Thought: response.ThoughtChain.Thought, - Content: response.ThoughtChain.Summary, + Thought: response.StepTextTrans, + Content: response.StepTextTrans, ModelName: "wings-api", }, nil } @@ -378,20 +381,7 @@ func (w *WingsService) resetHistory() { // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { - return uuid.New().String() -} - -// parseOriStepIndex converts string to int64 with fallback to 0 -func parseOriStepIndex(index string) int64 { - if index == "" { - return 0 - } - - val, err := strconv.ParseInt(index, 10, 64) - if err != nil { - return 0 - } - return val + return logid.GenLogID() } // extractScreenshotFromMessage extracts base64 screenshot from message @@ -481,7 +471,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ httpReq.Header.Add("Content-Type", "application/json") } - log.Info().Str("step_text", request.StepText).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") + log.Info().Str("step_text", request.StepText).Str("log_id", request.Base.LogID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") // Execute HTTP request client := &http.Client{ diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index decfbf7a..89fd65c8 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,10 +292,10 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "点击返回按钮" + prompt := "[目标导向]向上滑动屏幕2次" // Test both methods with the same prompt - aiResult, aiErr := driver.AIAction(context.Background(), prompt) + aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) From 0cf25836420959ead1c75cbd6c5a80c5ab627921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Thu, 31 Jul 2025 12:27:36 +0800 Subject: [PATCH 25/32] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=89=8D=E6=88=AA=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 ---- go.sum | 10 ---------- uixt/ai/wings_service.go | 19 ++++++++++++------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 392b2404..3abfe4f7 100644 --- a/go.mod +++ b/go.mod @@ -48,15 +48,11 @@ require ( ) require ( - code.byted.org/gopkg/ctxvalues v0.4.0 // indirect - code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830 // indirect - code.byted.org/gopkg/net2 v1.2.0 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/antchfx/xpath v1.3.3 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/catppuccin/go v0.2.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect diff --git a/go.sum b/go.sum index 818a0440..a4404074 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.byted.org/gopkg/ctxvalues v0.4.0 h1:31pwEthLdrjbn/vmq0AgehFHvH1QlMSBca+KFvfL2uY= -code.byted.org/gopkg/ctxvalues v0.4.0/go.mod h1:xaQkBQksiY6rtaDilHDy1yYvvOARMdOIPf1BvimjfGE= -code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830 h1:+Msg9hILmtkMi+MrWblG50517OAJiOJGCJNDroRjkDU= -code.byted.org/gopkg/logid v0.0.0-20241008043456-230d03adb830/go.mod h1:UWtjHynVwwLkHxYiarvtfXCkxiOMpEFKAu5n8pop9R4= -code.byted.org/gopkg/net2 v1.2.0 h1:Bxc8ixC/rOWNha1XXbkzHs+y60hWPNPtCuojCZbkg5o= -code.byted.org/gopkg/net2 v1.2.0/go.mod h1:R+TvSEn+fkrfLgVwQ7KtlssZ5bZ2Vcew9pyhL7LrlkU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -36,8 +30,6 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d h1:vibs+JRVU76xBvTK/EVfv8AwFr8+bZfUOgLwE6Kyq3A= -github.com/bytedance/gopkg v0.0.0-20211103090529-d4719f74be3d/go.mod h1:birsdqRCbwnckJbdAvcSao+AzOyibVEoWB55MjpYpB8= github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8= github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= @@ -496,7 +488,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -516,7 +507,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index fc28ba20..ab25e576 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "code.byted.org/gopkg/logid" "github.com/cloudwego/eino/schema" "github.com/google/uuid" "github.com/pkg/errors" @@ -381,7 +380,7 @@ func (w *WingsService) resetHistory() { // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { - return logid.GenLogID() + return uuid.New().String() } // extractScreenshotFromMessage extracts base64 screenshot from message @@ -412,12 +411,18 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo { // TODO: Extract device info from context if available - // use default device info + // Use last history's NowImage as PreImage if history exists + preImage := screenshot + if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 { + preImage = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImage + } + + // use default device info with optimized PreImage return []WingsDeviceInfo{ { DeviceID: "default-device", NowImage: screenshot, - PreImage: screenshot, + PreImage: preImage, NowLayoutJSON: "", OperationSystem: "android", }, @@ -471,8 +476,6 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ httpReq.Header.Add("Content-Type", "application/json") } - log.Info().Str("step_text", request.StepText).Str("log_id", request.Base.LogID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") - // Execute HTTP request client := &http.Client{ Timeout: 120 * time.Second, @@ -483,7 +486,9 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ return nil, errors.Wrap(err, "HTTP request failed") } defer resp.Body.Close() - + // resp X-Tt-Logid + logID := resp.Header.Get("X-Tt-Logid") + log.Info().Str("step_text", request.StepText).Str("log_id", logID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") // Read response body responseBody, err := io.ReadAll(resp.Body) if err != nil { From 1ff41e3bc882fe92ab7719688ac583dea2434189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Thu, 31 Jul 2025 21:46:24 +0800 Subject: [PATCH 26/32] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/android_device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uixt/android_device.go b/uixt/android_device.go index 413a171b..0a47eb95 100644 --- a/uixt/android_device.go +++ b/uixt/android_device.go @@ -245,7 +245,7 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er case err := <-done: return err case <-time.After(timeout): - return fmt.Errorf("installation timed out after %v", timeout) + return fmt.Errorf("install via installer timed out after %v", timeout) } } From 023b0a3c7f49e78dc20eb7a3d9a5378ca0c4a227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Thu, 31 Jul 2025 22:15:10 +0800 Subject: [PATCH 27/32] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81iOS=20=E6=89=8B?= =?UTF-8?q?=E5=8A=BF=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ios_driver_wda.go | 123 +++++++ uixt/touch_simulator_test.go | 618 +++++++++++++++++++++++++++++++++++ 2 files changed, 741 insertions(+) create mode 100644 uixt/touch_simulator_test.go diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 9042c736..283c5a92 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -620,6 +620,129 @@ func (wd *WDADriver) Swipe(fromX, fromY, toX, toY float64, opts ...option.Action return wd.Drag(fromX, fromY, toX, toY, opts...) } +// TouchByEvents performs a complex swipe using a sequence of touch events with pressure and size data +func (wd *WDADriver) TouchByEvents(events []types.TouchEvent, opts ...option.ActionOption) error { + log.Info().Int("eventCount", len(events)).Msg("WDADriver.SwipeSimulator") + + if len(events) == 0 { + return fmt.Errorf("no touch events provided") + } + + actionOptions := option.NewActionOptions(opts...) + + // Apply pre-handlers for the first and last events (start and end coordinates) + firstEvent := events[0] + lastEvent := events[len(events)-1] + + // Use rawX/rawY if available, otherwise fallback to X/Y for first event + startX, startY := firstEvent.RawX, firstEvent.RawY + if startX == 0 && startY == 0 { + startX, startY = firstEvent.X, firstEvent.Y + } + + // Use rawX/rawY if available, otherwise fallback to X/Y for last event + endX, endY := lastEvent.RawX, lastEvent.RawY + if endX == 0 && endY == 0 { + endX, endY = lastEvent.X, lastEvent.Y + } + + fromX, fromY, toX, toY, err := preHandler_Swipe(wd, option.ACTION_SwipeCoordinate, actionOptions, + startX, startY, endX, endY) + if err != nil { + return err + } + defer postHandler(wd, option.ACTION_SwipeCoordinate, actionOptions) + + var actions []interface{} + var prevEventTime int64 + + for i, event := range events { + var duration float64 + if i > 0 { + // Calculate duration from previous event using EventTime (milliseconds) + duration = float64(event.EventTime - prevEventTime) + } + prevEventTime = event.EventTime + + // Use rawX/rawY if available, otherwise fallback to X/Y + x, y := event.RawX, event.RawY + if x == 0 && y == 0 { + // Fallback to X/Y if rawX/rawY are not set + x, y = event.X, event.Y + } + + // Apply coordinate transformation if it's the first or last event + if i == 0 { + x, y = fromX, fromY + } else if i == len(events)-1 { + x, y = toX, toY + } + + var actionMap map[string]interface{} + + switch event.Action { + case 0: // ACTION_DOWN + actionMap = map[string]interface{}{ + "type": "pointerDown", + "duration": 0, + "button": 0, + "pressure": event.Pressure, + "size": event.Size, + } + // Add initial move to position before down + if i == 0 { + moveAction := map[string]interface{}{ + "type": "pointerMove", + "duration": 0, + "x": x, + "y": y, + "origin": "viewport", + "pressure": event.Pressure, + "size": event.Size, + } + actions = append(actions, moveAction) + } + case 1: // ACTION_UP + actionMap = map[string]interface{}{ + "type": "pointerUp", + "duration": 0, + "button": 0, + "pressure": event.Pressure, + "size": event.Size, + } + case 2: // ACTION_MOVE + actionMap = map[string]interface{}{ + "type": "pointerMove", + "duration": duration, + "x": x, + "y": y, + "origin": "viewport", + "pressure": event.Pressure, + "size": event.Size, + } + default: + log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping") + continue + } + actions = append(actions, actionMap) + } + + data := map[string]interface{}{ + "actions": []interface{}{ + map[string]interface{}{ + "type": "pointer", + "parameters": map[string]string{"pointerType": "touch"}, + "id": "touch", + "actions": actions, + }, + }, + } + option.MergeOptions(data, opts...) + + _, err = wd.Session.POST(data, "/wings/actions") + return err +} + func (wd *WDADriver) SetPasteboard(contentType types.PasteboardType, content string) (err error) { // [[FBRoute POST:@"/wda/setPasteboard"] respondWithTarget:self action:@selector(handleSetPasteboard:)] data := map[string]interface{}{ diff --git a/uixt/touch_simulator_test.go b/uixt/touch_simulator_test.go new file mode 100644 index 00000000..4cdcb453 --- /dev/null +++ b/uixt/touch_simulator_test.go @@ -0,0 +1,618 @@ +//go:build localtest + +package uixt + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/httprunner/httprunner/v5/uixt/option" + "github.com/httprunner/httprunner/v5/uixt/types" +) + +// ParseTouchEvents parses touch event data from comma-separated string format +func ParseTouchEvents(data string) ([]types.TouchEvent, error) { + lines := strings.Split(strings.TrimSpace(data), "\n") + events := make([]types.TouchEvent, 0, len(lines)) + + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + + parts := strings.Split(line, ",") + if len(parts) != 13 { + return nil, fmt.Errorf("invalid touch event data format: expected 13 fields, got %d", len(parts)) + } + + event := types.TouchEvent{} + var err error + + if event.X, err = strconv.ParseFloat(parts[1], 64); err != nil { + return nil, fmt.Errorf("invalid x coordinate: %v", err) + } + if event.Y, err = strconv.ParseFloat(parts[2], 64); err != nil { + return nil, fmt.Errorf("invalid y coordinate: %v", err) + } + if event.DeviceID, err = strconv.Atoi(parts[3]); err != nil { + return nil, fmt.Errorf("invalid device id: %v", err) + } + if event.Pressure, err = strconv.ParseFloat(parts[4], 64); err != nil { + return nil, fmt.Errorf("invalid pressure: %v", err) + } + if event.Size, err = strconv.ParseFloat(parts[5], 64); err != nil { + return nil, fmt.Errorf("invalid size: %v", err) + } + if event.RawX, err = strconv.ParseFloat(parts[6], 64); err != nil { + return nil, fmt.Errorf("invalid raw x: %v", err) + } + if event.RawY, err = strconv.ParseFloat(parts[7], 64); err != nil { + return nil, fmt.Errorf("invalid raw y: %v", err) + } + if event.DownTime, err = strconv.ParseInt(parts[8], 10, 64); err != nil { + return nil, fmt.Errorf("invalid down time: %v", err) + } + if event.EventTime, err = strconv.ParseInt(parts[9], 10, 64); err != nil { + return nil, fmt.Errorf("invalid event time: %v", err) + } + if event.ToolType, err = strconv.Atoi(parts[10]); err != nil { + return nil, fmt.Errorf("invalid tool type: %v", err) + } + if event.Flag, err = strconv.Atoi(parts[11]); err != nil { + return nil, fmt.Errorf("invalid flag: %v", err) + } + if event.Action, err = strconv.Atoi(parts[12]); err != nil { + return nil, fmt.Errorf("invalid action: %v", err) + } + + events = append(events, event) + } + + return events, nil +} + +func TestAndroidTouchByEvents(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Example touch event data as provided + touchEventData := `1752649131556,401.20703,1191.3164,2,1.0,0.03529412,457.20703,1359.3164,111586196,111586196,1,0,0 +1752649131595,402.913,1185.0792,2,1.0,0.039215688,458.913,1353.0792,111586196,111586236,1,0,2 +1752649131612,410.60825,1164.3806,2,1.0,0.03529412,466.60825,1332.3806,111586196,111586250,1,0,2 +1752649131629,437.7335,1093.1417,2,1.0,0.039215688,493.7335,1261.1417,111586196,111586270,1,0,2 +1752649131646,463.5786,1018.01746,2,1.0,0.039215688,519.5786,1186.0175,111586196,111586287,1,0,2 +1752649131662,487.56482,948.9773,2,1.0,0.03529412,543.5648,1116.9773,111586196,111586304,1,0,2 +1752649131679,511.81476,881.6183,2,1.0,0.039215688,567.81476,1049.6183,111586196,111586320,1,0,2 +1752649131696,543.4369,811.4982,2,1.0,0.03529412,599.4369,979.4982,111586196,111586337,1,0,2 +1752649131713,577.1632,747.4512,2,1.0,0.039215688,633.1632,915.4512,111586196,111586354,1,0,2 +1752649131729,610.1538,691.72034,2,1.0,0.03529412,666.1538,859.72034,111586196,111586370,1,0,2 +1752649131746,639.1683,642.6914,2,1.0,0.03529412,695.1683,810.6914,111586196,111586387,1,0,2 +1752649131763,658.9832,605.90857,2,1.0,0.03529412,714.9832,773.90857,111586196,111586404,1,0,2 +1752649131779,672.21954,581.1634,2,1.0,0.03529412,728.21954,749.1634,111586196,111586420,1,0,2 +1752649131796,680.7687,566.1778,2,1.0,0.03529412,736.7687,734.1778,111586196,111586434,1,0,2 +1752649131814,688.0894,554.2295,2,1.0,0.03529412,744.0894,722.2295,111586196,111586450,1,0,2 +1752649131830,694.542,544.7783,2,1.0,0.03529412,750.542,712.7783,111586196,111586466,1,0,2 +1752649131847,700.60645,537.2637,2,1.0,0.039215688,756.60645,705.2637,111586196,111586483,1,0,2 +1752649131863,705.08887,531.1406,2,1.0,0.039215688,761.08887,699.1406,111586196,111586500,1,0,2 +1752649131880,708.1211,527.8008,2,1.0,0.039215688,764.1211,695.8008,111586196,111586517,1,0,2 +1752649131897,709.43945,524.46094,2,1.0,0.039215688,765.43945,692.46094,111586196,111586533,1,0,2 +1752649131902,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586537,1,33554432,2 +1752649131907,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586546,1,0,1` + + // Parse touch events + events, err := ParseTouchEvents(touchEventData) + if err != nil { + t.Fatalf("ParseTouchEvents failed: %v", err) + } + + // Check first event + firstEvent := events[0] + if firstEvent.Action != 0 { // ACTION_DOWN + t.Errorf("Expected first event action to be 0 (ACTION_DOWN), got %d", firstEvent.Action) + } + + // Check last event + lastEvent := events[len(events)-1] + if lastEvent.Action != 1 { // ACTION_UP + t.Errorf("Expected last event action to be 1 (ACTION_UP), got %d", lastEvent.Action) + } + + // Use TouchByEvents with parsed events + err = driver.TouchByEvents(events) + if err != nil { + t.Fatalf("TouchByEvents failed: %v", err) + } + + t.Logf("Successfully executed touch events: %d events processed", len(events)) +} + +func TestIOSTouchByEvents(t *testing.T) { + driver := setupWDADriverExt(t) + + // Example touch event data as provided + touchEventData := `1752649131556,401.20703,1191.3164,2,1.0,0.03529412,457.20703,1359.3164,111586196,111586196,1,0,0 +1752649131595,402.913,1185.0792,2,1.0,0.039215688,458.913,1353.0792,111586196,111586236,1,0,2 +1752649131612,410.60825,1164.3806,2,1.0,0.03529412,466.60825,1332.3806,111586196,111586250,1,0,2 +1752649131629,437.7335,1093.1417,2,1.0,0.039215688,493.7335,1261.1417,111586196,111586270,1,0,2 +1752649131646,463.5786,1018.01746,2,1.0,0.039215688,519.5786,1186.0175,111586196,111586287,1,0,2 +1752649131662,487.56482,948.9773,2,1.0,0.03529412,543.5648,1116.9773,111586196,111586304,1,0,2 +1752649131679,511.81476,881.6183,2,1.0,0.039215688,567.81476,1049.6183,111586196,111586320,1,0,2 +1752649131696,543.4369,811.4982,2,1.0,0.03529412,599.4369,979.4982,111586196,111586337,1,0,2 +1752649131713,577.1632,747.4512,2,1.0,0.039215688,633.1632,915.4512,111586196,111586354,1,0,2 +1752649131729,610.1538,691.72034,2,1.0,0.03529412,666.1538,859.72034,111586196,111586370,1,0,2 +1752649131746,639.1683,642.6914,2,1.0,0.03529412,695.1683,810.6914,111586196,111586387,1,0,2 +1752649131763,658.9832,605.90857,2,1.0,0.03529412,714.9832,773.90857,111586196,111586404,1,0,2 +1752649131779,672.21954,581.1634,2,1.0,0.03529412,728.21954,749.1634,111586196,111586420,1,0,2 +1752649131796,680.7687,566.1778,2,1.0,0.03529412,736.7687,734.1778,111586196,111586434,1,0,2 +1752649131814,688.0894,554.2295,2,1.0,0.03529412,744.0894,722.2295,111586196,111586450,1,0,2 +1752649131830,694.542,544.7783,2,1.0,0.03529412,750.542,712.7783,111586196,111586466,1,0,2 +1752649131847,700.60645,537.2637,2,1.0,0.039215688,756.60645,705.2637,111586196,111586483,1,0,2 +1752649131863,705.08887,531.1406,2,1.0,0.039215688,761.08887,699.1406,111586196,111586500,1,0,2 +1752649131880,708.1211,527.8008,2,1.0,0.039215688,764.1211,695.8008,111586196,111586517,1,0,2 +1752649131897,709.43945,524.46094,2,1.0,0.039215688,765.43945,692.46094,111586196,111586533,1,0,2 +1752649131902,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586537,1,33554432,2 +1752649131907,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586546,1,0,1` + + // Parse touch events + events, err := ParseTouchEvents(touchEventData) + if err != nil { + t.Fatalf("ParseTouchEvents failed: %v", err) + } + + // Check first event + firstEvent := events[0] + if firstEvent.Action != 0 { // ACTION_DOWN + t.Errorf("Expected first event action to be 0 (ACTION_DOWN), got %d", firstEvent.Action) + } + + // Check last event + lastEvent := events[len(events)-1] + if lastEvent.Action != 1 { // ACTION_UP + t.Errorf("Expected last event action to be 1 (ACTION_UP), got %d", lastEvent.Action) + } + + // Use TouchByEvents with parsed events + err = driver.IDriver.(*WDADriver).TouchByEvents(events) + if err != nil { + t.Fatalf("TouchByEvents failed: %v", err) + } + + t.Logf("Successfully executed touch events: %d events processed", len(events)) +} + +func TestTouchEventParsing(t *testing.T) { + // Test single touch event parsing + singleEventData := "1752646457403,456.78418,1574.0195,7,1.0,0.016666668,504.78418,1721.0195,924451292,924451292,1,0,0" + + events, err := ParseTouchEvents(singleEventData) + if err != nil { + t.Fatalf("ParseTouchEvents failed: %v", err) + } + + if len(events) != 1 { + t.Fatalf("Expected 1 event, got %d", len(events)) + } + + event := events[0] + if event.X != 456.78418 { + t.Errorf("Expected X 456.78418, got %f", event.X) + } + if event.Y != 1574.0195 { + t.Errorf("Expected Y 1574.0195, got %f", event.Y) + } + if event.Action != 0 { + t.Errorf("Expected Action 0, got %d", event.Action) + } + if event.Pressure != 1.0 { + t.Errorf("Expected Pressure 1.0, got %f", event.Pressure) + } + if event.Size != 0.016666668 { + t.Errorf("Expected Size 0.016666668, got %f", event.Size) + } +} + +func TestTouchEventParsingInvalidData(t *testing.T) { + // Test with invalid data + testCases := []struct { + name string + data string + }{ + { + name: "too few fields", + data: "1752646457403,456.78418,1574.0195,7,1.0", + }, + { + name: "invalid timestamp", + data: "invalid,456.78418,1574.0195,7,1.0,0.016666668,504.78418,1721.0195,924451292,924451292,1,0,0", + }, + { + name: "invalid x coordinate", + data: "1752646457403,invalid,1574.0195,7,1.0,0.016666668,504.78418,1721.0195,924451292,924451292,1,0,0", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := ParseTouchEvents(tc.data) + if err == nil { + t.Errorf("Expected error for invalid data, but got none") + } + }) + } +} + +func TestTouchEventSequenceValidation(t *testing.T) { + // Test a complete touch sequence: DOWN -> MOVE -> MOVE -> UP + sequenceData := `1752646457403,100.0,100.0,7,1.0,0.016666668,100.0,100.0,924451292,924451292,1,0,0 +1752646457420,120.0,120.0,7,1.0,0.022058824,120.0,120.0,924451292,924451335,1,0,2 +1752646457440,140.0,140.0,7,1.0,0.022058824,140.0,140.0,924451292,924451351,1,0,2 +1752646457460,160.0,160.0,7,1.0,0.012254903,160.0,160.0,924451292,924451619,1,0,1` + + events, err := ParseTouchEvents(sequenceData) + if err != nil { + t.Fatalf("ParseTouchEvents failed: %v", err) + } + + if len(events) != 4 { + t.Fatalf("Expected 4 events, got %d", len(events)) + } + + // Validate sequence: DOWN -> MOVE -> MOVE -> UP + expectedActions := []int{0, 2, 2, 1} // ACTION_DOWN, ACTION_MOVE, ACTION_MOVE, ACTION_UP + for i, event := range events { + if event.Action != expectedActions[i] { + t.Errorf("Event %d: expected action %d, got %d", i, expectedActions[i], event.Action) + } + } + + t.Logf("Touch sequence validation passed: %d events with correct action sequence", len(events)) +} + +func TestSwipeWithDirection(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different directions and distance configurations + testCases := []struct { + name string + direction string + startX float64 + startY float64 + minDistance float64 + maxDistance float64 + }{ + { + name: "随机距离上滑", + direction: "up", + startX: 0.5, + startY: 0.5, + minDistance: 100.0, + maxDistance: 500.0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMSwipeWithDirection( + tc.direction, + tc.startX, + tc.startY, + tc.minDistance, + tc.maxDistance, + ) + if err != nil { + t.Errorf("SwipeWithDirection failed: %v", err) + } else { + t.Logf("Successfully executed swipe: direction=%s, start=(%.1f,%.1f), distance=%.1f-%.1f", + tc.direction, tc.startX, tc.startY, tc.minDistance, tc.maxDistance) + } + }) + } +} + +func TestSwipeWithDirectionInvalidInputs(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test invalid direction + err = driver.SIMSwipeWithDirection("invalid", 500.0, 500.0, 100.0, 200.0) + if err == nil { + t.Error("Expected error for invalid direction, but got none") + } + + // Test invalid distance range (max < min) + err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 200.0, 100.0) + if err == nil { + t.Error("Expected error for invalid distance range, but got none") + } + + // Test zero distance + err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 0.0, 0.0) + if err == nil { + t.Error("Expected error for zero distance, but got none") + } + + t.Log("Invalid input validation tests passed") +} + +func TestSwipeInArea(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different area configurations and directions + testCases := []struct { + name string + direction string + areaStartX float64 + areaStartY float64 + areaEndX float64 + areaEndY float64 + minDistance float64 + maxDistance float64 + }{ + { + name: "中心区域上滑_固定距离", + direction: "up", + areaStartX: 0.2, + areaStartY: 0.3, + areaEndX: 0.8, + areaEndY: 0.6, + minDistance: 500.0, + maxDistance: 700.0, // 固定距离 + }, + } + + for _, tc := range testCases { + for i := 0; i < 3; i++ { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMSwipeInArea( + tc.direction, + tc.areaStartX, + tc.areaStartY, + tc.areaEndX, + tc.areaEndY, + tc.minDistance, + tc.maxDistance, + ) + if err != nil { + t.Errorf("SwipeInArea failed: %v", err) + } else { + t.Logf("Successfully executed area swipe: direction=%s, area=(%.1f,%.1f)-(%.1f,%.1f), distance=%.1f-%.1f", + tc.direction, tc.areaStartX, tc.areaStartY, tc.areaEndX, tc.areaEndY, tc.minDistance, tc.maxDistance) + } + }) + } + } +} + +func TestSwipeFromPointToPoint(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different point-to-point swipes + testCases := []struct { + name string + startX float64 + startY float64 + endX float64 + endY float64 + }{ + { + name: "对角线滑动_左上到右下", + startX: 0.2, + startY: 0.3, + endX: 0.8, + endY: 0.5, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMSwipeFromPointToPoint( + tc.startX, + tc.startY, + tc.endX, + tc.endY, + ) + if err != nil { + t.Errorf("SwipeFromPointToPoint failed: %v", err) + } else { + t.Logf("Successfully executed point-to-point swipe: %s, from (%.1f,%.1f) to (%.1f,%.1f)", + tc.name, tc.startX, tc.startY, tc.endX, tc.endY) + } + }) + } +} + +func TestSwipeFromPointToPointInvalidInputs(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test same start and end point + err = driver.SIMSwipeFromPointToPoint(0.5, 0.5, 0.5, 0.5) + if err == nil { + t.Error("Expected error for same start and end point, but got none") + } + + // Test very close points (should result in distance too short) + err = driver.SIMSwipeFromPointToPoint(0.5, 0.5, 0.501, 0.501) + if err == nil { + t.Error("Expected error for very close points, but got none") + } + + t.Log("Point-to-point swipe invalid input validation tests passed") +} + +func TestClickAtPoint(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different click positions + testCases := []struct { + name string + x float64 + y float64 + }{ + { + name: "屏幕中心点击", + x: 0.5, + y: 0.5, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMClickAtPoint(tc.x, tc.y) + if err != nil { + t.Errorf("ClickAtPoint failed: %v", err) + } else { + t.Logf("Successfully executed click: %s at (%.1f, %.1f)", + tc.name, tc.x, tc.y) + } + }) + } +} + +func TestClickAtPointInvalidInputs(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test negative coordinates + err = driver.SIMClickAtPoint(-0.1, 0.5) + if err == nil { + t.Error("Expected error for negative x coordinate, but got none") + } + + err = driver.SIMClickAtPoint(0.5, -0.1) + if err == nil { + t.Error("Expected error for negative y coordinate, but got none") + } + + // Test coordinates out of range (though these should be handled by convertToAbsolutePoint) + err = driver.SIMClickAtPoint(1.5, 0.5) + if err != nil { + t.Logf("Out of range coordinates handled properly: %v", err) + } + + t.Log("Click invalid input validation tests passed") +} + +func TestSIMInput(t *testing.T) { + device, err := NewAndroidDevice( + option.WithSerialNumber(""), + ) + if err != nil { + t.Fatal(err) + } + + driver, err := NewUIA2Driver(device) + if err != nil { + t.Fatal(err) + } + defer driver.TearDown() + + // Test cases for different text inputs + testCases := []struct { + name string + text string + }{ + { + name: "长文本", + text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := driver.SIMInput(tc.text) + // err := driver.Input(tc.text) + if err != nil { + t.Errorf("SIMInput failed: %v", err) + } else { + t.Logf("Successfully executed SIMInput: %s with text '%s'", tc.name, tc.text) + } + }) + } +} From bf541785a1cb8290f5c2b5768ea7be137edfd103 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 2 Aug 2025 21:54:07 +0800 Subject: [PATCH 28/32] feat: add workflow claude code --- .github/workflows/claude-code.yml | 37 +++++++++++++++++++++++++++++++ internal/version/VERSION | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/claude-code.yml diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml new file mode 100644 index 00000000..5c1c9bd9 --- /dev/null +++ b/.github/workflows/claude-code.yml @@ -0,0 +1,37 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@beta + env: + ANTHROPIC_BASE_URL: "${{ secrets.ANTHROPIC_BASE_URL }}" + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} \ No newline at end of file diff --git a/internal/version/VERSION b/internal/version/VERSION index 043ab204..a36de551 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250731 +v5.0.0-250802 From c61c5af33412996e4f2d825d505e0b7af7f13190 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 2 Aug 2025 22:09:40 +0800 Subject: [PATCH 29/32] fix: claude code actions --- .github/workflows/claude-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml index 5c1c9bd9..5c8179fc 100644 --- a/.github/workflows/claude-code.yml +++ b/.github/workflows/claude-code.yml @@ -6,7 +6,7 @@ on: pull_request_review_comment: types: [created] issues: - types: [opened, assigned] + types: [opened, assigned, edited] pull_request_review: types: [submitted] From 84f038344a61d5f698881c620434cf4d6b3f0315 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 2 Aug 2025 22:13:32 +0800 Subject: [PATCH 30/32] test: debug workflow --- .github/workflows/claude-code.yml | 1 + .github/workflows/test.yml | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml index 5c8179fc..22afa6a6 100644 --- a/.github/workflows/claude-code.yml +++ b/.github/workflows/claude-code.yml @@ -9,6 +9,7 @@ on: types: [opened, assigned, edited] pull_request_review: types: [submitted] + workflow_dispatch: jobs: claude: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..c36da14b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: Test Actions + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Echo test + run: echo "GitHub Actions is working!" \ No newline at end of file From e1867f99ef4aba24662c0e813ba0ae00b15a19f1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 2 Aug 2025 22:18:15 +0800 Subject: [PATCH 31/32] test: debug workflow --- .github/workflows/claude-code.yml | 1 + .github/workflows/test.yml | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml index 22afa6a6..73abfe62 100644 --- a/.github/workflows/claude-code.yml +++ b/.github/workflows/claude-code.yml @@ -14,6 +14,7 @@ on: jobs: claude: if: | + github.event_name == 'workflow_dispatch' || (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index c36da14b..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Test Actions - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Echo test - run: echo "GitHub Actions is working!" \ No newline at end of file From 38acfe3e3a815a78d9e5e5670f08a0bd25d21548 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 2 Aug 2025 22:29:54 +0800 Subject: [PATCH 32/32] fix: claude code actions --- .github/workflows/claude-code.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml index 73abfe62..5c1c9bd9 100644 --- a/.github/workflows/claude-code.yml +++ b/.github/workflows/claude-code.yml @@ -6,15 +6,13 @@ on: pull_request_review_comment: types: [created] issues: - types: [opened, assigned, edited] + types: [opened, assigned] pull_request_review: types: [submitted] - workflow_dispatch: jobs: claude: if: | - github.event_name == 'workflow_dispatch' || (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||