From 147020cbe56c75ba53d1fb0c3deeb643ff005f20 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 4 Jul 2025 14:23:09 +0800 Subject: [PATCH 1/4] feat: add time limit for StartToGoal --- docs/uixt/ai-service.md | 303 ++++++++++++++++++ examples/game/llk/game_llk.json | 2 +- examples/game/llk/main_test.go | 7 +- .../game_romantic_restaurant.json | 2 +- .../game/romantic_restaurant/main_test.go | 2 +- examples/game/sudoku/game_sudoku.json | 2 +- examples/game/sudoku/main_test.go | 2 +- .../game/yanglegeyang/game_yanglegeyang.json | 2 +- examples/game/yanglegeyang/main_test.go | 2 +- .../yuedongxiaozi/game_yuedongxiaozi.json | 88 +++++ examples/game/yuedongxiaozi/main_test.go | 55 ++++ examples/game/zhuadae/game_zhuadae.json | 2 +- examples/game/zhuadae/main_test.go | 2 +- internal/version/VERSION | 2 +- uixt/ai/cv_vedem_test.go | 4 +- uixt/driver_ext_ai.go | 52 ++- uixt/option/action.go | 10 + 17 files changed, 513 insertions(+), 26 deletions(-) create mode 100644 examples/game/yuedongxiaozi/game_yuedongxiaozi.json create mode 100644 examples/game/yuedongxiaozi/main_test.go diff --git a/docs/uixt/ai-service.md b/docs/uixt/ai-service.md index a3267c67..12010e2d 100644 --- a/docs/uixt/ai-service.md +++ b/docs/uixt/ai-service.md @@ -686,6 +686,270 @@ AIQuery 可能遇到的常见错误： 3. 建议在查询中使用具体、明确的描述以获得更好的结果 4. 对于复杂的信息提取，可以要求返回 JSON 格式的结构化数据 +## StartToGoal 功能详解 + +### 概述 + +`StartToGoal` 是 HttpRunner v5 中的目标导向智能操作功能，它使用自然语言描述目标，然后自动规划和执行一系列操作来达成目标。该功能基于视觉语言模型（VLM）进行智能规划，能够理解屏幕内容并自动生成操作序列。 + +### 功能特点 + +- **目标导向**: 使用自然语言描述最终目标，AI 自动规划操作步骤 +- **智能规划**: 基于屏幕内容进行上下文相关的操作规划 +- **自动执行**: 自动执行规划的操作序列直到达成目标 +- **灵活控制**: 支持多种控制选项如重试次数、超时时间等 + +### 基本用法 + +#### 1. 基本示例 + +```go +// 基本目标导向操作 +results, err := driver.StartToGoal(ctx, "导航到设置页面并启用深色模式") + +// 带选项的目标导向操作 +results, err := driver.StartToGoal(ctx, "登录应用", + option.WithMaxRetryTimes(3), + option.WithIdentifier("user-login"), +) +``` + +#### 2. 在测试步骤中使用 + +```go +hrp.NewStep("Navigate to Settings"). + Android(). + StartToGoal("打开设置页面") + +hrp.NewStep("Enable Feature"). + Android(). + StartToGoal("启用深色模式功能", + option.WithMaxRetryTimes(3), + option.WithIdentifier("enable-dark-mode"), + ) +``` + +### TimeLimit 时间限制功能 + +`StartToGoal` 支持 `TimeLimit` 选项，用于设置执行时间限制。这是一个重要的资源管理功能。 + +#### 功能特性 + +- **时间限制**: 支持设置执行时间上限（秒） +- **优雅停止**: 超出时间限制后停止执行，但返回成功状态 +- **部分结果**: 即使达到时间限制，也会返回已完成的规划结果 + +#### 使用方法 + +##### 基本用法 + +```go +// 设置 30 秒时间限制 +results, err := driver.StartToGoal(ctx, prompt, option.WithTimeLimit(30)) +``` + +##### 与其他选项结合使用 + +```go +results, err := driver.StartToGoal(ctx, prompt, + option.WithTimeLimit(45), // 45秒时间限制 + option.WithMaxRetryTimes(3), // 最大重试3次 + option.WithIdentifier("my-task"), // 任务标识符 +) +``` + +#### TimeLimit vs Timeout + +| 特性 | TimeLimit | Timeout | Interrupt Signal | +|------|-----------|---------|------------------| +| 行为 | 优雅停止 | 强制取消 | 立即中断 | +| 返回值 | 成功 (err == nil) | 错误 (err != nil) | 错误 (err != nil) | +| 结果 | 返回部分结果 | 返回部分结果 | 返回部分结果 | +| 用途 | 资源管理，时间预算 | 防止无限等待 | 用户主动中断 | +| 优先级 | 中等 | 低 | 最高 | + +#### 使用场景 + +##### 使用 TimeLimit 的场景： +- 需要在指定时间内完成尽可能多的任务 +- 资源管理和时间预算控制 +- 希望获得部分结果而不是完全失败 +- 测试场景下的时间控制 + +##### 使用 Timeout 的场景： +- 防止无限等待 +- 超时即视为失败的场景 +- 需要严格的时间控制 + +##### Interrupt Signal 的特点： +- 用户主动中断（Ctrl+C） +- 优先级最高，立即生效 +- 无论是否设置 TimeLimit，都返回错误 +- 适用于需要立即停止的场景 + +#### 实现原理 + +1. **Context 复用**: `TimeLimit` 和 `Timeout` 复用相同的 context 超时机制 +2. **模式标记**: 通过 `isTimeLimitMode` 标记区分当前是时间限制模式还是超时模式 +3. **优先级处理**: 在 `ctx.Done()` 时按优先级检查取消原因 +4. **结果收集**: 返回所有已完成的规划结果 + +**技术实现**： +```go +// 复用 timeout context 机制，用标记区分模式 +var isTimeLimitMode bool +if options.TimeLimit > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(options.TimeLimit)*time.Second) + isTimeLimitMode = true +} else if options.Timeout > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) +} + +// 按优先级检查取消原因 +select { +case <-ctx.Done(): + cause := context.Cause(ctx) + // 1. 中断信号优先级最高，始终返回错误 + if errors.Is(cause, code.InterruptError) { + return allPlannings, errors.Wrap(cause, "StartToGoal interrupted") + } + // 2. TimeLimit 超时返回成功 + if isTimeLimitMode && errors.Is(cause, context.DeadlineExceeded) { + return allPlannings, nil + } + // 3. 其他取消原因返回错误 + return allPlannings, errors.Wrap(cause, "StartToGoal cancelled") +} +``` + +#### 注意事项 + +1. **检测精度**: 时间限制的检测精度依赖于规划和工具调用的频率，基于 Go context 机制更加精确 +2. **资源清理**: 即使达到时间限制，也会完成当前操作以确保资源正确清理 +3. **结果可用性**: 返回的结果包含会话数据，可用于生成报告 +4. **Context 复用**: `TimeLimit` 和 `Timeout` 复用相同的 context 超时机制，简化了实现 +5. **优先级**: 如果同时设置了 `TimeLimit` 和 `Timeout`，`TimeLimit` 优先生效 +6. **中断信号**: 用户中断信号（如 Ctrl+C）优先级最高，无论是否设置 `TimeLimit` 都会返回错误 + +### 支持的选项 + +`StartToGoal` 支持多种控制选项： + +```go +// 全面的选项示例 +results, err := driver.StartToGoal(ctx, prompt, + option.WithTimeLimit(60), // 时间限制（秒） + option.WithTimeout(120), // 超时时间（秒） + option.WithMaxRetryTimes(5), // 最大重试次数 + option.WithIdentifier("task-id"), // 任务标识符 + option.WithLLMService("gpt-4o"), // LLM 服务 + option.WithCVService("vedem"), // CV 服务 + option.WithResetHistory(true), // 重置对话历史 +) +``` + +### 最佳实践 + +#### 1. 明确的目标描述 + +```go +// 好的示例：具体明确 +StartToGoal("打开设置页面，找到显示选项，然后启用深色模式") + +// 避免：过于模糊 +StartToGoal("做一些设置") +``` + +#### 2. 合理的时间限制 + +```go +// 根据任务复杂度设置合理的时间限制 +StartToGoal("完成用户注册流程", option.WithTimeLimit(120)) // 复杂任务 +StartToGoal("点击登录按钮", option.WithTimeLimit(30)) // 简单任务 +``` + +#### 3. 错误处理和重试 + +```go +// 设置重试机制 +results, err := driver.StartToGoal(ctx, prompt, + option.WithMaxRetryTimes(3), + option.WithTimeLimit(90), +) + +if err != nil { + // 处理错误 + log.Printf("StartToGoal failed: %v", err) + // 可以分析 results 中的部分结果 +} +``` + +### 实际应用场景 + +#### 1. 复杂的操作流程 + +```go +// 完成整个购物流程 +hrp.NewStep("Complete Purchase"). + Android(). + StartToGoal("搜索商品'手机'，选择第一个商品，添加到购物车，然后结账", + option.WithTimeLimit(180), + option.WithMaxRetryTimes(2), + ) +``` + +#### 2. 应用初始化设置 + +```go +// 首次使用应用的设置流程 +hrp.NewStep("Initial Setup"). + Android(). + StartToGoal("跳过引导页，允许所有权限，然后进入主界面", + option.WithTimeLimit(60), + ) +``` + +#### 3. 测试场景验证 + +```go +// 验证特定功能流程 +hrp.NewStep("Verify Feature"). + Android(). + StartToGoal("验证分享功能是否正常工作", + option.WithTimeLimit(45), + option.WithIdentifier("share-test"), + ) +``` + +### 返回结果 + +`StartToGoal` 返回 `PlanningExecutionResult` 数组，包含详细的执行信息： + +```go +type PlanningExecutionResult struct { + PlanningResult ai.PlanningResult `json:"planning_result"` + SubActions []*SubActionResult `json:"sub_actions"` + StartTime int64 `json:"start_time"` + Elapsed int64 `json:"elapsed"` +} +``` + +可以通过返回结果分析执行过程： + +```go +results, err := driver.StartToGoal(ctx, prompt, option.WithTimeLimit(60)) +if err != nil { + log.Printf("Task failed: %v", err) +} + +// 分析执行结果 +for i, result := range results { + log.Printf("Planning %d: %s", i+1, result.PlanningResult.Thought) + log.Printf("Actions executed: %d", len(result.SubActions)) + log.Printf("Elapsed time: %d ms", result.Elapsed) +} +``` + ## 完整示例 以下是一个完整的 AIQuery 使用示例： @@ -714,4 +978,43 @@ func TestAIQuery(t *testing.T) { err := hrp.NewRunner(t).Run(testCase) assert.Nil(t, err) } +``` + +## StartToGoal 完整示例 + +以下是 `StartToGoal` 功能的完整使用示例： + +```go +func TestStartToGoal(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("StartToGoal Demo"). + SetLLMService(option.OPENAI_GPT_4O), + TestSteps: []hrp.IStep{ + hrp.NewStep("App Launch"). + Android(). + AppLaunch("com.example.app"), + hrp.NewStep("Complete User Setup"). + Android(). + StartToGoal("跳过引导页，创建新用户账户", + option.WithTimeLimit(120), + option.WithMaxRetryTimes(3), + ), + hrp.NewStep("Navigate to Feature"). + Android(). + StartToGoal("导航到设置页面并启用深色模式", + option.WithTimeLimit(60), + option.WithIdentifier("enable-dark-mode"), + ), + hrp.NewStep("Complex Workflow"). + Android(). + StartToGoal("搜索'测试'，选择第一个结果，然后分享给朋友", + option.WithTimeLimit(180), + option.WithMaxRetryTimes(2), + ), + }, + } + + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} ``` \ No newline at end of file diff --git a/examples/game/llk/game_llk.json b/examples/game/llk/game_llk.json index 07e39089..f74ab27d 100644 --- a/examples/game/llk/game_llk.json +++ b/examples/game/llk/game_llk.json @@ -36,7 +36,7 @@ "method": "start_to_goal", "params": "连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：\n1. 游戏目标: 玩家需要通过连接相同的图案或图标，将它们从游戏界面中消除。\n2. 连接规则:\n- 两个相同的图案可以通过不超过三条直线连接。\n- 连接线可以水平或垂直，但不能斜线，也不能跨过其他图案。\n- 连接线的转折次数不能超过两次。\n3. 游戏界面:\n- 游戏界面是一个矩形区域，内含多个图案或图标，排列成行和列；图案或图标在未选中状态下背景为白色，选中状态下背景为绿色。\n- 游戏界面下方是道具区域，共有 3 种道具，从左到右分别是：「高亮显示」、「随机打乱」、「减少种类」。\n4、游戏攻略：建议多次使用道具，可以降低游戏难度\n- 优先使用「减少种类」道具，可以将图案种类随机减少一种\n- 遇到困难时，推荐使用「随机打乱」道具，可以获得很多新的消除机会\n- 观看广告视频，待屏幕右上角出现「领取成功」后，点击其右侧的 X 即可关闭广告，继续游戏\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "max_retry_times": 100 + "time_limit": 300 } } ] diff --git a/examples/game/llk/main_test.go b/examples/game/llk/main_test.go index a7eb28fb..d55667ed 100644 --- a/examples/game/llk/main_test.go +++ b/examples/game/llk/main_test.go @@ -7,13 +7,14 @@ import ( "os" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + hrp "github.com/httprunner/httprunner/v5" "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestGameLianliankan(t *testing.T) { @@ -45,7 +46,7 @@ func TestGameLianliankan(t *testing.T) { AssertAI("当前位于抖音「连了又连」小游戏页面"), hrp.NewStep("开始游戏"). Android(). - StartToGoal(userInstruction, option.WithMaxRetryTimes(100)), + StartToGoal(userInstruction, option.WithTimeLimit(300)), }, } err := testCase.Dump2JSON("game_llk.json") diff --git a/examples/game/romantic_restaurant/game_romantic_restaurant.json b/examples/game/romantic_restaurant/game_romantic_restaurant.json index b0aea6ce..2a5e2654 100644 --- a/examples/game/romantic_restaurant/game_romantic_restaurant.json +++ b/examples/game/romantic_restaurant/game_romantic_restaurant.json @@ -65,7 +65,7 @@ "method": "start_to_goal", "params": "浪漫餐厅是一款经营类游戏，以下是游戏的基本规则说明：\n1、点击右下角锅铲，开始任务\n2、将棋子拖拽至相同棋子，可升级生成新棋子\n3、拖拽相同棋子时，被部分遮挡的棋子只能作为拖拽终点，不能作为拖拽起点\n4、当游戏界面中没有相同棋子时，可以点击游戏页面中央的购物袋生成新的棋子\n5、若不知道如何操作，请按照游戏指引进行游玩\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "timeout": 300, + "time_limit": 300, "pre_mark_operation": true } } diff --git a/examples/game/romantic_restaurant/main_test.go b/examples/game/romantic_restaurant/main_test.go index 14af94a7..b028243e 100644 --- a/examples/game/romantic_restaurant/main_test.go +++ b/examples/game/romantic_restaurant/main_test.go @@ -42,7 +42,7 @@ func TestGameRomanticRestaurant(t *testing.T) { Android(). StartToGoal(userInstruction, option.WithPreMarkOperation(true), - option.WithTimeout(300)), // 5 minutes + option.WithTimeLimit(300)), // 5 minutes hrp.NewStep("退出抖音 app"). Android(). AppTerminate("$package_name"), diff --git a/examples/game/sudoku/game_sudoku.json b/examples/game/sudoku/game_sudoku.json index 063a8ccc..01e0f53f 100644 --- a/examples/game/sudoku/game_sudoku.json +++ b/examples/game/sudoku/game_sudoku.json @@ -65,7 +65,7 @@ "method": "start_to_goal", "params": "每天数独是一款逻辑推理游戏，玩家需要通过推理来确定黄色方块的所在位置，以下是游戏的基本规则说明：\n1、方块外面的数字代表所在那一行或一列的黄色方块数量。\n2、初始状态为白色方块，选择正确后变为黄色方块，选择错误后变为红底的 X。\n3、如果同一行或列有两个数字，则至少需要一个白底 X 分割它们作为间隔。\n4、如果数字与格子最大数相同时，该列或行必然全都是黄色方块。\n5、只能点击白色方块，不要重复点击同一个方块。\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "timeout": 300, + "time_limit": 300, "pre_mark_operation": true } } diff --git a/examples/game/sudoku/main_test.go b/examples/game/sudoku/main_test.go index dd509b9b..a8830dae 100644 --- a/examples/game/sudoku/main_test.go +++ b/examples/game/sudoku/main_test.go @@ -42,7 +42,7 @@ func TestGameSudoku(t *testing.T) { Android(). StartToGoal(userInstruction, option.WithPreMarkOperation(true), - option.WithTimeout(300)), // 5 minutes + option.WithTimeLimit(300)), // 5 minutes hrp.NewStep("退出抖音 app"). Android(). AppTerminate("$package_name"), diff --git a/examples/game/yanglegeyang/game_yanglegeyang.json b/examples/game/yanglegeyang/game_yanglegeyang.json index 8ce49876..1b31dcbc 100644 --- a/examples/game/yanglegeyang/game_yanglegeyang.json +++ b/examples/game/yanglegeyang/game_yanglegeyang.json @@ -65,7 +65,7 @@ "method": "start_to_goal", "params": "羊了个羊是一款热门的消除类小游戏，玩法简单但具有挑战性。以下是游戏的基本规则说明：\n1. 游戏目标: 玩家需要通过消除图案来完成关卡，最终目标是清空所有图案。\n2. 消除规则:\n- 游戏界面中会出现多个图案，玩家需要点击图案将其放入底部的槽中。\n- 图案存在多层堆叠的情况，只能点击最上层的完整图案。\n- 当槽中有三个相同的图案时，这三个图案会自动消除。\n- 玩家需要尽量避免槽中积累过多不同的图案，以免无法继续消除。\n- 严禁点击收集槽里的图案，严禁观看广告和使用道具（移出、撤回、洗牌）。\n- 请持续推进游戏进程，游戏通关后继续下一关，游戏失败后重新开始。\n3. 游戏界面: 图案通常以堆叠的方式呈现，玩家需要逐层消除。\n4. 关卡设计: 游戏包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。\n5. 策略性: 玩家需要规划消除顺序，以避免槽中积累过多无法消除的图案。\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "timeout": 300, + "time_limit": 300, "pre_mark_operation": true } } diff --git a/examples/game/yanglegeyang/main_test.go b/examples/game/yanglegeyang/main_test.go index 691c749c..6da55b1c 100644 --- a/examples/game/yanglegeyang/main_test.go +++ b/examples/game/yanglegeyang/main_test.go @@ -49,7 +49,7 @@ func TestGameYanglegeyang(t *testing.T) { Android(). StartToGoal(userInstruction, option.WithPreMarkOperation(true), - option.WithTimeout(300)), // 5 minutes + option.WithTimeLimit(300)), // 5 minutes hrp.NewStep("退出抖音 app"). Android(). AppTerminate("$package_name"), diff --git a/examples/game/yuedongxiaozi/game_yuedongxiaozi.json b/examples/game/yuedongxiaozi/game_yuedongxiaozi.json new file mode 100644 index 00000000..a7776210 --- /dev/null +++ b/examples/game/yuedongxiaozi/game_yuedongxiaozi.json @@ -0,0 +1,88 @@ +{ + "config": { + "name": "跃动小子小游戏自动化测试", + "variables": { + "package_name": "com.ss.android.ugc.aweme" + }, + "ai_options": { + "llm_service": "doubao-1.5-thinking-vision-pro-250428" + } + }, + "teststeps": [ + { + "name": "启动抖音 app", + "android": { + "os_type": "android", + "actions": [ + { + "method": "app_launch", + "params": "$package_name" + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_foreground_app", + "assert": "equal", + "expect": "$package_name", + "msg": "app [$package_name] should be in foreground" + } + ] + }, + { + "name": "启动「跃动小子」小游戏", + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "搜索「跃动小子」，启动小游戏", + "options": { + "pre_mark_operation": true + } + } + ] + }, + "validate": [ + { + "check": "ui_ai", + "assert": "ai_assert", + "expect": "当前页面底部包含「领地」「试炼」按钮", + "msg": "assert ai prompt [当前页面底部包含「领地」「试炼」按钮] failed" + } + ] + }, + { + "name": "开始游戏", + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "跃动小子是一款开宝箱类的小游戏，以下是游戏的基本规则说明：\n1、打开宝箱，按照游戏指引进行「出售」或「装备」操作。\n2、请持续推进游戏进程。\n3、屏幕底部的黑白按钮不要进行点击操作。\n\n请严格按照以上游戏规则，开始游戏\n", + "options": { + "time_limit": 300, + "pre_mark_operation": true + } + } + ] + } + }, + { + "name": "退出抖音 app", + "android": { + "os_type": "android", + "actions": [ + { + "method": "app_terminate", + "params": "$package_name" + } + ] + } + } + ] +} diff --git a/examples/game/yuedongxiaozi/main_test.go b/examples/game/yuedongxiaozi/main_test.go new file mode 100644 index 00000000..29ac3882 --- /dev/null +++ b/examples/game/yuedongxiaozi/main_test.go @@ -0,0 +1,55 @@ +package game_yuedongxiaozi + +import ( + "testing" + + "github.com/stretchr/testify/require" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" +) + +func TestGameZhuadaE(t *testing.T) { + userInstruction := `跃动小子是一款开宝箱类的小游戏，以下是游戏的基本规则说明： +1、打开宝箱，按照游戏指引进行「出售」或「装备」操作。 +2、请持续推进游戏进程。 +3、屏幕底部的黑白按钮不要进行点击操作。 + +请严格按照以上游戏规则，开始游戏 +` + + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("跃动小子小游戏自动化测试"). + SetLLMService(option.DOUBAO_1_5_THINKING_VISION_PRO_250428). + WithVariables(map[string]interface{}{ + "package_name": "com.ss.android.ugc.aweme", + }), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动抖音 app"). + Android(). + AppLaunch("$package_name"). + Sleep(5). + Validate(). + AssertAppInForeground("$package_name"), + hrp.NewStep("启动「跃动小子」小游戏"). + Android(). + StartToGoal("搜索「跃动小子」，启动小游戏", + option.WithPreMarkOperation(true)). + Validate(). + AssertAI("当前页面底部包含「领地」「试炼」按钮"), + hrp.NewStep("开始游戏"). + Android(). + StartToGoal(userInstruction, + option.WithPreMarkOperation(true), + option.WithTimeLimit(300)), // 5 minutes + hrp.NewStep("退出抖音 app"). + Android(). + AppTerminate("$package_name"), + }, + } + err := testCase.Dump2JSON("game_yuedongxiaozi.json") + require.Nil(t, err) + + // err = hrp.NewRunner(t).Run(testCase) + // assert.Nil(t, err) +} diff --git a/examples/game/zhuadae/game_zhuadae.json b/examples/game/zhuadae/game_zhuadae.json index 5fe446dd..991f6795 100644 --- a/examples/game/zhuadae/game_zhuadae.json +++ b/examples/game/zhuadae/game_zhuadae.json @@ -88,7 +88,7 @@ "method": "start_to_goal", "params": "抓大鹅是一款抓取类小游戏，以下是游戏的基本规则说明：\n1. 游戏目标: 玩家需要通过抓取图案来完成关卡，最终目标是清空所有图案。\n2. 抓取规则:\n- 游戏界面中会出现多个图案，图案存在多层堆叠的情况，玩家需要点击图案将其抓取放入到槽中。\n- 当抓取到三个相同的图案放入抓取槽时，这三个图案会成功消除。\n- 需要尽量避免抓取槽满的情况，抓取槽满时游戏失败。\n- 游戏通关后继续进入下一关，游戏失败后重新开始游戏。\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "timeout": 300, + "time_limit": 300, "pre_mark_operation": true } } diff --git a/examples/game/zhuadae/main_test.go b/examples/game/zhuadae/main_test.go index 239a00e4..bffc5cdf 100644 --- a/examples/game/zhuadae/main_test.go +++ b/examples/game/zhuadae/main_test.go @@ -50,7 +50,7 @@ func TestGameZhuadaE(t *testing.T) { Android(). StartToGoal(userInstruction, option.WithPreMarkOperation(true), - option.WithTimeout(300)), // 5 minutes + option.WithTimeLimit(300)), // 5 minutes hrp.NewStep("退出抖音 app"). Android(). AppTerminate("$package_name"), diff --git a/internal/version/VERSION b/internal/version/VERSION index bce4fd6e..0e7166d7 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250703 +v5.0.0-250704 diff --git a/uixt/ai/cv_vedem_test.go b/uixt/ai/cv_vedem_test.go index ee2d2636..1524061f 100644 --- a/uixt/ai/cv_vedem_test.go +++ b/uixt/ai/cv_vedem_test.go @@ -23,7 +23,7 @@ func TestGetImageFromBuffer(t *testing.T) { require.Nil(t, err) cvResult, err := service.ReadFromBuffer(buf) assert.Nil(t, err) - fmt.Println(fmt.Sprintf("cvResult: %v", cvResult)) + fmt.Printf("cvResult: %v\n", cvResult) } func TestGetImageFromPath(t *testing.T) { @@ -32,5 +32,5 @@ func TestGetImageFromPath(t *testing.T) { require.Nil(t, err) cvResult, err := service.ReadFromPath(imagePath) assert.Nil(t, err) - fmt.Println(fmt.Sprintf("cvResult: %v", cvResult)) + fmt.Printf("cvResult: %v\n", cvResult) } diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 97d9393e..2f1eca55 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -21,18 +21,24 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op if options.MaxRetryTimes > 0 { logger = logger.Int("max_retry_times", options.MaxRetryTimes) } - if options.Timeout > 0 { - logger = logger.Int("timeout_seconds", options.Timeout) - } - logger.Msg("StartToGoal") - // Create timeout context for entire StartToGoal process if Timeout is specified - if options.Timeout > 0 { + // Handle TimeLimit and Timeout with unified context mechanism + var isTimeLimitMode bool + if options.TimeLimit > 0 { + // TimeLimit takes precedence over Timeout + logger = logger.Int("time_limit_seconds", options.TimeLimit) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(options.TimeLimit)*time.Second) + defer cancel() + isTimeLimitMode = true + } else if options.Timeout > 0 { + // Use Timeout only if TimeLimit is not set + logger = logger.Int("timeout_seconds", options.Timeout) var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) defer cancel() - log.Info().Int("timeout_seconds", options.Timeout).Msg("StartToGoal timeout configured for entire process") } + logger.Msg("StartToGoal") var allPlannings []*PlanningExecutionResult var attempt int @@ -40,16 +46,26 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op attempt++ log.Info().Int("attempt", attempt).Msg("planning attempt") - // Check for context cancellation (interrupt signal or timeout) + // Check for context cancellation (timeout, time limit, or interrupt) select { case <-ctx.Done(): cause := context.Cause(ctx) + // Handle TimeLimit timeout - return success + if isTimeLimitMode && errors.Is(cause, context.DeadlineExceeded) { + log.Info(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Int("time_limit_seconds", options.TimeLimit). + Msg("StartToGoal time limit reached, stopping gracefully") + return allPlannings, nil + } + + // Handle other cancellations (Timeout, interrupt, external cancellation) - return error log.Warn(). Int("attempt", attempt). Int("completed_plannings", len(allPlannings)). Err(cause). Msg("StartToGoal cancelled") - // Return the specific error type based on the cancellation cause return allPlannings, errors.Wrap(cause, "StartToGoal cancelled") default: } @@ -99,10 +115,25 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Invoke tool calls for _, toolCall := range planningResult.ToolCalls { - // Check for context cancellation before each action + // Check for context cancellation (timeout, time limit, or interrupt) before each action select { case <-ctx.Done(): cause := context.Cause(ctx) + // Handle TimeLimit timeout - return success + if isTimeLimitMode && errors.Is(cause, context.DeadlineExceeded) { + log.Info(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Int("completed_tool_calls", len(planningResult.SubActions)). + Int("total_tool_calls", len(planningResult.ToolCalls)). + Int("time_limit_seconds", options.TimeLimit). + Msg("StartToGoal time limit reached during tool call execution, stopping gracefully") + planningResult.Elapsed = time.Since(planningStartTime).Milliseconds() + allPlannings = append(allPlannings, planningResult) + return allPlannings, nil + } + + // Handle other cancellations (Timeout, external cancellation) - return error log.Warn(). Int("attempt", attempt). Int("completed_plannings", len(allPlannings)). @@ -112,7 +143,6 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op Msg("invokeToolCalls cancelled") planningResult.Elapsed = time.Since(planningStartTime).Milliseconds() allPlannings = append(allPlannings, planningResult) - // Return the specific error type based on the cancellation cause return allPlannings, errors.Wrap(cause, "invokeToolCalls cancelled") default: } diff --git a/uixt/option/action.go b/uixt/option/action.go index 8f473bb7..83ec760e 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -201,6 +201,7 @@ type ActionOptions struct { Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` + TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` ScreenOptions @@ -281,6 +282,9 @@ func (o *ActionOptions) Options() []ActionOption { if o.Timeout != 0 { options = append(options, WithTimeout(o.Timeout)) } + if o.TimeLimit != 0 { + options = append(options, WithTimeLimit(o.TimeLimit)) + } if o.Frequency != 0 { options = append(options, WithFrequency(o.Frequency)) } @@ -562,6 +566,12 @@ func WithTimeout(seconds int) ActionOption { } } +func WithTimeLimit(seconds int) ActionOption { + return func(o *ActionOptions) { + o.TimeLimit = seconds + } +} + func WithIgnoreNotFoundError(ignoreError bool) ActionOption { return func(o *ActionOptions) { o.IgnoreNotFoundError = ignoreError From e734424382cc8fee11f6731018986145efe92590 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 4 Jul 2025 14:32:07 +0800 Subject: [PATCH 2/4] refactor: add WDA/UIA logs to summary --- runner.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/runner.go b/runner.go index 62b266d0..a1f13c4a 100644 --- a/runner.go +++ b/runner.go @@ -711,27 +711,27 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) (summary *TestCa } } - // TODO: move to mobile ui step // Collect logs from cached drivers for _, cached := range uixt.ListCachedDrivers() { - // add WDA/UIA logs to summary + client := cached.Item + if !client.GetDevice().LogEnabled() { + continue + } + + // add WDA/UIA2 logs to summary logs := map[string]interface{}{ "uuid": cached.Key, } - - client := cached.Item - if client.GetDevice().LogEnabled() { - log, err1 := client.StopCaptureLog() - if err1 != nil { - if err == nil { - err = errors.Wrap(err1, "stop capture log failed") - } else { - err = errors.Wrap(err, "stop capture log failed") - } - return + log, err1 := client.StopCaptureLog() + if err1 != nil { + if err == nil { + err = errors.Wrap(err1, "stop capture log failed") + } else { + err = errors.Wrap(err, "stop capture log failed") } - logs["content"] = log + return } + logs["content"] = log summary.Logs = append(summary.Logs, logs) } From 55c4f8b9b53b91d4f44be9fc99242f7bf6018013 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 4 Jul 2025 14:43:30 +0800 Subject: [PATCH 3/4] test: add android example as demo --- examples/uitest/demo_android_uia2_log.json | 170 ++++++++++++++++++ examples/uitest/demo_android_uia2_log_test.go | 60 +++++++ 2 files changed, 230 insertions(+) create mode 100644 examples/uitest/demo_android_uia2_log.json create mode 100644 examples/uitest/demo_android_uia2_log_test.go diff --git a/examples/uitest/demo_android_uia2_log.json b/examples/uitest/demo_android_uia2_log.json new file mode 100644 index 00000000..1f1f172b --- /dev/null +++ b/examples/uitest/demo_android_uia2_log.json @@ -0,0 +1,170 @@ +{ + "config": { + "name": "验证 UIA2 打点数据准确性", + "variables": { + "app_name": "抖音" + }, + "android": [ + { + "log_on": true, + "adb_server_host": "localhost", + "adb_server_port": 5037, + "uia2_ip": "localhost", + "uia2_port": 6790, + "uia2_server_package_name": "io.appium.uiautomator2.server", + "uia2_server_test_package_name": "io.appium.uiautomator2.server.test" + } + ] + }, + "teststeps": [ + { + "name": "启动抖音", + "android": { + "os_type": "android", + "actions": [ + { + "method": "home" + }, + { + "method": "app_terminate", + "params": "com.ss.android.ugc.aweme" + }, + { + "method": "swipe_to_tap_app", + "params": "$app_name", + "options": { + "identifier": "启动抖音", + "max_retry_times": 5, + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_ocr", + "assert": "exists", + "expect": "推荐", + "msg": "抖音启动失败，「推荐」不存在" + } + ] + }, + { + "name": "处理青少年弹窗", + "android": { + "os_type": "android", + "actions": [ + { + "method": "tap_ocr", + "params": "我知道了", + "options": { + "ignore_NotFoundError": true + } + } + ] + } + }, + { + "name": "进入推荐页", + "android": { + "os_type": "android", + "actions": [ + { + "method": "tap_ocr", + "params": "推荐", + "options": { + "identifier": "点击推荐", + "offset": [ + 0, + -1 + ], + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 5 + } + ] + } + }, + { + "name": "向上滑动 2 次", + "android": { + "os_type": "android", + "actions": [ + { + "method": "swipe_direction", + "params": "up", + "options": { + "identifier": "第 1 次上划", + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "swipe_direction", + "params": "up", + "options": { + "identifier": "第 2 次上划", + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "swipe_direction", + "params": "up", + "options": { + "identifier": "第 3 次上划", + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "tap_xy", + "params": [ + 0.9, + 0.1 + ], + "options": { + "identifier": "点击进入搜索框", + "pre_mark_operation": true + } + }, + { + "method": "sleep", + "params": 2 + }, + { + "method": "input", + "params": "httprunner 发版记录", + "options": { + "identifier": "输入搜索关键词", + "pre_mark_operation": true + } + }, + { + "method": "tap_ocr", + "params": "搜索", + "options": { + "identifier": "点击搜索" + } + } + ] + } + } + ] +} diff --git a/examples/uitest/demo_android_uia2_log_test.go b/examples/uitest/demo_android_uia2_log_test.go new file mode 100644 index 00000000..a890bf96 --- /dev/null +++ b/examples/uitest/demo_android_uia2_log_test.go @@ -0,0 +1,60 @@ +//go:build localtest + +package uitest + +import ( + "testing" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" +) + +func TestUIA2Log(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("验证 UIA2 打点数据准确性"). + WithVariables(map[string]interface{}{ + "app_name": "抖音", + }). + SetAndroid( + option.WithAdbLogOn(true), + ), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动抖音"). + Android(). + Home(). + AppTerminate("com.ss.android.ugc.aweme"). // 关闭已运行的抖音 + SwipeToTapApp("$app_name", + option.WithMaxRetryTimes(5), + option.WithIdentifier("启动抖音"), + option.WithPreMarkOperation(true)).Sleep(5). + Validate(). + AssertOCRExists("推荐", "抖音启动失败，「推荐」不存在"), + hrp.NewStep("处理青少年弹窗"). + Android(). + TapByOCR("我知道了", + option.WithIgnoreNotFoundError(true)), + hrp.NewStep("进入推荐页"). + Android().TapByOCR("推荐", + option.WithIdentifier("点击推荐"), + option.WithPreMarkOperation(true), + option.WithTapOffset(0, -1)).Sleep(5), + hrp.NewStep("向上滑动 2 次"). + Android(). + SwipeUp(option.WithIdentifier("第 1 次上划"), option.WithPreMarkOperation(true)).Sleep(2). + SwipeUp(option.WithIdentifier("第 2 次上划"), option.WithPreMarkOperation(true)).Sleep(2). + SwipeUp(option.WithIdentifier("第 3 次上划"), option.WithPreMarkOperation(true)).Sleep(2). + TapXY(0.9, 0.1, option.WithIdentifier("点击进入搜索框"), option.WithPreMarkOperation(true)).Sleep(2). + Input("httprunner 发版记录", option.WithIdentifier("输入搜索关键词"), option.WithPreMarkOperation(true)). + TapByOCR("搜索", option.WithIdentifier("点击搜索")), + }, + } + + if err := testCase.Dump2JSON("demo_android_uia2_log.json"); err != nil { + t.Fatal(err) + } + + err := hrp.Run(t, testCase) + if err != nil { + t.Fatal(err) + } +} From b5514ee460d7337bd3a874c0ed6a18077ae6e49f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 4 Jul 2025 18:34:27 +0800 Subject: [PATCH 4/4] fix: Remove identifier for swipe operations to avoid WDA/UIA2 logging --- uixt/driver_ext_swipe.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/uixt/driver_ext_swipe.go b/uixt/driver_ext_swipe.go index cfbda133..d8012fa0 100644 --- a/uixt/driver_ext_swipe.go +++ b/uixt/driver_ext_swipe.go @@ -98,6 +98,12 @@ func (dExt *XTDriver) SwipeToTapTexts(texts []string, opts ...option.ActionOptio log.Info().Strs("texts", texts).Msg("swipe to tap texts") opts = append(opts, option.WithMatchOne(true), option.WithRegex(true)) + + // Remove identifier for swipe operations to avoid WDA/UIA2 logging + actionOptions := option.NewActionOptions(opts...) + actionOptions.Identifier = "" + optionsWithoutIdentifier := actionOptions.Options() + var point ai.PointF findTexts := func(d *XTDriver) error { var err error @@ -129,7 +135,7 @@ func (dExt *XTDriver) SwipeToTapTexts(texts []string, opts ...option.ActionOptio return d.TapAbsXY(point.X, point.Y, opts...) } - findAction := prepareSwipeAction(dExt, nil, opts...) + findAction := prepareSwipeAction(dExt, nil, optionsWithoutIdentifier...) return dExt.LoopUntil(findAction, findTexts, foundTextAction, opts...) } @@ -146,15 +152,19 @@ func (dExt *XTDriver) SwipeToTapApp(appName string, opts ...option.ActionOption) log.Error().Err(err).Msg("auto handle popup failed") } + // Remove identifier for swipe operations to avoid WDA/UIA2 logging + actionOptions := option.NewActionOptions(opts...) + actionOptions.Identifier = "" + optionsWithoutIdentifier := actionOptions.Options() + // swipe to first screen for i := 0; i < 5; i++ { - dExt.Swipe(0.5, 0.5, 0.9, 0.5, opts...) + dExt.Swipe(0.5, 0.5, 0.9, 0.5, optionsWithoutIdentifier...) } opts = append(opts, option.WithDirection("left")) opts = append(opts, option.WithMaxRetryTimes(5)) - actionOptions := option.NewActionOptions(opts...) // tap app icon above the text if len(actionOptions.TapOffset) == 0 { opts = append(opts, option.WithTapOffset(0, -100))