From 93007d5eb7dc1cf04392ba07f915bd944ceb7676 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 23 Jun 2025 15:04:32 +0800 Subject: [PATCH 01/16] docs: add summary structure --- docs/summary-structure.md | 437 ++++++++++++++++++++++++++++++++++++++ internal/version/VERSION | 2 +- 2 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 docs/summary-structure.md diff --git a/docs/summary-structure.md b/docs/summary-structure.md new file mode 100644 index 00000000..c7785f68 --- /dev/null +++ b/docs/summary-structure.md @@ -0,0 +1,437 @@ +# Summary 数据结构说明文档 + +## 概述 + +HttpRunner 的 Summary 数据结构用于存储测试执行的完整汇总信息，包括测试结果、统计数据、时间信息、平台信息以及详细的测试步骤记录。本文档基于 `summary.go` 的代码定义和实际执行产物进行详细说明。 + +## 数据结构层次关系 + +``` +Summary (根结构) +├── Success (bool) +├── Stat (统计信息) +│ ├── TestCases (测试用例统计) +│ └── TestSteps (测试步骤统计) +├── Time (时间信息) +├── Platform (平台信息) +└── Details (测试用例详情列表) + └── TestCaseSummary (单个测试用例汇总) + ├── Stat (步骤统计) + ├── Time (用例时间) + ├── InOut (输入输出) + ├── Logs (日志) + └── Records (步骤记录) + └── StepResult (步骤结果) + ├── Data (步骤数据) + │ ├── ReqResps (请求响应) + │ └── Validators (验证器) + ├── Actions (操作列表) + │ ├── Requests (请求记录) + │ ├── Plannings (AI规划) + │ │ ├── ToolCalls (工具调用) + │ │ ├── Usage (模型使用统计) + │ │ ├── ScreenResult (屏幕结果) + │ │ └── SubActions (子操作) + │ └── ScreenResults (屏幕截图) + └── Attachments (附件信息) +``` + +## 详细数据结构说明 + +### 1. Summary (主汇总结构) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Success | bool | `success` | 整体测试执行是否成功 | +| Stat | *Stat | `stat` | 汇总统计信息 | +| Time | *TestCaseTime | `time` | 整体执行时间信息 | +| Platform | *Platform | `platform` | 平台和版本信息 | +| Details | []*TestCaseSummary | `details` | 各个测试用例的详细信息 | +| rootDir | string | - | 根目录路径（私有字段） | + +**示例数据**: +```json +{ + "success": true, + "stat": { /* 统计信息 */ }, + "time": { /* 时间信息 */ }, + "platform": { /* 平台信息 */ }, + "details": [ /* 测试用例详情 */ ] +} +``` + +### 2. Stat (统计信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| TestCases | TestCaseStat | `testcases` | 测试用例统计 | +| TestSteps | TestStepStat | `teststeps` | 测试步骤统计 | + +### 3. TestCaseStat (测试用例统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Total | int | `total` | 测试用例总数 | +| Success | int | `success` | 成功的测试用例数 | +| Fail | int | `fail` | 失败的测试用例数 | + +**示例数据**: +```json +{ + "testcases": { + "total": 1, + "success": 1, + "fail": 0 + } +} +``` + +### 4. TestStepStat (测试步骤统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Total | int | `total` | 测试步骤总数 | +| Successes | int | `successes` | 成功的步骤数 | +| Failures | int | `failures` | 失败的步骤数 | +| Actions | map[option.ActionName]int | `actions` | 各种操作的统计计数 | + +**示例数据**: +```json +{ + "teststeps": { + "total": 5, + "successes": 5, + "failures": 0, + "actions": {} + } +} +``` + +### 5. TestCaseTime (时间信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| StartAt | time.Time | `start_at,omitempty` | 开始时间 | +| Duration | float64 | `duration,omitempty` | 持续时间（秒） | + +**示例数据**: +```json +{ + "time": { + "start_at": "2025-06-23T13:41:06.150641+08:00", + "duration": 188.998332334 + } +} +``` + +### 6. Platform (平台信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| HttprunnerVersion | string | `httprunner_version` | HttpRunner 版本号 | +| GoVersion | string | `go_version` | Go 语言版本 | +| Platform | string | `platform` | 操作系统平台信息 | + +**示例数据**: +```json +{ + "platform": { + "httprunner_version": "v5.0.0-beta-2506222254", + "go_version": "go1.24.1", + "platform": "darwin-arm64" + } +} +``` + +### 7. TestCaseSummary (测试用例汇总) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Name | string | `name` | 测试用例名称 | +| Success | bool | `success` | 用例执行是否成功 | +| CaseId | string | `case_id,omitempty` | 用例ID（可选） | +| Stat | *TestStepStat | `stat` | 该用例的步骤统计 | +| Time | *TestCaseTime | `time` | 该用例的时间信息 | +| InOut | *TestCaseInOut | `in_out` | 输入输出信息 | +| Logs | []interface{} | `logs,omitempty` | 日志信息 | +| Records | []*StepResult | `records` | 步骤执行记录 | +| RootDir | string | `root_dir,omitempty` | 根目录路径 | + +### 8. TestCaseInOut (输入输出信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ConfigVars | map[string]interface{} | `config_vars` | 配置变量 | +| ExportVars | map[string]interface{} | `export_vars` | 导出变量 | + +**示例数据**: +```json +{ + "in_out": { + "config_vars": { + "OPENAI_API_KEY": "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", + "dramaName": "涂山赊刀", + "userName": "青榕小剧场" + }, + "export_vars": {} + } +} +``` + +### 9. StepResult (步骤结果) + +步骤结果是 Records 数组中的元素，包含每个测试步骤的详细执行信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| name | string | `name` | 步骤名称 | +| start_time | int64 | `start_time` | 开始时间（Unix时间戳，毫秒） | +| step_type | string | `step_type` | 步骤类型（如 "android_validation", "android"） | +| success | bool | `success` | 步骤执行是否成功 | +| elapsed_ms | int | `elapsed_ms` | 执行耗时（毫秒） | +| data | *SessionData | `data` | 步骤相关数据（包含请求响应和验证结果） | +| actions | []Action | `actions` | 执行的操作列表 | +| attachments | map[string]interface{} | `attachments` | 附件信息（如截图等） | + +**示例数据**: +```json +{ + "name": "启动快手 app", + "start_time": 1750657267057, + "step_type": "android_validation", + "success": true, + "elapsed_ms": 8797, + "data": { /* 步骤数据 */ }, + "actions": [ /* 操作列表 */ ], + "attachments": { /* 附件信息 */ } +} +``` + +### 10. SessionData (步骤数据) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ReqResps | *ReqResps | `req_resps` | 请求响应数据 | +| Address | *Address | `address,omitempty` | 网络地址信息 | +| Validators | []*ValidationResult | `validators,omitempty` | 验证结果列表 | + +### 11. ReqResps (请求响应) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Request | interface{} | `request` | 请求信息 | +| Response | interface{} | `response` | 响应信息 | + +### 12. ValidationResult (验证结果) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Check | string | `check` | 验证检查项 | +| Assert | string | `assert` | 断言类型 | +| Expect | interface{} | `expect` | 期望值 | +| Msg | string | `msg` | 验证消息 | +| CheckValue | interface{} | `check_value` | 实际检查值 | +| CheckResult | string | `check_result` | 检查结果（"pass"/"fail"） | + +**示例数据**: +```json +{ + "check": "ui_foreground_app", + "assert": "equal", + "expect": "com.smile.gifmaker", + "msg": "app [com.smile.gifmaker] should be in foreground", + "check_value": null, + "check_result": "pass" +} +``` + +### 13. Action (操作信息) + +每个步骤可能包含多个操作： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| method | string | `method` | 操作方法名（如 "app_launch", "start_to_goal"） | +| params | interface{} | `params` | 操作参数 | +| start_time | int64 | `start_time` | 操作开始时间（Unix时间戳，毫秒） | +| elapsed_ms | int | `elapsed_ms` | 操作耗时（毫秒） | +| requests | []Request | `requests` | HTTP请求记录（如果有） | +| plannings | []Planning | `plannings` | AI规划信息（UI自动化场景） | +| screen_results | []ScreenResult | `screen_results` | 屏幕截图结果 | + +**示例数据**: +```json +{ + "method": "start_to_goal", + "params": "搜索「青榕小剧场」，切换到「用户」搜索结果页，点击进入第一个搜索结果的用户个人主页", + "start_time": 1750657275855, + "elapsed_ms": 109543, + "plannings": [ /* AI规划列表 */ ] +} +``` + +### 14. Request (请求记录) + +ADB或HTTP请求的详细记录： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| request_method | string | `request_method` | 请求方法（如 "adb", "http"） | +| request_url | string | `request_url` | 请求URL或命令 | +| request_body | string | `request_body` | 请求体或命令参数 | +| request_time | string | `request_time` | 请求时间（ISO格式） | +| response_status | int | `response_status` | 响应状态码 | +| response_duration_ms | int | `response_duration(ms)` | 响应耗时（毫秒） | +| response_body | string | `response_body` | 响应内容 | +| success | bool | `success` | 请求是否成功 | + +**示例数据**: +```json +{ + "request_method": "adb", + "request_url": "monkey", + "request_body": "-p com.smile.gifmaker -c android.intent.category.LAUNCHER 1", + "request_time": "2025-06-23T13:41:07.200504+08:00", + "response_status": 0, + "response_duration(ms)": 566, + "response_body": "Events injected: 1\n## Network stats: elapsed time=45ms", + "success": true +} +``` + +### 15. Planning (AI规划信息) + +UI自动化测试中的AI规划详情： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| tool_calls | []ToolCall | `tool_calls` | 工具调用信息 | +| thought | string | `thought` | AI的思考过程 | +| content | string | `content` | 规划内容（JSON格式的操作描述） | +| model_name | string | `model_name` | 使用的AI模型名称 | +| usage | Usage | `usage` | 模型使用统计 | +| screenshot_elapsed_ms | int | `screenshot_elapsed_ms` | 截图耗时（毫秒） | +| image_path | string | `image_path` | 截图文件路径 | +| resolution | Resolution | `resolution` | 屏幕分辨率 | +| screen_result | ScreenResult | `screen_result` | 屏幕分析结果 | +| model_call_elapsed_ms | int | `model_call_elapsed_ms` | 模型调用耗时（毫秒） | +| tool_calls_count | int | `tool_calls_count` | 工具调用次数 | +| action_names | []string | `action_names` | 执行的操作名称列表 | +| start_time | int64 | `start_time` | 规划开始时间 | +| elapsed_ms | int | `elapsed_ms` | 规划总耗时 | +| sub_actions | []SubAction | `sub_actions` | 子操作列表 | + +**示例数据**: +```json +{ + "tool_calls": [ + { + "id": "tap_xy_1750657286", + "type": "function", + "function": { + "name": "uixt__tap_xy", + "arguments": "{\"x\":1107.6,\"y\":232.4}" + } + } + ], + "thought": "点击页面右上角的搜索图标，打开搜索界面以进行后续搜索操作。", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 2199, + "completion_tokens": 135, + "total_tokens": 2334 + } +} +``` + +### 16. ToolCall (工具调用) + +AI规划中的工具调用信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| id | string | `id` | 工具调用唯一标识 | +| type | string | `type` | 调用类型（通常为 "function"） | +| function | Function | `function` | 函数调用详情 | + +### 17. Function (函数调用) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| name | string | `name` | 函数名称（如 "uixt__tap_xy"） | +| arguments | string | `arguments` | 函数参数（JSON字符串格式） | + +### 18. Usage (模型使用统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| prompt_tokens | int | `prompt_tokens` | 输入token数量 | +| completion_tokens | int | `completion_tokens` | 输出token数量 | +| total_tokens | int | `total_tokens` | 总token数量 | + +### 19. Resolution (分辨率) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| width | int | `width` | 屏幕宽度（像素） | +| height | int | `height` | 屏幕高度（像素） | + +### 20. ScreenResult (屏幕结果) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| image_path | string | `image_path` | 截图文件路径 | +| resolution | Resolution | `resolution` | 屏幕分辨率 | +| uploaded_url | string | `uploaded_url` | 上传后的URL（通常为空） | +| texts | []Text | `texts` | 识别的文本信息（可为null） | +| icons | []Icon | `icons` | 识别的图标信息（可为null） | +| tags | []Tag | `tags` | 识别的标签信息（可为null） | + +### 21. SubAction (子操作) + +规划中实际执行的具体操作： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| action_name | string | `action_name` | 操作名称（如 "uixt__tap_xy"） | +| arguments | string | `arguments` | 操作参数（JSON字符串） | +| start_time | int64 | `start_time` | 操作开始时间 | +| elapsed_ms | int | `elapsed_ms` | 操作耗时 | +| requests | []Request | `requests` | 相关的请求记录 | +| screen_results | []ScreenResult | `screen_results` | 操作后的屏幕截图 | + +**示例数据**: +```json +{ + "action_name": "uixt__tap_xy", + "arguments": "{\"x\":1107.6,\"y\":232.4}", + "start_time": 1750657286274, + "elapsed_ms": 319, + "requests": [ /* 请求记录 */ ], + "screen_results": [ /* 屏幕截图 */ ] +} +``` + +### 22. Attachments (附件信息) + +步骤执行过程中产生的附件，主要是截图： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| screen_results | []ScreenResult | `screen_results` | 屏幕截图列表 | + +## 数据类型层次关系 + +### 时间戳格式 +- **Unix时间戳（毫秒）**: 用于 `start_time` 字段，如 `1750657267057` +- **ISO时间格式**: 用于 `start_at` 和 `request_time` 字段，如 `"2025-06-23T13:41:06.150641+08:00"` + +### 耗时统计 +- **毫秒级**: `elapsed_ms`, `response_duration(ms)`, `screenshot_elapsed_ms`, `model_call_elapsed_ms` +- **秒级**: `duration` 字段使用浮点数表示秒 + +### 状态标识 +- **布尔值**: `success`, `Success` 表示操作或测试是否成功 +- **字符串**: `check_result` 使用 "pass"/"fail" 表示验证结果 + +这个数据结构设计充分考虑了测试执行的各种场景，特别是UI自动化测试中的复杂交互和AI规划过程，为测试结果的分析和报告提供了完整的数据基础。通过详细的嵌套字段定义，开发者可以精确理解和使用每个数据元素，实现更强大的测试分析和报告功能。 \ No newline at end of file diff --git a/internal/version/VERSION b/internal/version/VERSION index 3c80d5e7..2f551a4e 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506222254 +v5.0.0-beta-2506231504 From b320bbda3107e8a327ace2a8be43cf6dccde1bbf Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 23 Jun 2025 21:14:29 +0800 Subject: [PATCH 02/16] feat: add parameterization support for test cases and steps with configuration options --- config.go | 14 ++ docs/parameters.md | 387 +++++++++++++++++++++++++++++++ examples/parameters/main_test.go | 135 +++++++++++ internal/version/VERSION | 2 +- parameters.go | 43 +++- runner.go | 195 +++++++++++++--- step.go | 20 +- step_request.go | 22 +- 8 files changed, 772 insertions(+), 46 deletions(-) create mode 100644 docs/parameters.md create mode 100644 examples/parameters/main_test.go diff --git a/config.go b/config.go index 34399576..cbb165b5 100644 --- a/config.go +++ b/config.go @@ -88,6 +88,20 @@ func (c *TConfig) WithParameters(parameters map[string]interface{}) *TConfig { return c } +// WithParametersSetting sets parameters setting for current testcase. +func (c *TConfig) WithParametersSetting(options ...ParametersOption) *TConfig { + if c.ParametersSetting == nil { + c.ParametersSetting = &TParamsConfig{} + } + + // apply all options + for _, option := range options { + option(c.ParametersSetting) + } + + return c +} + // SetThinkTime sets think time config for current testcase. func (c *TConfig) SetThinkTime(strategy ThinkTimeStrategy, cfg interface{}, limit float64) *TConfig { c.ThinkTimeSetting = &ThinkTimeConfig{strategy, cfg, limit} diff --git a/docs/parameters.md b/docs/parameters.md new file mode 100644 index 00000000..c8c9aeb3 --- /dev/null +++ b/docs/parameters.md @@ -0,0 +1,387 @@ +# HttpRunner 参数化功能 (Parameters) + +## 概述 + +HttpRunner 支持强大的**数据驱动测试**能力，允许用户在**测试用例（Testcase）**和**测试步骤（Step）**两个层级上进行参数化。这使得测试用例可以与外部数据文件解耦，实现更灵活、可维护性更高的自动化测试。 + +- **测试用例层级参数化**：对整个测试流程使用多组不同的数据重复执行。适用于需要验证完整业务流程的场景，例如使用不同用户登录并执行相同操作。 +- **测试步骤层级参数化**：仅在单个步骤内使用不同的参数重复执行。适用于需要验证单个功能点的场景，例如在搜索框中输入不同的关键词。 + +## 测试用例层级参数化 (TestCase-Level) + +当您需要使用不同的数据集完整地运行整个测试流程时，应使用测试用例层级的参数化。 + +### 使用方法 + +通过在 `hrp.TestCase` 的 `Parameters` 字段中定义参数，并可选择使用 `WithParametersSetting` 进行策略配置。 + +```go +// testcase_parameters_test.go +func TestTestcaseParameters(t *testing.T) { + testcase := &hrp.TestCase{ + Config: hrp.NewConfig("测试用例层级参数化"). + WithParameters(map[string]interface{}{ + "username-password": [][]interface{}{ + {"user1", "pass1"}, + {"user2", "pass2"}, + {"user3", "pass3"}, + }, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机选择参数 + hrp.WithLimit(2), // 只执行2次 + ), + TestSteps: []hrp.IStep{ + hrp.NewStep("登录"). + POST("/api/login"). + WithBody(map[string]interface{}{ + "username": "$username", + "password": "$password", + }), + hrp.NewStep("获取用户信息"). + GET("/api/user/info"), + }, + } + + err := hrp.NewRunner(t).Run(testcase) + assert.Nil(t, err) +} +``` + +### 执行结果 + +- 整个测试用例将运行两次。 +- 第一次运行时，`$username` 为 `user1`，`$password` 为 `pass1`。 +- 第二次运行时，`$username` 为 `user2`，`$password` 为 `pass2`。 +- 测试报告中会生成两个独立的测试结果，每个结果对应一组参数。 + +--- + +## 测试步骤层级参数化 (TestStep-Level) + +当您只需要在一个测试流程中对某个特定步骤使用不同参数进行多次测试时，应使用测试步骤层级的参数化。 + +### 使用场景 + +- 需要对同一个操作使用不同参数进行多次测试 +- 减少重复的步骤定义代码 +- 在 UI 自动化测试中，对同一个界面操作使用不同的输入数据 + +### 核心设计 + +#### 1. 架构改动 + +##### StepConfig 结构扩展 +```go +type StepConfig struct { + StepName string `json:"name" yaml:"name"` + Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` + Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` // 新增 + ParametersSetting *TParamsConfig `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` // 新增 + Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` // 已弃用，建议使用 Loop() 方法 + // ... 其他字段保持不变 +} +``` + +##### TParamsConfig 参数配置 +```go +type TParamsConfig struct { + PickOrder iteratorPickOrder `json:"pick_order,omitempty" yaml:"pick_order,omitempty"` + Strategies map[string]IteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` +} +``` + +#### 2. Limit 和 Loops 的关系 + +**重要**：`Limit` 和 `Loops` 不是替代关系，而是可以同时存在的不同控制机制： + +1. **Limit（总数限制）**： + - 作用：限制参数迭代的**总执行次数** + - 位置：`TParamsConfig.Limit` + - 示例：`Limit = 4` 表示最多执行4次，无论有多少参数 + +2. **Loops（循环次数）**： + - 作用：设置单个 parameter 的**循环使用次数** + - 位置：`StepConfig.Loops`（通过 `Loop()` 方法设置） + - 示例：`Loops = 2` 表示每个参数都循环执行2次 + +##### 执行顺序优化 + +用户期望的执行顺序（外层循环）： +```go +// 参数：["A", "B"]，Loops = 2 +// 执行顺序：A, B, A, B（先遍历所有参数，再进行下一轮循环） +// 步骤命名：[loop_1_params_1], [loop_1_params_2], [loop_2_params_1], [loop_2_params_2] +``` + +#### 3. 统一执行流程 + +``` +1. 收集所有参数组合 +2. 外层循环：循环次数 (1 到 loopTimes) +3. 内层循环：遍历所有参数 +4. 执行步骤并收集结果 +``` + +### API 设计 + +#### 统一的参数配置方法 + +为了避免代码重复和保持API一致性，参数配置方法只在 `StepRequest` 中定义： + +- `WithParameters()` - 设置步骤级参数 +- `WithParametersSetting()` - 设置参数选择策略（支持 option 模式） +- `Loop()` - 设置循环次数 +- `WithVariables()` - 设置步骤变量 + +这些方法返回 `*StepRequest`，可以与其他步骤类型（如 Mobile、HTTP 等）配合使用。 + +#### Option API（推荐） + +新的 option API 提供更简洁和灵活的参数配置方式： + +```go +// 单个 option +hrp.NewStep("测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京"}, + }). + WithParametersSetting(hrp.WithSequentialOrder()). + Loop(2). + Android() + +// 多个 options 组合 +hrp.NewStep("测试"). + WithParameters(...). + WithParametersSetting( + hrp.WithRandomOrder(), // 设置随机选择顺序 + hrp.WithLimit(4), // 限制总执行次数 + hrp.WithStrategy("param1", hrp.IteratorStrategy{ + Name: "custom", + PickOrder: "random", + }), + ). + Loop(2). + Android() +``` + +##### 可用的 Option 函数 + +###### 参数选择顺序 +- `hrp.WithSequentialOrder()` - 设置参数按顺序执行（默认） +- `hrp.WithRandomOrder()` - 设置参数随机选择 +- `hrp.WithUniqueOrder()` - 设置参数唯一选择，避免重复 + +###### 其他配置 +- `hrp.WithLimit(limit int)` - 设置参数迭代总数限制 +- `hrp.WithStrategy(paramName string, strategy IteratorStrategy)` - 为特定参数设置策略 + +#### 正确的调用顺序 + +**重要**：参数配置必须在指定步骤类型之前调用： + +```go +// ✅ 推荐：使用 Loop 方法 +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + Loop(3). // 使用 Loop 方法 + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ✅ 推荐：使用新的 Option API +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting( + hrp.WithSequentialOrder(), + hrp.WithLimit(3), + ). + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ✅ 兼容：使用传统 TParamsConfig +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting(&hrp.TParamsConfig{ + PickOrder: "sequential", + Limit: 3, + }). + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ❌ 错误：在指定步骤类型后无法配置参数 +hrp.NewStep("搜索测试"). + Android(). + WithParameters(...) // 这里会编译错误 +``` + +### 基本用法 + +#### 1. 简单参数列表 + +```go +hrp.NewStep("搜索查询词"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + Android(). + StartToGoal("进入搜索框，输入 query「$query」，等待加载出 sug 提示词") +``` + +#### 2. 复合参数 + +```go +hrp.NewStep("登录测试"). + WithParameters(map[string]interface{}{ + "username-password": [][]interface{}{ + {"user1", "pass1"}, + {"user2", "pass2"}, + {"user3", "pass3"}, + }, + }). + POST("/api/login"). + WithBody(map[string]interface{}{ + "username": "$username", + "password": "$password", + }) +``` + +#### 3. 参数策略配置 + +```go +hrp.NewStep("随机测试数据"). + WithParameters(map[string]interface{}{ + "data": []interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机选择 + hrp.WithLimit(3), // 只执行3次 + ). + POST("/api/test"). + WithBody(map[string]interface{}{ + "value": "$data", + }) +``` + +#### 4. 循环配置 + +```go +// 推荐方式：使用 Loop 方法 +hrp.NewStep("循环测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京"}, + }). + Loop(3). // 每个参数重复执行3次 + Android(). + StartToGoal("搜索「$query」") + +// 总执行次数：2 × 3 = 6 次 +// 执行顺序：成都, 北京, 成都, 北京, 成都, 北京 +``` + +### 参数策略选项 + +- `sequential`: 顺序执行（默认） +- `random`: 随机选择 +- `unique`: 唯一值，避免重复 + +### 高级特性 + +#### 1. 与现有功能兼容 + +步骤级参数化与现有功能完全兼容： + +- ✅ 支持 Loop 循环 +- ✅ 支持变量提取和校验 +- ✅ 支持 SetupHooks 和 TeardownHooks +- ✅ 支持 failfast 控制 + +#### 2. 步骤命名规则 + +- 无参数，无循环：`步骤名称` +- 无参数，有循环：`步骤名称_loop_N` +- 有参数，无循环：`步骤名称 [params_N]` +- 有参数，有循环：`步骤名称 [loop_M_params_N]`（外层循环优化） + +#### 3. 变量恢复机制 + +每次执行完成后，会自动恢复原始的步骤变量，避免副作用影响后续执行。 + +### 完整示例 + +```go +func TestStepParametersExample(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("步骤级参数化示例"), + TestSteps: []hrp.IStep{ + hrp.NewStep("多查询词搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting( + hrp.WithSequentialOrder(), + hrp.WithLimit(3), + ). + Android(). + StartToGoal("搜索「$query」并等待结果加载"). + AIQuery("提取搜索结果"), + hrp.NewStep("循环执行测试"). + WithParameters(map[string]interface{}{ + "data": []interface{}{1, 2, 3}, + }). + Loop(2). // 每个参数执行2次 + GET("/api/test"). + WithParams(map[string]interface{}{ + "value": "$data", + }), + }, + } + + // 预期执行： + // 第一个步骤：成都, 北京, 重庆 = 3次 + // 第二个步骤：1, 2, 3, 1, 2, 3 = 6次 + + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} +``` + +## 对比与选择 + +| 特性 | 测试用例层级 (Testcase-Level) | 测试步骤层级 (Step-Level) | +| :--- | :--- | :--- | +| **适用场景** | 完整的业务流程验证 | 单个功能点或API的验证 | +| **数据作用域** | 整个 `TestCase` 中的所有 `Step` | 单个 `Step` | +| **报告结果** | 每组参数生成一个独立的测试报告 | 所有参数在同一个测试报告中 | +| **性能** | 开销较大，每次都需执行完整流程 | 开销较小，仅重复执行单个步骤 | +| **配置方法** | `hrp.TestCase{ Parameters: ... }` | `hrp.Step{}.WithParameters(...)` | + +**选择建议**： +- 当您需要用多组数据测试一个完整的端到端流程时（例如，不同的用户配置），请使用**测试用例层级**参数化。 +- 当您只需要在一个流程中，针对某个特定步骤进行多次数据测试时（例如，测试接口对不同输入的响应），请使用**测试步骤层级**参数化。 + +## 变量优先级 + +HttpRunner 中的变量优先级遵循覆盖原则，顺序如下（从高到低）： + +1. **步骤级参数化变量** (`WithParameters`) +2. **用例级参数化变量** (`TestCase.Parameters`) +3. **步骤级变量** (`WithVariables`) +4. **会话变量** (前面步骤提取的变量) +5. **配置变量** (`Config.Variables`) + +## 注意事项 + +1. **API 调用顺序**：步骤级参数化方法必须在步骤类型方法（如 `Android()`、`POST()` 等）之前调用 +2. **参数继承**：`StepMobile` 等步骤类型继承 `StepConfig`，参数会正确传递 +3. **编译时检查**：错误的调用顺序会在编译时报错，确保API使用正确性 +4. **变量覆盖**：Parameters 会覆盖同名的 Variables，但不会影响其他变量 +5. **变量恢复**：每次执行后自动恢复原始变量，避免副作用 +6. **执行顺序**：步骤级参数化采用外层循环优化，先遍历所有参数，再进行下一轮循环 +7. **Option API 推荐**：新项目建议使用 option 模式的 API，更加灵活和简洁 \ No newline at end of file diff --git a/examples/parameters/main_test.go b/examples/parameters/main_test.go new file mode 100644 index 00000000..c18ce7fa --- /dev/null +++ b/examples/parameters/main_test.go @@ -0,0 +1,135 @@ +package parameters + +import ( + "testing" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" + "github.com/stretchr/testify/assert" +) + +// TestParametersExecutionScenarios 涵盖了参数化的核心执行场景， +// 包括纯参数驱动、参数与循环结合，以及使用随机和限制等设置。 +func TestParametersExecutionScenarios(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("测试参数化核心执行场景"). + WithVariables(map[string]interface{}{"loops": 2}), + TestSteps: []hrp.IStep{ + // 场景1: 纯参数驱动 + hrp.NewStep("API请求 - 纯参数"). + WithParameters(map[string]interface{}{ + "arg1": []int{10, 20}, + "arg2": []string{"a", "b"}, + }). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"p1": "$arg1", "p2": "$arg2"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + + // 场景2: 参数与 Loops 结合 + hrp.NewStep("API请求 - 参数与Loops结合"). + WithParameters(map[string]interface{}{ + "word": []string{"hello", "world"}, + }). + Loop(3). // 每个参数执行3次 + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"search": "$word"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + + // 场景3: 参数设置 (随机, 限制) + hrp.NewStep("API请求 - 参数设置"). + WithParameters(map[string]interface{}{ + "city": []string{"chengdu", "beijing", "shanghai", "guangzhou"}, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机顺序 + hrp.WithLimit(2), // 总共执行2次 + ). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"city": "$city"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + }, + } + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} + +// TestParametersVariableOverride 用于验证参数(parameters)如何覆盖测试用例配置(config) +// 和步骤(step)中定义的同名变量(variables)。 +func TestParametersVariableOverride(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("测试参数覆盖变量的优先级"). + WithVariables(map[string]interface{}{ + "p1": "config_level", // 将被步骤级变量覆盖 + "p2": "config_level", // 将被参数覆盖 + }), + TestSteps: []hrp.IStep{ + hrp.NewStep("API请求 - 验证变量覆盖"). + WithVariables(map[string]interface{}{ + "p1": "step_level", // 不会被参数覆盖, 最终值为 "step_level" + "p2": "step_level", // 会被参数覆盖 + "p3": "step_level", // 新增的步骤级变量 + }). + WithParameters(map[string]interface{}{ + "p2-p4": [][]interface{}{ + {"param_level_2", "param_level_4_a"}, + {"param_level_2", "param_level_4_b"}, + }, + }). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{ + "param1": "$p1", // 预期: step_level + "param2": "$p2", // 预期: param_level_2 + "param3": "$p3", // 预期: step_level + "param4": "$p4", // 预期: param_level_4_a/b + }). + Validate(). + AssertEqual("status_code", 200, "check status code"). + AssertEqual("body.args.param1", "step_level", "p1 should be step_level"). + AssertEqual("body.args.param2", "param_level_2", "p2 should be param_level_2"). + AssertEqual("body.args.param3", "step_level", "p3 should be step_level"), + }, + } + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} + +// TestParametersForMobileUI 演示了如何在移动端UI测试中使用参数化来驱动测试。 +func TestParametersForMobileUI(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("小红书UI参数化搜索"). + SetAIOptions(option.WithLLMConfig( + option.NewLLMServiceConfig(option.DOUBAO_1_5_UI_TARS_250328), + )), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动APP"). + Android(). + AppLaunch("com.xingin.xhs"). + Sleep(5). + Validate(). + AssertAppInForeground("com.xingin.xhs"), + hrp.NewStep("UI搜索 - 单参数"). + WithParameters(map[string]interface{}{ + "query": []string{"成都", "北京"}, + }). + Android(). + StartToGoal("进入搜索框，输入「$query」，等待搜索建议出现"). + Sleep(2), + hrp.NewStep("UI搜索 - 复合参数"). + WithParameters(map[string]interface{}{ + "query-category": [][]string{ + {"美食", "食物"}, + {"旅游", "地点"}, + }, + }). + Android(). + StartToGoal("进入搜索框，输入「$query」，并确认其类别为「$category」，等待搜索建议出现"). + Sleep(2), + }, + } + + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} diff --git a/internal/version/VERSION b/internal/version/VERSION index 2f551a4e..d2798a2a 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506231504 +v5.0.0-beta-2506232134 diff --git a/parameters.go b/parameters.go index 1de3d21c..c698dd57 100644 --- a/parameters.go +++ b/parameters.go @@ -11,10 +11,51 @@ import ( "github.com/rs/zerolog/log" ) +// ParametersOption defines options for parameters configuration +type ParametersOption func(*TParamsConfig) + +// WithSequentialOrder sets parameters to be picked in sequential order +func WithSequentialOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("sequential") + } +} + +// WithRandomOrder sets parameters to be picked in random order +func WithRandomOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("random") + } +} + +// WithUniqueOrder sets parameters to be picked uniquely (no duplicates) +func WithUniqueOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("unique") + } +} + +// WithLimit sets the limit count for parameter iteration +func WithLimit(limit int) ParametersOption { + return func(config *TParamsConfig) { + config.Limit = limit + } +} + +// WithStrategy sets individual strategy for a specific parameter +func WithStrategy(paramName string, strategy IteratorStrategy) ParametersOption { + return func(config *TParamsConfig) { + if config.Strategies == nil { + config.Strategies = make(map[string]IteratorStrategy) + } + config.Strategies[paramName] = strategy + } +} + type TParamsConfig struct { PickOrder iteratorPickOrder `json:"pick_order,omitempty" yaml:"pick_order,omitempty"` // overall pick-order strategy Strategies map[string]IteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` // individual strategies for each parameters - Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` // limit count for parameter iteration } type iteratorPickOrder string diff --git a/runner.go b/runner.go index 34d0d5cb..ed13eb05 100644 --- a/runner.go +++ b/runner.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "reflect" + "sort" "strings" "syscall" "testing" @@ -734,6 +735,98 @@ const ( RUN_STEP_END = "run step end" ) +// executionTask holds the necessary information for a single step execution. +type executionTask struct { + stepName string + parameters map[string]interface{} +} + +// formatParameters formats parameter values into a string for display in step names. +// e.g. {"foo": "bar", "age": 18} -> "bar-18" +func formatParameters(params map[string]interface{}) string { + if len(params) == 0 { + return "" + } + + // sort keys to ensure consistent order + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + var values []string + for _, k := range keys { + values = append(values, fmt.Sprintf("%v", params[k])) + } + return strings.Join(values, "-") +} + +// generateExecutionTasks generates a list of execution tasks based on step parameters and loops. +func (r *SessionRunner) generateExecutionTasks(step IStep) ([]executionTask, error) { + stepConfig := step.Config() + stepName := step.Name() + + // determine effective loop times + loopTimes := stepConfig.Loops + if loopTimes <= 0 { + loopTimes = 1 // default to 1 if not set + } + + // initialize parameters iterator + parametersIterator, err := r.caseRunner.parser.InitParametersIterator(&TConfig{ + Parameters: stepConfig.Parameters, + ParametersSetting: stepConfig.ParametersSetting, + Variables: stepConfig.Variables, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize parameters iterator") + } + + // collect all parameter combinations first + var allParameters []map[string]interface{} + if parametersIterator != nil { + for parametersIterator.HasNext() { + allParameters = append(allParameters, parametersIterator.Next()) + } + } + + // if no parameters are specified, but loop times are set, + // we should run the step loopTimes with empty parameters. + if len(allParameters) == 0 && loopTimes > 0 { + allParameters = append(allParameters, make(map[string]interface{})) + } + + // generate execution tasks + var tasks []executionTask + for loopIndex := 1; loopIndex <= loopTimes; loopIndex++ { + for _, params := range allParameters { + // determine step name based on parameters and loops + currentStepName := stepName + hasParameters := len(params) > 0 + hasLoops := loopTimes > 1 + + if hasParameters { + paramStr := formatParameters(params) + if hasLoops { + currentStepName = fmt.Sprintf("%s [loop_%d_params_%s]", stepName, loopIndex, paramStr) + } else { + currentStepName = fmt.Sprintf("%s [params_%s]", stepName, paramStr) + } + } else if hasLoops { + currentStepName = fmt.Sprintf("%s_loop_%d", stepName, loopIndex) + } + + tasks = append(tasks, executionTask{ + stepName: currentStepName, + parameters: params, + }) + } + } + + return tasks, nil +} + func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) { // check for interrupt signal before running step select { @@ -753,56 +846,90 @@ func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) stepName := step.Name() stepType := string(step.Type()) + log.Info().Str("step", stepName).Str("type", stepType).Msg(RUN_STEP_START) - // run times of step - loopTimes := step.Config().Loops - if loopTimes == 0 { - loopTimes = 1 // default run once - } else if loopTimes > 1 { - log.Info().Int("loops", loopTimes).Msg("set multiple loop times") + // execute step with parameters iterator + tasks, err := r.generateExecutionTasks(step) + if err != nil { + return nil, errors.Wrap(err, "failed to generate execution tasks") } - // run step with specified loop times - for i := 1; i <= loopTimes; i++ { - var loopIndex string - if loopTimes > 1 { - log.Info().Int("index", i).Msg("start running step in loop") - loopIndex = fmt.Sprintf("_loop_%d", i) + var stepResults []*StepResult + + // execute with loops as outer iteration + for _, task := range tasks { + // execute step with merged variables + stepResult, err := r.executeStepWithVariables(step, task.stepName, task.parameters) + if err != nil { + if r.caseRunner.hrpRunner.failfast { + return nil, errors.Wrap(err, "execute step failed") + } + log.Error().Err(err).Str("step", task.stepName).Msg("execute step failed") } - // run step - stepResult, err = step.Run(r) - stepResult.Name = stepName + loopIndex + stepResults = append(stepResults, stepResult) + } - // add step result to summary - r.summary.AddStepResult(stepResult) - - // update extracted variables - for k, v := range stepResult.ExportVars { - r.sessionVariables[k] = v + // return the last step result, or nil if no steps were executed + if len(stepResults) > 0 { + // add all step results to summary + for _, result := range stepResults { + r.summary.AddStepResult(result) + // update extracted variables from the last result + for k, v := range result.ExportVars { + r.sessionVariables[k] = v + } } - // run step success - if err == nil { + // log final result + lastResult := stepResults[len(stepResults)-1] + if lastResult.Success { log.Info().Str("step", stepName). Str("type", stepType). Bool("success", true). - Int64("elapsed(ms)", stepResult.Elapsed). - Interface("exportVars", stepResult.ExportVars). + Int64("elapsed(ms)", lastResult.Elapsed). + Interface("exportVars", lastResult.ExportVars). + Msg(RUN_STEP_END) + } else { + log.Error().Str("step", stepName). + Str("type", stepType). + Bool("success", false). + Int64("elapsed(ms)", lastResult.Elapsed). Msg(RUN_STEP_END) - continue } - // run step failed - log.Error().Err(err).Str("step", stepName). - Str("type", stepType). - Bool("success", false). - Int64("elapsed(ms)", stepResult.Elapsed). - Msg(RUN_STEP_END) - return stepResult, err + + return lastResult, nil } - return stepResult, nil + return nil, errors.New("no steps were executed") +} + +// executeStepWithVariables executes a single step with given parameters +// parameters will override step variables with the same name +func (r *SessionRunner) executeStepWithVariables(step IStep, stepName string, parameters map[string]interface{}) (stepResult *StepResult, err error) { + stepConfig := step.Config() + + // backup original variables + originalVariables := make(map[string]interface{}) + for k, v := range stepConfig.Variables { + originalVariables[k] = v + } + + // merge parameters into step variables + // parameters have higher priority than variables + for k, v := range parameters { + stepConfig.Variables[k] = v + } + + // execute step + stepResult, err = step.Run(r) + stepResult.Name = stepName + + // restore original variables to avoid side effects + stepConfig.Variables = originalVariables + + return stepResult, err } func (r *SessionRunner) GetSummary() *TestCaseSummary { diff --git a/step.go b/step.go index 3b2c866f..53c6bd0b 100644 --- a/step.go +++ b/step.go @@ -27,15 +27,17 @@ const ( ) type StepConfig struct { - StepName string `json:"name" yaml:"name"` // required - Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` - SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` - TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` - Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` - Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` - StepExport []string `json:"export,omitempty" yaml:"export,omitempty"` - Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` - AutoPopupHandler bool `json:"auto_popup_handler,omitempty" yaml:"auto_popup_handler,omitempty"` // enable auto popup handler for this step + StepName string `json:"name" yaml:"name"` // required + Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` + Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` + ParametersSetting *TParamsConfig `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` + SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` + TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` + Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` + Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` + StepExport []string `json:"export,omitempty" yaml:"export,omitempty"` + Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` + AutoPopupHandler bool `json:"auto_popup_handler,omitempty" yaml:"auto_popup_handler,omitempty"` // enable auto popup handler for this step } // define struct for teststep diff --git a/step_request.go b/step_request.go index 25b88666..8f9bc23f 100644 --- a/step_request.go +++ b/step_request.go @@ -543,6 +543,26 @@ func (s *StepRequest) WithVariables(variables map[string]interface{}) *StepReque return s } +// WithParameters sets parameters for step-level data driven +func (s *StepRequest) WithParameters(parameters map[string]interface{}) *StepRequest { + s.Parameters = parameters + return s +} + +// WithParametersSetting sets parameters setting for step-level data driven +func (s *StepRequest) WithParametersSetting(options ...ParametersOption) *StepRequest { + if s.ParametersSetting == nil { + s.ParametersSetting = &TParamsConfig{} + } + + // apply all options + for _, option := range options { + option(s.ParametersSetting) + } + + return s +} + // SetupHook adds a setup hook for current teststep. func (s *StepRequest) SetupHook(hook string) *StepRequest { s.SetupHooks = append(s.SetupHooks, hook) @@ -557,7 +577,7 @@ func (s *StepRequest) HTTP2() *StepRequest { return s } -// Loop specify running times for the current step +// Loop sets loop count for step execution. func (s *StepRequest) Loop(times int) *StepRequest { s.Loops = times return s From e6ce61368ebc3cdfd612161fc1e7f08dab8f32c2 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 23 Jun 2025 21:42:09 +0800 Subject: [PATCH 03/16] fix: enhance step log retrieval with prefix matching for parameterized steps --- internal/version/VERSION | 2 +- report.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index d2798a2a..356259f9 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506232134 +v5.0.0-beta-2506232142 diff --git a/report.go b/report.go index eb27c714..4d411386 100644 --- a/report.go +++ b/report.go @@ -221,8 +221,9 @@ func (g *HTMLReportGenerator) getStepLogs(stepName string, startTime int64, elap for _, logEntry := range g.LogData { // Check for step boundaries to control inclusion if logEntry.Message == RUN_STEP_START { - if stepFieldValue, exists := logEntry.Fields["step"]; exists { - if stepFieldValue == stepName { + if stepFieldValue, exists := logEntry.Fields["step"].(string); exists { + // use prefix matching for parameterized steps + if strings.HasPrefix(stepName, stepFieldValue) { inCurrentStep = true stepLogs = append(stepLogs, logEntry) continue @@ -234,8 +235,9 @@ func (g *HTMLReportGenerator) getStepLogs(stepName string, startTime int64, elap } if logEntry.Message == RUN_STEP_END { - if stepFieldValue, exists := logEntry.Fields["step"]; exists { - if stepFieldValue == stepName { + if stepFieldValue, exists := logEntry.Fields["step"].(string); exists { + // use prefix matching for parameterized steps + if strings.HasPrefix(stepName, stepFieldValue) { stepLogs = append(stepLogs, logEntry) inCurrentStep = false continue From b1b4e5c1dcab482d70794b7c84b1853857a23f11 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 23 Jun 2025 22:32:30 +0800 Subject: [PATCH 04/16] feat: add toggle functionality for test summary steps with UI enhancements --- internal/version/VERSION | 2 +- report.go | 66 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 356259f9..79feea89 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506232142 +v5.0.0-beta-2506232232 diff --git a/report.go b/report.go index 4d411386..e18a1b6f 100644 --- a/report.go +++ b/report.go @@ -657,9 +657,9 @@ const htmlTemplate = ` .summary h2 { color: #2c3e50; - margin-bottom: 20px; - border-bottom: 2px solid #3498db; - padding-bottom: 10px; + margin: 0; + padding: 0; + border: none; } .summary-grid { @@ -669,6 +669,32 @@ const htmlTemplate = ` margin-bottom: 20px; } + .summary-title-bar { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 20px; + border-bottom: 2px solid #3498db; + padding-bottom: 10px; + } + + .toggle-all-btn { + background-color: #ffc107; + color: #212529; + border: none; + padding: 8px 16px; + border-radius: 5px; + cursor: pointer; + font-size: 0.9em; + font-weight: 500; + transition: background-color 0.2s ease; + flex-shrink: 0; + } + + .toggle-all-btn:hover { + background-color: #e0a800; + } + .summary-item { text-align: center; padding: 15px; @@ -2120,7 +2146,10 @@ const htmlTemplate = `
-

📊 Test Summary

+
+

📊 Test Summary

+ +
{{.Stat.TestCases.Success}}
@@ -2818,6 +2847,35 @@ const htmlTemplate = ` contents.forEach(content => content.classList.add('show')); icons.forEach(icon => icon.classList.add('rotated')); }); + + function toggleAllSteps() { + const contents = document.querySelectorAll('.step-content'); + const icons = document.querySelectorAll('.toggle-icon'); + const btn = document.getElementById('toggle-all-steps-btn'); + + if (!contents || contents.length === 0) { + return; + } + + // If any step is expanded, collapse all. Otherwise expand all. + let isAnyExpanded = false; + for (let i = 0; i < contents.length; i++) { + if (contents[i].classList.contains('show')) { + isAnyExpanded = true; + break; + } + } + + if (isAnyExpanded) { + contents.forEach(content => content.classList.remove('show')); + icons.forEach(icon => icon.classList.remove('rotated')); + btn.textContent = 'Expand All Steps'; + } else { + contents.forEach(content => content.classList.add('show')); + icons.forEach(icon => icon.classList.add('rotated')); + btn.textContent = 'Collapse All Steps'; + } + } ` From fc32b5d8742c7558f6a4f59a66277117a6e9a34c Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 11:50:37 +0800 Subject: [PATCH 05/16] feat: enhance AI query handling with detailed result structure and improved UI display --- internal/version/VERSION | 2 +- report.go | 74 +++++++++++++++++----------------------- step.go | 11 +++--- step_ui.go | 18 ++++++++++ uixt/driver_ext_ai.go | 37 ++++++++++++++++++-- uixt/mcp_tools_ai.go | 4 +-- 6 files changed, 93 insertions(+), 53 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 79feea89..50b56ec5 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506232232 +v5.0.0-beta-2506241150 diff --git a/report.go b/report.go index e18a1b6f..16beeb20 100644 --- a/report.go +++ b/report.go @@ -2379,59 +2379,34 @@ const htmlTemplate = ` {{end}} {{end}} - {{/* Handle special case: ai_query needs enhanced display even when not in planning */}} + {{/* Enhanced AI Query Display - using QueryResult data structure */}} {{if eq $action.Method "ai_query"}} + {{if $action.QueryResult}}
-
- - {{$stepLogs := getStepLogs $step}} - {{$queryThought := ""}} - {{$queryModel := ""}} - {{$queryUsage := ""}} - {{$queryScreenshot := ""}} - {{$queryResult := ""}} - {{range $logEntry := $stepLogs}} - {{if and (eq $logEntry.Message "log response message") (index $logEntry.Fields "content")}} - {{$content := index $logEntry.Fields "content"}} - {{if $content}} - {{$queryResult = $content}} - {{end}} - {{end}} - {{if and (eq $logEntry.Message "call model service for query") (index $logEntry.Fields "model")}} - {{$queryModel = index $logEntry.Fields "model"}} - {{end}} - {{if and (eq $logEntry.Message "usage statistics") (index $logEntry.Fields "input_tokens")}} - {{$inputTokens := index $logEntry.Fields "input_tokens"}} - {{$outputTokens := index $logEntry.Fields "output_tokens"}} - {{$totalTokens := index $logEntry.Fields "total_tokens"}} - {{$queryUsage = printf "📊 Tokens: %v in / %v out / %v total" $inputTokens $outputTokens $totalTokens}} - {{end}} - {{if and (eq $logEntry.Message "log screenshot") (index $logEntry.Fields "imagePath")}} - {{$queryScreenshot = index $logEntry.Fields "imagePath"}} - {{end}} + + {{if $action.QueryResult.Thought}} +
{{$action.QueryResult.Thought}}
{{end}} - - {{if $queryResult}} -
{{$queryResult}}
- {{end}} - - +
- {{if $queryScreenshot}} + {{if $action.QueryResult.ImagePath}}
📸 Query Screenshot + {{if $action.QueryResult.ScreenshotElapsed}} + {{formatDuration $action.QueryResult.ScreenshotElapsed}} + {{end}}
- {{$base64Image := encodeImageBase64 $queryScreenshot}} + {{$base64Image := encodeImageBase64 $action.QueryResult.ImagePath}} {{if $base64Image}}
- Query Screenshot + AI Query Screenshot
{{end}} @@ -2440,18 +2415,30 @@ const htmlTemplate = `
{{end}} - +
- 🤖 AI Query + 🤖 AI Query Analysis + {{if $action.QueryResult.ModelCallElapsed}} + {{formatDuration $action.QueryResult.ModelCallElapsed}} + {{end}}
- {{if $queryModel}} -
🤖 Model: {{$queryModel}}
+ {{if $action.QueryResult.ModelName}} +
🤖 Model: {{$action.QueryResult.ModelName}}
{{end}} - {{if $queryUsage}} -
{{$queryUsage}}
+ {{if $action.QueryResult.Resolution}} +
📐 Resolution: {{$action.QueryResult.Resolution.Width}}x{{$action.QueryResult.Resolution.Height}}
+ {{end}} + {{if $action.QueryResult.Usage}} +
📊 Tokens: {{$action.QueryResult.Usage.PromptTokens}} in / {{$action.QueryResult.Usage.CompletionTokens}} out / {{$action.QueryResult.Usage.TotalTokens}} total
+ {{end}} + {{if $action.QueryResult.Content}} +
💬 Query Result: {{$action.QueryResult.Content}}
+ {{end}} + {{if $action.QueryResult.Error}} +
❌ Error: {{$action.QueryResult.Error}}
{{end}}
@@ -2460,6 +2447,7 @@ const htmlTemplate = `
{{end}} + {{end}} {{/* Handle SessionData: display requests and screen results for non-planning actions */}} {{if not $action.Plannings}} diff --git a/step.go b/step.go index 53c6bd0b..f15bf745 100644 --- a/step.go +++ b/step.go @@ -60,11 +60,12 @@ type TStep struct { // one step contains one or multiple actions type ActionResult struct { option.MobileAction `json:",inline"` - StartTime int64 `json:"start_time"` // action start time in millisecond(ms) - Elapsed int64 `json:"elapsed_ms"` // action elapsed time(ms) - Error string `json:"error,omitempty"` // action execution result - Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions - uixt.SessionData // store session data for other actions besides start_to_goal + StartTime int64 `json:"start_time"` // action start time in millisecond(ms) + Elapsed int64 `json:"elapsed_ms"` // action elapsed time(ms) + Error string `json:"error,omitempty"` // action execution result + Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions + QueryResult *uixt.QueryExecutionResult `json:"query_result,omitempty"` // store query result for ai_query actions + uixt.SessionData // store session data for other actions besides start_to_goal & ai_query } // one testcase contains one or multiple steps diff --git a/step_ui.go b/step_ui.go index 426621cb..2c31c77f 100644 --- a/step_ui.go +++ b/step_ui.go @@ -943,6 +943,24 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err actionResult.Plannings = planningResults stepResult.Actions = append(stepResult.Actions, actionResult) if err != nil { + actionResult.Error = err.Error() + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.MobileUIDriverError, err.Error()) + } + return stepResult, err + } + continue + } + + // handle ai_query action + if action.Method == option.ACTION_Query { + queryResult, err := uiDriver.AIQuery( + action.Params.(string), action.GetOptions()...) + actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() + actionResult.QueryResult = queryResult + stepResult.Actions = append(stepResult.Actions, actionResult) + if err != nil { + actionResult.Error = err.Error() if !code.IsErrorPredefined(err) { err = errors.Wrap(code.MobileUIDriverError, err.Error()) } diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 773e4c1c..c0ec94a0 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -301,6 +301,17 @@ type PlanningExecutionResult struct { SubActions []*SubActionResult `json:"sub_actions,omitempty"` // sub-actions generated from this planning } +// QueryExecutionResult contains the result of AI query execution with timing and metadata +type QueryExecutionResult struct { + ai.QueryResult // inherit from ai.QueryResult + ModelCallElapsed int64 `json:"model_call_elapsed"` // model call elapsed time in milliseconds + ScreenshotElapsed int64 `json:"screenshot_elapsed"` // screenshot elapsed time in milliseconds + ImagePath string `json:"image_path"` // path to screenshot used for query + Resolution *types.Size `json:"resolution"` // screen resolution + ModelName string `json:"model_name"` // model name used for query + Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics +} + // SubActionResult represents a sub-action within a start_to_goal action type SubActionResult struct { ActionName string `json:"action_name"` // name of the sub-action (e.g., "tap", "input") @@ -316,11 +327,21 @@ type SessionData struct { ScreenResults []*ScreenResult `json:"screen_results,omitempty"` // store sub-action specific screen_results } -func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.QueryResult, error) { +func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*QueryExecutionResult, error) { if dExt.LLMService == nil { return nil, errors.New("LLM service is not initialized") } + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() + if err != nil { + return nil, err + } + screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize() if err != nil { return nil, err @@ -329,6 +350,9 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.Que // parse action options to extract OutputSchema actionOptions := option.NewActionOptions(opts...) + // Step 2: Call model and measure time + modelCallStartTime := time.Now() + // execute query queryOpts := &ai.QueryOptions{ Query: text, @@ -337,11 +361,20 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.Que OutputSchema: actionOptions.OutputSchema, } result, err := dExt.LLMService.Query(context.Background(), queryOpts) + modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() if err != nil { return nil, errors.Wrap(err, "AI query failed") } - return result, nil + // Create QueryExecutionResult with all timing and metadata + queryExecResult := &QueryExecutionResult{ + QueryResult: *result, // inherit from ai.QueryResult + ModelCallElapsed: modelCallElapsed, // model call timing + ScreenshotElapsed: screenshotElapsed, // screenshot timing + ImagePath: screenResult.ImagePath, // screenshot path + Resolution: &screenResult.Resolution, // screen resolution + } + return queryExecResult, nil } func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) error { diff --git a/uixt/mcp_tools_ai.go b/uixt/mcp_tools_ai.go index bc9a42c0..791470d9 100644 --- a/uixt/mcp_tools_ai.go +++ b/uixt/mcp_tools_ai.go @@ -166,7 +166,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc { opts := unifiedReq.Options() // AI query logic with options - result, err := driverExt.AIQuery(unifiedReq.Prompt, opts...) + queryResult, err := driverExt.AIQuery(unifiedReq.Prompt, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("AI query failed: %s", err.Error())), nil } @@ -174,7 +174,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc { message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt) returnData := ToolAIQuery{ Prompt: unifiedReq.Prompt, - Result: result.Content, + Result: queryResult.Content, } return NewMCPSuccessResponse(message, &returnData), nil From 8fc8d066044c02aa5c014b1ffa92a3d728df55e6 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 13:22:00 +0800 Subject: [PATCH 06/16] feat: unify AI action handling with detailed execution results and enhanced UI integration --- internal/version/VERSION | 2 +- report.go | 135 +++++++++---------------------------- step.go | 10 +-- step_ui.go | 25 +++++-- uixt/driver_ext_ai.go | 122 ++++++++++++++++++++++++++------- uixt/driver_ext_ai_test.go | 4 +- uixt/driver_utils.go | 2 +- uixt/mcp_tools_ai.go | 2 +- 8 files changed, 156 insertions(+), 146 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 50b56ec5..f9a3ba97 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241150 +v5.0.0-beta-2506241342 diff --git a/report.go b/report.go index 16beeb20..61ffb0b0 100644 --- a/report.go +++ b/report.go @@ -491,6 +491,12 @@ func (g *HTMLReportGenerator) GenerateReport(outputFile string) error { "add": func(a, b int) int { return a + b }, "base": filepath.Base, "index": func(m map[string]any, key string) any { return m[key] }, + "title": func(s string) string { + if s == "" { + return "" + } + return strings.ToUpper(s[:1]) + s[1:] + }, "extractThought": func(content string) string { if content == "" { return "" @@ -2379,34 +2385,34 @@ const htmlTemplate = ` {{end}} {{end}} - {{/* Enhanced AI Query Display - using QueryResult data structure */}} - {{if eq $action.Method "ai_query"}} - {{if $action.QueryResult}} + {{/* Enhanced AI Operations Display - using unified AIResult data structure */}} + {{if or (eq $action.Method "ai_query") (eq $action.Method "ai_action") (eq $action.Method "ai_assert")}} + {{if $action.AIResult}}
- {{if $action.QueryResult.Thought}} -
{{$action.QueryResult.Thought}}
+ {{if $action.AIResult.Thought}} +
{{$action.AIResult.Thought}}
{{end}} - +
- {{if $action.QueryResult.ImagePath}} + {{if $action.AIResult.ImagePath}}
- 📸 Query Screenshot - {{if $action.QueryResult.ScreenshotElapsed}} - {{formatDuration $action.QueryResult.ScreenshotElapsed}} + 📸 {{title $action.AIResult.Type}} Screenshot + {{if $action.AIResult.ScreenshotElapsed}} + {{formatDuration $action.AIResult.ScreenshotElapsed}} {{end}}
- {{$base64Image := encodeImageBase64 $action.QueryResult.ImagePath}} + {{$base64Image := encodeImageBase64 $action.AIResult.ImagePath}} {{if $base64Image}}
- AI Query Screenshot + AI {{title $action.AIResult.Type}} Screenshot
{{end}} @@ -2415,30 +2421,27 @@ const htmlTemplate = `
{{end}} - +
- 🤖 AI Query Analysis - {{if $action.QueryResult.ModelCallElapsed}} - {{formatDuration $action.QueryResult.ModelCallElapsed}} + 🤖 AI {{title $action.AIResult.Type}} Analysis + {{if $action.AIResult.ModelCallElapsed}} + {{formatDuration $action.AIResult.ModelCallElapsed}} {{end}}
- {{if $action.QueryResult.ModelName}} -
🤖 Model: {{$action.QueryResult.ModelName}}
+ {{if $action.AIResult.ModelName}} +
🤖 Model: {{$action.AIResult.ModelName}}
{{end}} - {{if $action.QueryResult.Resolution}} -
📐 Resolution: {{$action.QueryResult.Resolution.Width}}x{{$action.QueryResult.Resolution.Height}}
+ {{if $action.AIResult.Resolution}} +
📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}
{{end}} - {{if $action.QueryResult.Usage}} -
📊 Tokens: {{$action.QueryResult.Usage.PromptTokens}} in / {{$action.QueryResult.Usage.CompletionTokens}} out / {{$action.QueryResult.Usage.TotalTokens}} total
+ {{if $action.AIResult.Usage}} +
📊 Tokens: {{$action.AIResult.Usage.PromptTokens}} in / {{$action.AIResult.Usage.CompletionTokens}} out / {{$action.AIResult.Usage.TotalTokens}} total
{{end}} - {{if $action.QueryResult.Content}} -
💬 Query Result: {{$action.QueryResult.Content}}
- {{end}} - {{if $action.QueryResult.Error}} -
❌ Error: {{$action.QueryResult.Error}}
+ {{if $action.AIResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.Content}}
{{end}}
@@ -2531,84 +2534,6 @@ const htmlTemplate = ` {{if and $validator.msg (ne $validator.check_result "pass")}}
{{$validator.msg}}
{{end}} - - - {{if or (eq $validator.check "ui_ai") (eq $validator.assert "ai_assert")}} -
- - {{$stepLogs := getStepLogs $step}} - {{$validationThought := ""}} - {{$validationModel := ""}} - {{$validationUsage := ""}} - {{$validationScreenshot := ""}} - {{range $logEntry := $stepLogs}} - {{if and (eq $logEntry.Message "log response message") (index $logEntry.Fields "content")}} - {{$content := index $logEntry.Fields "content"}} - {{if $content}} - {{$validationThought = $content}} - {{end}} - {{end}} - {{if and (eq $logEntry.Message "call model service for assertion") (index $logEntry.Fields "model")}} - {{$validationModel = index $logEntry.Fields "model"}} - {{end}} - {{if and (eq $logEntry.Message "usage statistics") (index $logEntry.Fields "input_tokens")}} - {{$inputTokens := index $logEntry.Fields "input_tokens"}} - {{$outputTokens := index $logEntry.Fields "output_tokens"}} - {{$totalTokens := index $logEntry.Fields "total_tokens"}} - {{$validationUsage = printf "📊 Tokens: %v in / %v out / %v total" $inputTokens $outputTokens $totalTokens}} - {{end}} - {{if and (eq $logEntry.Message "log screenshot") (index $logEntry.Fields "imagePath")}} - {{$validationScreenshot = index $logEntry.Fields "imagePath"}} - {{end}} - {{end}} - - - {{if $validationThought}} -
{{extractThought $validationThought}}
- {{end}} - - -
- - {{if $validationScreenshot}} -
-
-
- 📸 Validation Screenshot -
-
- {{$base64Image := encodeImageBase64 $validationScreenshot}} - {{if $base64Image}} -
-
- Validation Screenshot -
-
- {{end}} -
-
-
- {{end}} - - -
-
-
- 🤖 AI Analysis -
-
- {{if $validationModel}} -
🤖 Model: {{$validationModel}}
- {{end}} - {{if $validationUsage}} -
{{$validationUsage}}
- {{end}} -
-
-
-
-
- {{end}}
{{end}}
diff --git a/step.go b/step.go index f15bf745..266a937a 100644 --- a/step.go +++ b/step.go @@ -60,11 +60,11 @@ type TStep struct { // one step contains one or multiple actions type ActionResult struct { option.MobileAction `json:",inline"` - StartTime int64 `json:"start_time"` // action start time in millisecond(ms) - Elapsed int64 `json:"elapsed_ms"` // action elapsed time(ms) - Error string `json:"error,omitempty"` // action execution result - Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions - QueryResult *uixt.QueryExecutionResult `json:"query_result,omitempty"` // store query result for ai_query actions + StartTime int64 `json:"start_time"` // action start time in millisecond(ms) + Elapsed int64 `json:"elapsed_ms"` // action elapsed time(ms) + Error string `json:"error,omitempty"` // action execution result + Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions + AIResult *uixt.AIExecutionResult `json:"ai_result,omitempty"` // store unified AI execution result for ai_query/ai_action/ai_assert actions uixt.SessionData // store session data for other actions besides start_to_goal & ai_query } diff --git a/step_ui.go b/step_ui.go index 2c31c77f..9971ca08 100644 --- a/step_ui.go +++ b/step_ui.go @@ -935,7 +935,7 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err } }() - // handle start_to_goal action + // handle start_to_goal AI action if action.Method == option.ACTION_StartToGoal { planningResults, err := uiDriver.StartToGoal(ctx, action.Params.(string), action.GetOptions()...) @@ -952,12 +952,23 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err continue } - // handle ai_query action - if action.Method == option.ACTION_Query { - queryResult, err := uiDriver.AIQuery( - action.Params.(string), action.GetOptions()...) + // handle AI operations (ai_action, ai_query, ai_assert) with unified result storage + if action.Method == option.ACTION_AIAction || action.Method == option.ACTION_Query || action.Method == option.ACTION_AIAssert { + var aiResult *uixt.AIExecutionResult + var err error + + prompt := action.Params.(string) + switch action.Method { + case option.ACTION_AIAction: + aiResult, err = uiDriver.AIAction(ctx, prompt, action.GetOptions()...) + case option.ACTION_Query: + aiResult, err = uiDriver.AIQuery(prompt, action.GetOptions()...) + case option.ACTION_AIAssert: + aiResult, err = uiDriver.AIAssert(prompt, action.GetOptions()...) + } + actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() - actionResult.QueryResult = queryResult + actionResult.AIResult = aiResult stepResult.Actions = append(stepResult.Actions, actionResult) if err != nil { actionResult.Error = err.Error() @@ -969,7 +980,7 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err continue } - // handle other actions + // handle other non-AI actions sessionData, err := uiDriver.ExecuteAction(ctx, action) actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() actionResult.SessionData = sessionData diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index c0ec94a0..f7f21009 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -131,24 +131,52 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op } } -func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) error { +// AIAction performs AI-driven action and returns detailed execution result +func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) (*AIExecutionResult, error) { log.Info().Str("prompt", prompt).Msg("performing AI action") - // plan next action - planningResult, err := dExt.PlanNextAction(ctx, prompt, opts...) + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() if err != nil { - return err + return nil, err } - // Invoke tool calls + // Step 2: Plan next action and measure time + modelCallStartTime := time.Now() + planningResult, err := dExt.PlanNextAction(ctx, prompt, opts...) + modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() + aiExecutionResult := &AIExecutionResult{ + Type: "action", + ModelCallElapsed: modelCallElapsed, + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, + ModelName: planningResult.ModelName, + Usage: planningResult.Usage, + PlanningResult: &planningResult.PlanningResult, + Thought: planningResult.Thought, + Content: planningResult.Content, + } + + if err != nil { + aiExecutionResult.Error = err.Error() + return aiExecutionResult, errors.Wrap(err, "get next action failed") + } + + // Step 3: Execute tool calls for _, toolCall := range planningResult.ToolCalls { err = dExt.invokeToolCall(ctx, toolCall) if err != nil { - return err + aiExecutionResult.Error = err.Error() + return aiExecutionResult, errors.Wrap(err, "invoke tool call failed") } } - return nil + return aiExecutionResult, nil } // PlanNextAction performs planning and returns unified planning information @@ -301,15 +329,25 @@ type PlanningExecutionResult struct { SubActions []*SubActionResult `json:"sub_actions,omitempty"` // sub-actions generated from this planning } -// QueryExecutionResult contains the result of AI query execution with timing and metadata -type QueryExecutionResult struct { - ai.QueryResult // inherit from ai.QueryResult +// AIExecutionResult represents a unified result structure for all AI operations +type AIExecutionResult struct { + Type string `json:"type"` // operation type: "query", "action", "assert" ModelCallElapsed int64 `json:"model_call_elapsed"` // model call elapsed time in milliseconds ScreenshotElapsed int64 `json:"screenshot_elapsed"` // screenshot elapsed time in milliseconds - ImagePath string `json:"image_path"` // path to screenshot used for query + ImagePath string `json:"image_path"` // path to screenshot used for operation Resolution *types.Size `json:"resolution"` // screen resolution - ModelName string `json:"model_name"` // model name used for query + ModelName string `json:"model_name"` // model name used for operation Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics + + // Operation-specific results (only one will be populated based on Type) + QueryResult *ai.QueryResult `json:"query_result,omitempty"` // for ai_query operations + PlanningResult *ai.PlanningResult `json:"planning_result,omitempty"` // for ai_action operations + AssertionResult *ai.AssertionResult `json:"assertion_result,omitempty"` // for ai_assert operations + + // Common fields + Thought string `json:"thought,omitempty"` // AI reasoning/thought process + Content string `json:"content,omitempty"` // operation result content + Error string `json:"error,omitempty"` // error message if operation failed } // SubActionResult represents a sub-action within a start_to_goal action @@ -327,7 +365,7 @@ type SessionData struct { ScreenResults []*ScreenResult `json:"screen_results,omitempty"` // store sub-action specific screen_results } -func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*QueryExecutionResult, error) { +func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*AIExecutionResult, error) { if dExt.LLMService == nil { return nil, errors.New("LLM service is not initialized") } @@ -366,41 +404,77 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*QueryE return nil, errors.Wrap(err, "AI query failed") } - // Create QueryExecutionResult with all timing and metadata - queryExecResult := &QueryExecutionResult{ - QueryResult: *result, // inherit from ai.QueryResult + // Create AIExecutionResult with all timing and metadata + aiResult := &AIExecutionResult{ + Type: "query", ModelCallElapsed: modelCallElapsed, // model call timing ScreenshotElapsed: screenshotElapsed, // screenshot timing ImagePath: screenResult.ImagePath, // screenshot path Resolution: &screenResult.Resolution, // screen resolution + QueryResult: result, // query-specific result + Thought: result.Thought, // AI reasoning + Content: result.Content, // query result content } - return queryExecResult, nil + return aiResult, nil } -func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) error { +// AIAssert performs AI-driven assertion and returns detailed execution result +func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*AIExecutionResult, error) { if dExt.LLMService == nil { - return errors.New("LLM service is not initialized") + return nil, errors.New("LLM service is not initialized") + } + + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() + if err != nil { + return nil, err } screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize() if err != nil { - return err + return &AIExecutionResult{ + Type: "assert", + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, + Error: err.Error(), + }, err } - // execute assertion + // Step 2: Call model and measure time + modelCallStartTime := time.Now() assertOpts := &ai.AssertOptions{ Assertion: assertion, Screenshot: screenShotBase64, Size: size, } result, err := dExt.LLMService.Assert(context.Background(), assertOpts) + modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() + + aiResult := &AIExecutionResult{ + Type: "assert", + ModelCallElapsed: modelCallElapsed, + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, + AssertionResult: result, + Thought: result.Thought, + } + if err != nil { - return errors.Wrap(err, "AI assertion failed") + aiResult.Error = err.Error() + return aiResult, errors.Wrap(err, "AI assertion failed") } if !result.Pass { - return errors.New(result.Thought) + aiResult.Error = result.Thought + return aiResult, errors.New(result.Thought) } - return nil + aiResult.Content = "Assertion passed" + return aiResult, nil } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 5db77431..3167c6cf 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -15,10 +15,10 @@ import ( func TestDriverExt_TapByLLM(t *testing.T) { driver := setupDriverExt(t) - err := driver.AIAction(context.Background(), "点击第一个帖子的作者头像") + _, err := driver.AIAction(context.Background(), "点击第一个帖子的作者头像") assert.Nil(t, err) - err = driver.AIAssert("当前在个人介绍页") + _, err = driver.AIAssert("当前在个人介绍页") assert.Nil(t, err) } diff --git a/uixt/driver_utils.go b/uixt/driver_utils.go index 787f6f9b..c24abcc6 100644 --- a/uixt/driver_utils.go +++ b/uixt/driver_utils.go @@ -195,7 +195,7 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st case option.SelectorOCR: err = dExt.assertOCR(expected, assert) case option.SelectorAI: - err = dExt.AIAssert(expected) + _, err = dExt.AIAssert(expected) case option.SelectorForegroundApp: err = dExt.assertForegroundApp(expected, assert) case option.SelectorSelector: diff --git a/uixt/mcp_tools_ai.go b/uixt/mcp_tools_ai.go index 791470d9..5fa87c05 100644 --- a/uixt/mcp_tools_ai.go +++ b/uixt/mcp_tools_ai.go @@ -102,7 +102,7 @@ func (t *ToolAIAction) Implement() server.ToolHandlerFunc { } // AI action logic - err = driverExt.AIAction(ctx, unifiedReq.Prompt) + _, err = driverExt.AIAction(ctx, unifiedReq.Prompt) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("AI action failed: %s", err.Error())), nil } From b1719344c087ba70c2f9169fafd4627ef3460f7b Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 15:06:58 +0800 Subject: [PATCH 07/16] feat: enhance AI result handling with model name and usage statistics for query, action, and assertion types --- internal/version/VERSION | 2 +- report.go | 111 +++++++++++++++++++++++++++++++-------- uixt/ai/asserter.go | 20 +++++-- uixt/ai/asserter_test.go | 2 +- uixt/ai/querier.go | 27 +++++++--- uixt/ai/querier_test.go | 3 +- uixt/driver_ext_ai.go | 14 ++--- 7 files changed, 130 insertions(+), 49 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index f9a3ba97..9caa1f35 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241342 +v5.0.0-beta-2506241525 diff --git a/report.go b/report.go index 61ffb0b0..fbc9933e 100644 --- a/report.go +++ b/report.go @@ -417,16 +417,53 @@ func (g *HTMLReportGenerator) calculateTotalUsage() map[string]interface{} { continue } for _, action := range step.Actions { - if action.Plannings == nil { - continue - } - for _, planning := range action.Plannings { - if planning.Usage == nil { - continue + // Calculate planning usage + if action.Plannings != nil { + for _, planning := range action.Plannings { + if planning.Usage != nil { + totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + planning.Usage.PromptTokens + totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + planning.Usage.CompletionTokens + totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + planning.Usage.TotalTokens + } + } + } + + // Calculate AI operations usage (ai_query, ai_action, ai_assert) + if action.AIResult != nil { + var usage *map[string]interface{} + + switch action.AIResult.Type { + case "query": + if action.AIResult.QueryResult != nil && action.AIResult.QueryResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.QueryResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.QueryResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.QueryResult.Usage.TotalTokens, + } + } + case "action": + if action.AIResult.PlanningResult != nil && action.AIResult.PlanningResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.PlanningResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.PlanningResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.PlanningResult.Usage.TotalTokens, + } + } + case "assert": + if action.AIResult.AssertionResult != nil && action.AIResult.AssertionResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.AssertionResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.AssertionResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.AssertionResult.Usage.TotalTokens, + } + } + } + + if usage != nil { + totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + (*usage)["prompt_tokens"].(int) + totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + (*usage)["completion_tokens"].(int) + totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + (*usage)["total_tokens"].(int) } - totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + planning.Usage.PromptTokens - totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + planning.Usage.CompletionTokens - totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + planning.Usage.TotalTokens } } } @@ -1318,6 +1355,21 @@ const htmlTemplate = ` color: #495057; } + .structured-data { + background: #f8f9fa; + border: 1px solid #28a745; + border-radius: 6px; + padding: 10px 12px; + margin: 8px 0; + font-size: 0.85em; + color: #495057; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + white-space: pre-wrap; + word-wrap: break-word; + max-height: 200px; + overflow-y: auto; + } + @media screen and (max-width: 768px) { .planning-three-columns { flex-direction: column; @@ -2431,15 +2483,37 @@ const htmlTemplate = ` {{end}}
- {{if $action.AIResult.ModelName}} -
🤖 Model: {{$action.AIResult.ModelName}}
+ {{/* Model name and usage from specific result types */}} + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.ModelName}} +
🤖 Model: {{$action.AIResult.QueryResult.ModelName}}
+ {{end}} + {{if $action.AIResult.QueryResult.Usage}} +
📊 Tokens: {{$action.AIResult.QueryResult.Usage.PromptTokens}} in / {{$action.AIResult.QueryResult.Usage.CompletionTokens}} out / {{$action.AIResult.QueryResult.Usage.TotalTokens}} total
+ {{end}} + {{/* Display structured data for query results */}} + {{if $action.AIResult.QueryResult.Data}} +
📥 Structured Data:
+
{{safeHTML (toJSON $action.AIResult.QueryResult.Data)}}
+ {{end}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.ModelName}} +
🤖 Model: {{$action.AIResult.PlanningResult.ModelName}}
+ {{end}} + {{if $action.AIResult.PlanningResult.Usage}} +
📊 Tokens: {{$action.AIResult.PlanningResult.Usage.PromptTokens}} in / {{$action.AIResult.PlanningResult.Usage.CompletionTokens}} out / {{$action.AIResult.PlanningResult.Usage.TotalTokens}} total
+ {{end}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.ModelName}} +
🤖 Model: {{$action.AIResult.AssertionResult.ModelName}}
+ {{end}} + {{if $action.AIResult.AssertionResult.Usage}} +
📊 Tokens: {{$action.AIResult.AssertionResult.Usage.PromptTokens}} in / {{$action.AIResult.AssertionResult.Usage.CompletionTokens}} out / {{$action.AIResult.AssertionResult.Usage.TotalTokens}} total
+ {{end}} {{end}} {{if $action.AIResult.Resolution}}
📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}
{{end}} - {{if $action.AIResult.Usage}} -
📊 Tokens: {{$action.AIResult.Usage.PromptTokens}} in / {{$action.AIResult.Usage.CompletionTokens}} out / {{$action.AIResult.Usage.TotalTokens}} total
- {{end}} {{if $action.AIResult.Content}}
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.Content}}
{{end}} @@ -2495,9 +2569,6 @@ const htmlTemplate = `
{{base $screenshot.ImagePath}} - {{if $screenshot.Resolution}} - {{$screenshot.Resolution.Width}}x{{$screenshot.Resolution.Height}} - {{end}}
Screenshot @@ -2560,12 +2631,6 @@ const htmlTemplate = `
{{base $imagePath}} - {{if $screenshot.Resolution}} - {{$screenshot.Resolution.Width}}x{{$screenshot.Resolution.Height}} - {{else if index $screenshot "resolution"}} - {{$resolution := index $screenshot "resolution"}} - {{index $resolution "width"}}x{{index $resolution "height"}} - {{end}}
Screenshot diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index 58880c51..2760f6ff 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -30,8 +30,10 @@ type AssertOptions struct { // AssertionResult represents the response from an AI assertion type AssertionResult struct { - Pass bool `json:"pass"` - Thought string `json:"thought"` + Pass bool `json:"pass"` + Thought string `json:"thought"` + ModelName string `json:"model_name"` // model name used for assertion + Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics } // Asserter handles assertion using different AI models @@ -85,7 +87,7 @@ func NewAsserter(ctx context.Context, modelConfig *ModelConfig) (*Asserter, erro } // Assert performs the assertion check on the screenshot -func (a *Asserter) Assert(ctx context.Context, opts *AssertOptions) (*AssertionResult, error) { +func (a *Asserter) Assert(ctx context.Context, opts *AssertOptions) (result *AssertionResult, err error) { // Validate input parameters if err := validateAssertionInput(opts); err != nil { return nil, errors.Wrap(err, "validate assertion parameters failed") @@ -132,8 +134,15 @@ Here is the assertion. Please tell whether it is truthy according to the screens return nil, errors.Wrap(code.LLMRequestServiceError, err.Error()) } + defer func() { + // Extract usage information if available + if message.ResponseMeta != nil && message.ResponseMeta.Usage != nil { + result.Usage = message.ResponseMeta.Usage + } + }() + // Parse result - result, err := parseAssertionResult(message.Content) + result, err = parseAssertionResult(message.Content, a.modelConfig.ModelType) if err != nil { return nil, errors.Wrap(code.LLMParseAssertionResponseError, err.Error()) } @@ -159,7 +168,7 @@ func validateAssertionInput(opts *AssertOptions) error { } // parseAssertionResult parses the model response into AssertionResponse -func parseAssertionResult(content string) (*AssertionResult, error) { +func parseAssertionResult(content string, modelType option.LLMServiceType) (*AssertionResult, error) { var result AssertionResult // Use the generic structured response parser @@ -170,5 +179,6 @@ func parseAssertionResult(content string) (*AssertionResult, error) { return nil, errors.Wrap(code.LLMParseAssertionResponseError, err.Error()) } + result.ModelName = string(modelType) return &result, nil } diff --git a/uixt/ai/asserter_test.go b/uixt/ai/asserter_test.go index d9bdaaba..18fd3cdd 100644 --- a/uixt/ai/asserter_test.go +++ b/uixt/ai/asserter_test.go @@ -136,7 +136,7 @@ func TestParseAssertionResult(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := parseAssertionResult(tt.input) + result, err := parseAssertionResult(tt.input, option.DOUBAO_1_5_UI_TARS_250328) if tt.shouldSucceed { require.NoError(t, err) assert.NotNil(t, result) diff --git a/uixt/ai/querier.go b/uixt/ai/querier.go index 75d5dbc3..54194fad 100644 --- a/uixt/ai/querier.go +++ b/uixt/ai/querier.go @@ -32,9 +32,11 @@ type QueryOptions struct { // QueryResult represents the response from an AI query type QueryResult struct { - Content string `json:"content"` // The extracted content/information - Thought string `json:"thought"` // The reasoning process - Data interface{} `json:"data,omitempty"` // Structured data when OutputSchema is provided + Content string `json:"content"` // The extracted content/information + Thought string `json:"thought"` // The reasoning process + Data interface{} `json:"data,omitempty"` // Structured data when OutputSchema is provided + ModelName string `json:"model_name"` // model name used for query + Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics } // Querier handles query operations using different AI models @@ -89,7 +91,7 @@ func NewQuerier(ctx context.Context, modelConfig *ModelConfig) (*Querier, error) // callModelWithLogging calls the model with automatic logging and timing // Query performs the information extraction from the screenshot -func (q *Querier) Query(ctx context.Context, opts *QueryOptions) (*QueryResult, error) { +func (q *Querier) Query(ctx context.Context, opts *QueryOptions) (result *QueryResult, err error) { // Validate input parameters if err := validateQueryInput(opts); err != nil { return nil, errors.Wrap(err, "validate query parameters failed") @@ -141,8 +143,15 @@ Here is the query. Please extract the requested information from the screenshot. return nil, errors.Wrap(code.LLMRequestServiceError, err.Error()) } + defer func() { + // Extract usage information if available + if message.ResponseMeta != nil && message.ResponseMeta.Usage != nil { + result.Usage = message.ResponseMeta.Usage + } + }() + // Parse result - result, err := parseQueryResult(message.Content) + result, err = parseQueryResult(message.Content, q.modelConfig.ModelType) if err != nil { return nil, errors.Wrap(code.LLMParseQueryResponseError, err.Error()) } @@ -168,18 +177,20 @@ func validateQueryInput(opts *QueryOptions) error { } // parseQueryResult parses the model response into QueryResult -func parseQueryResult(content string) (*QueryResult, error) { +func parseQueryResult(content string, modelType option.LLMServiceType) (*QueryResult, error) { var result QueryResult // Use the generic structured response parser with enhanced error recovery if err := parseStructuredResponse(content, &result); err != nil { // If parseStructuredResponse fails completely, treat content as plain text return &QueryResult{ - Content: content, - Thought: "Failed to parse response, returning raw content", + Content: content, + Thought: "Failed to parse response, returning raw content", + ModelName: string(modelType), }, nil } + result.ModelName = string(modelType) return &result, nil } diff --git a/uixt/ai/querier_test.go b/uixt/ai/querier_test.go index 3793f7bf..99a0fed1 100644 --- a/uixt/ai/querier_test.go +++ b/uixt/ai/querier_test.go @@ -87,7 +87,6 @@ func loadTestImage(t *testing.T, path string) (string, types.Size) { } // Test functions - func TestParseQueryResult(t *testing.T) { tests := []struct { name string @@ -130,7 +129,7 @@ func TestParseQueryResult(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := parseQueryResult(tt.content) + result, err := parseQueryResult(tt.content, option.DOUBAO_1_5_UI_TARS_250328) assert.NoError(t, err) assert.Equal(t, tt.expected.Content, result.Content) assert.Equal(t, tt.expected.Thought, result.Thought) diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index f7f21009..e6aaf723 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -155,8 +155,6 @@ func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...optio ScreenshotElapsed: screenshotElapsed, ImagePath: screenResult.ImagePath, Resolution: &screenResult.Resolution, - ModelName: planningResult.ModelName, - Usage: planningResult.Usage, PlanningResult: &planningResult.PlanningResult, Thought: planningResult.Thought, Content: planningResult.Content, @@ -331,13 +329,11 @@ type PlanningExecutionResult struct { // AIExecutionResult represents a unified result structure for all AI operations type AIExecutionResult struct { - Type string `json:"type"` // operation type: "query", "action", "assert" - ModelCallElapsed int64 `json:"model_call_elapsed"` // model call elapsed time in milliseconds - ScreenshotElapsed int64 `json:"screenshot_elapsed"` // screenshot elapsed time in milliseconds - ImagePath string `json:"image_path"` // path to screenshot used for operation - Resolution *types.Size `json:"resolution"` // screen resolution - ModelName string `json:"model_name"` // model name used for operation - Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics + Type string `json:"type"` // operation type: "query", "action", "assert" + ModelCallElapsed int64 `json:"model_call_elapsed"` // model call elapsed time in milliseconds + ScreenshotElapsed int64 `json:"screenshot_elapsed"` // screenshot elapsed time in milliseconds + ImagePath string `json:"image_path"` // path to screenshot used for operation + Resolution *types.Size `json:"resolution"` // screen resolution // Operation-specific results (only one will be populated based on Type) QueryResult *ai.QueryResult `json:"query_result,omitempty"` // for ai_query operations From 4522bb2dbfa9147b57bad8720d4de79966f3cacf Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 15:36:19 +0800 Subject: [PATCH 08/16] fix: examples --- examples/game/llk/main.go | 22 ++++++++++------- examples/game/llk/main_test.go | 45 +++++++++++++++++++++++++++++++++- internal/version/VERSION | 2 +- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/examples/game/llk/main.go b/examples/game/llk/main.go index 28eed8df..2e0bfafa 100644 --- a/examples/game/llk/main.go +++ b/examples/game/llk/main.go @@ -12,7 +12,6 @@ import ( "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/uixt" - "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/rs/zerolog/log" ) @@ -124,14 +123,19 @@ func (bot *LLKGameBot) AnalyzeGameInterface() (*GameElement, error) { return gameElement, nil } -// convertToGameElement converts AI query result to GameElement -func convertToGameElement(result *ai.QueryResult) (*GameElement, error) { +// convertToGameElement converts AI execution result to GameElement +func convertToGameElement(result *uixt.AIExecutionResult) (*GameElement, error) { if result == nil { - return nil, fmt.Errorf("query result is nil") + return nil, fmt.Errorf("AI execution result is nil") } + if result.QueryResult == nil { + return nil, fmt.Errorf("query result is nil in AI execution result") + } + queryResult := result.QueryResult + // Try direct conversion first - if gameElement, ok := result.Data.(*GameElement); ok { + if gameElement, ok := queryResult.Data.(*GameElement); ok { return gameElement, nil } @@ -140,11 +144,11 @@ func convertToGameElement(result *ai.QueryResult) (*GameElement, error) { var sourceData interface{} // Use Data if available, otherwise try Content - if result.Data != nil { - sourceData = result.Data - } else if result.Content != "" { + if queryResult.Data != nil { + sourceData = queryResult.Data + } else if queryResult.Content != "" { var contentData map[string]interface{} - if err := json.Unmarshal([]byte(result.Content), &contentData); err != nil { + if err := json.Unmarshal([]byte(queryResult.Content), &contentData); err != nil { return nil, fmt.Errorf("failed to parse JSON from Content: %w", err) } sourceData = contentData diff --git a/examples/game/llk/main_test.go b/examples/game/llk/main_test.go index cb2bf680..981318ba 100644 --- a/examples/game/llk/main_test.go +++ b/examples/game/llk/main_test.go @@ -2,6 +2,8 @@ package llk import ( "context" + "encoding/json" + "fmt" "os" "testing" @@ -13,6 +15,47 @@ import ( "github.com/stretchr/testify/require" ) +// convertToGameElementFromQueryResult converts AI query result to GameElement for testing +func convertToGameElementFromQueryResult(result *ai.QueryResult) (*GameElement, error) { + if result == nil { + return nil, fmt.Errorf("query result is nil") + } + + // Try direct conversion first + if gameElement, ok := result.Data.(*GameElement); ok { + return gameElement, nil + } + + // Convert to JSON and back for flexible parsing + var gameElement GameElement + var sourceData interface{} + + // Use Data if available, otherwise try Content + if result.Data != nil { + sourceData = result.Data + } else if result.Content != "" { + var contentData map[string]interface{} + if err := json.Unmarshal([]byte(result.Content), &contentData); err != nil { + return nil, fmt.Errorf("failed to parse JSON from Content: %w", err) + } + sourceData = contentData + } else { + return nil, fmt.Errorf("no data available in query result") + } + + // Convert via JSON marshaling/unmarshaling + jsonBytes, err := json.Marshal(sourceData) + if err != nil { + return nil, fmt.Errorf("failed to marshal result data: %w", err) + } + + if err := json.Unmarshal(jsonBytes, &gameElement); err != nil { + return nil, fmt.Errorf("failed to unmarshal to GameElement: %w", err) + } + + return &gameElement, nil +} + // hasRequiredEnvVars checks if the required environment variables are set for testing func hasRequiredEnvVars() bool { // Check for OpenAI environment variables @@ -83,7 +126,7 @@ Return JSON with: content, dimensions{rows,cols}, elements[{type,position{row,co require.NoError(t, err, "Failed to query AI model") // Convert result using enhanced compatibility logic - gameElement, err := convertToGameElement(result) + gameElement, err := convertToGameElementFromQueryResult(result) require.NoError(t, err, "Failed to convert query result to GameElement") require.NotNil(t, gameElement, "GameElement should not be nil") diff --git a/internal/version/VERSION b/internal/version/VERSION index 9caa1f35..a40a80c8 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241525 +v5.0.0-beta-2506241536 From d0ceeb6c518aaee21ed32bd0e2a0751fca4fd248 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 16:01:50 +0800 Subject: [PATCH 09/16] refactor: update AI result handling to differentiate content and thought based on result types in report generation --- internal/version/VERSION | 2 +- report.go | 31 ++++++++++++++++++++++++++----- uixt/driver_ext_ai.go | 10 +--------- uixt/mcp_tools_ai.go | 2 +- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index a40a80c8..0c419f8c 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241536 +v5.0.0-beta-2506241601 diff --git a/report.go b/report.go index fbc9933e..9ab06436 100644 --- a/report.go +++ b/report.go @@ -2442,9 +2442,19 @@ const htmlTemplate = ` {{if $action.AIResult}}
- - {{if $action.AIResult.Thought}} -
{{$action.AIResult.Thought}}
+ + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.Thought}} +
{{$action.AIResult.QueryResult.Thought}}
+ {{end}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.Thought}} +
{{$action.AIResult.PlanningResult.Thought}}
+ {{end}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.Thought}} +
{{$action.AIResult.AssertionResult.Thought}}
+ {{end}} {{end}} @@ -2514,8 +2524,19 @@ const htmlTemplate = ` {{if $action.AIResult.Resolution}}
📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}
{{end}} - {{if $action.AIResult.Content}} -
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.Content}}
+ {{/* Display Content from specific result types */}} + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.QueryResult.Content}}
+ {{end}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.PlanningResult.Content}}
+ {{end}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.AssertionResult.Content}}
+ {{end}} {{end}}
diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index e6aaf723..bef8ac5f 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -156,8 +156,6 @@ func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...optio ImagePath: screenResult.ImagePath, Resolution: &screenResult.Resolution, PlanningResult: &planningResult.PlanningResult, - Thought: planningResult.Thought, - Content: planningResult.Content, } if err != nil { @@ -341,9 +339,7 @@ type AIExecutionResult struct { AssertionResult *ai.AssertionResult `json:"assertion_result,omitempty"` // for ai_assert operations // Common fields - Thought string `json:"thought,omitempty"` // AI reasoning/thought process - Content string `json:"content,omitempty"` // operation result content - Error string `json:"error,omitempty"` // error message if operation failed + Error string `json:"error,omitempty"` // error message if operation failed } // SubActionResult represents a sub-action within a start_to_goal action @@ -408,8 +404,6 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*AIExec ImagePath: screenResult.ImagePath, // screenshot path Resolution: &screenResult.Resolution, // screen resolution QueryResult: result, // query-specific result - Thought: result.Thought, // AI reasoning - Content: result.Content, // query result content } return aiResult, nil } @@ -458,7 +452,6 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* ImagePath: screenResult.ImagePath, Resolution: &screenResult.Resolution, AssertionResult: result, - Thought: result.Thought, } if err != nil { @@ -471,6 +464,5 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* return aiResult, errors.New(result.Thought) } - aiResult.Content = "Assertion passed" return aiResult, nil } diff --git a/uixt/mcp_tools_ai.go b/uixt/mcp_tools_ai.go index 5fa87c05..e29e3cc6 100644 --- a/uixt/mcp_tools_ai.go +++ b/uixt/mcp_tools_ai.go @@ -174,7 +174,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc { message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt) returnData := ToolAIQuery{ Prompt: unifiedReq.Prompt, - Result: queryResult.Content, + Result: queryResult.QueryResult.Content, } return NewMCPSuccessResponse(message, &returnData), nil From 5e4b5db64abd94fc09173c064785a4449042f506 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 17:00:29 +0800 Subject: [PATCH 10/16] change: update docs for summary --- docs/summary-structure.md | 354 ++++++++++++++++++++++---------------- internal/version/VERSION | 2 +- 2 files changed, 208 insertions(+), 148 deletions(-) diff --git a/docs/summary-structure.md b/docs/summary-structure.md index c7785f68..041b56f6 100644 --- a/docs/summary-structure.md +++ b/docs/summary-structure.md @@ -2,7 +2,7 @@ ## 概述 -HttpRunner 的 Summary 数据结构用于存储测试执行的完整汇总信息，包括测试结果、统计数据、时间信息、平台信息以及详细的测试步骤记录。本文档基于 `summary.go` 的代码定义和实际执行产物进行详细说明。 +HttpRunner 的 Summary 数据结构用于存储测试执行的完整汇总信息，包括测试结果、统计数据、时间信息、平台信息以及详细的测试步骤记录。本文档基于 `summary.go` 和相关代码的最新定义进行详细说明。 ## 数据结构层次关系 @@ -32,6 +32,10 @@ Summary (根结构) │ │ ├── Usage (模型使用统计) │ │ ├── ScreenResult (屏幕结果) │ │ └── SubActions (子操作) + │ ├── AIResult (统一AI操作结果) + │ │ ├── QueryResult (查询结果) + │ │ ├── PlanningResult (规划结果) + │ │ └── AssertionResult (断言结果) │ └── ScreenResults (屏幕截图) └── Attachments (附件信息) ``` @@ -184,14 +188,18 @@ Summary (根结构) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| name | string | `name` | 步骤名称 | -| start_time | int64 | `start_time` | 开始时间（Unix时间戳，毫秒） | -| step_type | string | `step_type` | 步骤类型（如 "android_validation", "android"） | -| success | bool | `success` | 步骤执行是否成功 | -| elapsed_ms | int | `elapsed_ms` | 执行耗时（毫秒） | -| data | *SessionData | `data` | 步骤相关数据（包含请求响应和验证结果） | -| actions | []Action | `actions` | 执行的操作列表 | -| attachments | map[string]interface{} | `attachments` | 附件信息（如截图等） | +| Name | string | `name` | 步骤名称 | +| Identifier | string | `identifier,omitempty` | 步骤标识符 | +| StartTime | int64 | `start_time` | 开始时间（Unix时间戳，毫秒） | +| StepType | StepType | `step_type` | 步骤类型（如 "android_validation", "android"） | +| Success | bool | `success` | 步骤执行是否成功 | +| Elapsed | int64 | `elapsed_ms` | 执行耗时（毫秒） | +| HttpStat | map[string]int64 | `httpstat,omitempty` | HTTP统计信息（毫秒） | +| Data | interface{} | `data,omitempty` | 步骤相关数据 | +| ContentSize | int64 | `content_size,omitempty` | 响应体长度 | +| ExportVars | map[string]interface{} | `export_vars,omitempty` | 提取的变量 | +| Actions | []*ActionResult | `actions,omitempty` | 执行的操作列表 | +| Attachments | interface{} | `attachments,omitempty` | 附件信息（如截图等） | **示例数据**: ```json @@ -207,7 +215,136 @@ Summary (根结构) } ``` -### 10. SessionData (步骤数据) +### 10. ActionResult (操作结果) + +每个步骤可能包含多个操作，每个操作的详细执行信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| MobileAction | option.MobileAction | `,inline` | 移动端操作信息（内联） | +| StartTime | int64 | `start_time` | 操作开始时间（Unix时间戳，毫秒） | +| Elapsed | int64 | `elapsed_ms` | 操作耗时（毫秒） | +| Error | string | `error,omitempty` | 操作执行错误信息 | +| Plannings | []*uixt.PlanningExecutionResult | `plannings,omitempty` | AI规划执行结果（用于start_to_goal操作） | +| AIResult | *uixt.AIExecutionResult | `ai_result,omitempty` | 统一AI执行结果（用于ai_query/ai_action/ai_assert操作） | +| SessionData | uixt.SessionData | - | 会话数据（内联，包含请求和屏幕截图信息） | + +### 11. AIExecutionResult (统一AI执行结果) + +这是所有AI操作（ai_query、ai_action、ai_assert）的统一结果结构： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Type | string | `type` | 操作类型："query"、"action"、"assert" | +| ModelCallElapsed | int64 | `model_call_elapsed` | 模型调用耗时（毫秒） | +| ScreenshotElapsed | int64 | `screenshot_elapsed` | 截图耗时（毫秒） | +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | *types.Size | `resolution` | 屏幕分辨率 | +| QueryResult | *ai.QueryResult | `query_result,omitempty` | 查询操作结果（仅query类型） | +| PlanningResult | *ai.PlanningResult | `planning_result,omitempty` | 规划操作结果（仅action类型） | +| AssertionResult | *ai.AssertionResult | `assertion_result,omitempty` | 断言操作结果（仅assert类型） | +| Error | string | `error,omitempty` | 操作失败的错误信息 | + +**示例数据**: +```json +{ + "type": "query", + "model_call_elapsed": 1234, + "screenshot_elapsed": 567, + "image_path": "/path/to/screenshot.png", + "resolution": {"width": 1080, "height": 1920}, + "query_result": { /* 查询结果详情 */ } +} +``` + +### 12. QueryResult (查询结果) + +用于ai_query操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Content | string | `content` | 提取的内容/信息 | +| Thought | string | `thought` | AI的推理过程 | +| Data | interface{} | `data,omitempty` | 结构化数据（当提供OutputSchema时） | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "content": "搜索框位于屏幕右上角", + "thought": "通过分析截图，我看到了页面右上角有一个搜索图标", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 1234, + "completion_tokens": 56, + "total_tokens": 1290 + } +} +``` + +### 13. PlanningResult (规划结果) + +用于ai_action操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ToolCalls | []schema.ToolCall | `tool_calls` | 工具调用列表 | +| Thought | string | `thought` | AI的思考过程 | +| Content | string | `content` | 模型的原始内容 | +| Error | string | `error,omitempty` | 规划错误信息 | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "tool_calls": [ + { + "id": "tap_xy_1750657286", + "type": "function", + "function": { + "name": "uixt__tap_xy", + "arguments": "{\"x\":1107.6,\"y\":232.4}" + } + } + ], + "thought": "点击页面右上角的搜索图标，打开搜索界面", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 2199, + "completion_tokens": 135, + "total_tokens": 2334 + } +} +``` + +### 14. AssertionResult (断言结果) + +用于ai_assert操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Pass | bool | `pass` | 断言是否通过 | +| Thought | string | `thought` | AI的推理过程 | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "pass": true, + "thought": "根据截图分析，当前页面确实显示了搜索结果", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 1500, + "completion_tokens": 45, + "total_tokens": 1545 + } +} +``` + +### 15. SessionData (会话数据) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| @@ -215,14 +352,14 @@ Summary (根结构) | Address | *Address | `address,omitempty` | 网络地址信息 | | Validators | []*ValidationResult | `validators,omitempty` | 验证结果列表 | -### 11. ReqResps (请求响应) +### 16. ReqResps (请求响应) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| | Request | interface{} | `request` | 请求信息 | | Response | interface{} | `response` | 响应信息 | -### 12. ValidationResult (验证结果) +### 17. ValidationResult (验证结果) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| @@ -245,166 +382,85 @@ Summary (根结构) } ``` -### 13. Action (操作信息) +### 18. PlanningExecutionResult (规划执行结果) -每个步骤可能包含多个操作： +用于复杂的start_to_goal操作，包含规划和执行的完整信息： | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| method | string | `method` | 操作方法名（如 "app_launch", "start_to_goal"） | -| params | interface{} | `params` | 操作参数 | -| start_time | int64 | `start_time` | 操作开始时间（Unix时间戳，毫秒） | -| elapsed_ms | int | `elapsed_ms` | 操作耗时（毫秒） | -| requests | []Request | `requests` | HTTP请求记录（如果有） | -| plannings | []Planning | `plannings` | AI规划信息（UI自动化场景） | -| screen_results | []ScreenResult | `screen_results` | 屏幕截图结果 | +| PlanningResult | ai.PlanningResult | - | 继承的规划结果字段 | +| ScreenshotElapsed | int64 | `screenshot_elapsed_ms` | 截图耗时（毫秒） | +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | *types.Size | `resolution` | 图像分辨率 | +| ScreenResult | *ScreenResult | `screen_result` | 完整屏幕结果数据 | +| ModelCallElapsed | int64 | `model_call_elapsed_ms` | 模型调用耗时（毫秒） | +| ToolCallsCount | int | `tool_calls_count` | 生成的工具调用数量 | +| ActionNames | []string | `action_names` | 解析的操作名称列表 | +| StartTime | int64 | `start_time` | 规划开始时间 | +| Elapsed | int64 | `elapsed_ms` | 规划耗时（毫秒） | +| SubActions | []*SubActionResult | `sub_actions,omitempty` | 此规划生成的子操作 | -**示例数据**: -```json -{ - "method": "start_to_goal", - "params": "搜索「青榕小剧场」，切换到「用户」搜索结果页，点击进入第一个搜索结果的用户个人主页", - "start_time": 1750657275855, - "elapsed_ms": 109543, - "plannings": [ /* AI规划列表 */ ] -} -``` - -### 14. Request (请求记录) - -ADB或HTTP请求的详细记录： - -| 字段名 | 类型 | JSON标签 | 说明 | -|--------|------|----------|------| -| request_method | string | `request_method` | 请求方法（如 "adb", "http"） | -| request_url | string | `request_url` | 请求URL或命令 | -| request_body | string | `request_body` | 请求体或命令参数 | -| request_time | string | `request_time` | 请求时间（ISO格式） | -| response_status | int | `response_status` | 响应状态码 | -| response_duration_ms | int | `response_duration(ms)` | 响应耗时（毫秒） | -| response_body | string | `response_body` | 响应内容 | -| success | bool | `success` | 请求是否成功 | - -**示例数据**: -```json -{ - "request_method": "adb", - "request_url": "monkey", - "request_body": "-p com.smile.gifmaker -c android.intent.category.LAUNCHER 1", - "request_time": "2025-06-23T13:41:07.200504+08:00", - "response_status": 0, - "response_duration(ms)": 566, - "response_body": "Events injected: 1\n## Network stats: elapsed time=45ms", - "success": true -} -``` - -### 15. Planning (AI规划信息) - -UI自动化测试中的AI规划详情： - -| 字段名 | 类型 | JSON标签 | 说明 | -|--------|------|----------|------| -| tool_calls | []ToolCall | `tool_calls` | 工具调用信息 | -| thought | string | `thought` | AI的思考过程 | -| content | string | `content` | 规划内容（JSON格式的操作描述） | -| model_name | string | `model_name` | 使用的AI模型名称 | -| usage | Usage | `usage` | 模型使用统计 | -| screenshot_elapsed_ms | int | `screenshot_elapsed_ms` | 截图耗时（毫秒） | -| image_path | string | `image_path` | 截图文件路径 | -| resolution | Resolution | `resolution` | 屏幕分辨率 | -| screen_result | ScreenResult | `screen_result` | 屏幕分析结果 | -| model_call_elapsed_ms | int | `model_call_elapsed_ms` | 模型调用耗时（毫秒） | -| tool_calls_count | int | `tool_calls_count` | 工具调用次数 | -| action_names | []string | `action_names` | 执行的操作名称列表 | -| start_time | int64 | `start_time` | 规划开始时间 | -| elapsed_ms | int | `elapsed_ms` | 规划总耗时 | -| sub_actions | []SubAction | `sub_actions` | 子操作列表 | - -**示例数据**: -```json -{ - "tool_calls": [ - { - "id": "tap_xy_1750657286", - "type": "function", - "function": { - "name": "uixt__tap_xy", - "arguments": "{\"x\":1107.6,\"y\":232.4}" - } - } - ], - "thought": "点击页面右上角的搜索图标，打开搜索界面以进行后续搜索操作。", - "model_name": "doubao-1.5-thinking-vision-pro-250428", - "usage": { - "prompt_tokens": 2199, - "completion_tokens": 135, - "total_tokens": 2334 - } -} -``` - -### 16. ToolCall (工具调用) +### 19. ToolCall (工具调用) AI规划中的工具调用信息： | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| id | string | `id` | 工具调用唯一标识 | -| type | string | `type` | 调用类型（通常为 "function"） | -| function | Function | `function` | 函数调用详情 | +| ID | string | `id` | 工具调用唯一标识 | +| Type | string | `type` | 调用类型（通常为 "function"） | +| Function | Function | `function` | 函数调用详情 | -### 17. Function (函数调用) +### 20. Function (函数调用) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| name | string | `name` | 函数名称（如 "uixt__tap_xy"） | -| arguments | string | `arguments` | 函数参数（JSON字符串格式） | +| Name | string | `name` | 函数名称（如 "uixt__tap_xy"） | +| Arguments | string | `arguments` | 函数参数（JSON字符串格式） | -### 18. Usage (模型使用统计) +### 21. TokenUsage (token使用统计) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| prompt_tokens | int | `prompt_tokens` | 输入token数量 | -| completion_tokens | int | `completion_tokens` | 输出token数量 | -| total_tokens | int | `total_tokens` | 总token数量 | +| PromptTokens | int | `prompt_tokens` | 输入token数量 | +| CompletionTokens | int | `completion_tokens` | 输出token数量 | +| TotalTokens | int | `total_tokens` | 总token数量 | -### 19. Resolution (分辨率) +### 22. Size (分辨率) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| width | int | `width` | 屏幕宽度（像素） | -| height | int | `height` | 屏幕高度（像素） | +| Width | int | `width` | 屏幕宽度（像素） | +| Height | int | `height` | 屏幕高度（像素） | -### 20. ScreenResult (屏幕结果) +### 23. ScreenResult (屏幕结果) | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| image_path | string | `image_path` | 截图文件路径 | -| resolution | Resolution | `resolution` | 屏幕分辨率 | -| uploaded_url | string | `uploaded_url` | 上传后的URL（通常为空） | -| texts | []Text | `texts` | 识别的文本信息（可为null） | -| icons | []Icon | `icons` | 识别的图标信息（可为null） | -| tags | []Tag | `tags` | 识别的标签信息（可为null） | +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | Size | `resolution` | 屏幕分辨率 | +| UploadedURL | string | `uploaded_url` | 上传后的URL（通常为空） | +| Texts | []Text | `texts` | 识别的文本信息（可为null） | +| Icons | []Icon | `icons` | 识别的图标信息（可为null） | +| Tags | []Tag | `tags` | 识别的标签信息（可为null） | -### 21. SubAction (子操作) +### 24. SubActionResult (子操作结果) 规划中实际执行的具体操作： | 字段名 | 类型 | JSON标签 | 说明 | |--------|------|----------|------| -| action_name | string | `action_name` | 操作名称（如 "uixt__tap_xy"） | -| arguments | string | `arguments` | 操作参数（JSON字符串） | -| start_time | int64 | `start_time` | 操作开始时间 | -| elapsed_ms | int | `elapsed_ms` | 操作耗时 | -| requests | []Request | `requests` | 相关的请求记录 | -| screen_results | []ScreenResult | `screen_results` | 操作后的屏幕截图 | +| ActionName | string | `action_name` | 操作名称（如 "uixt__tap_xy"） | +| Arguments | interface{} | `arguments,omitempty` | 操作参数 | +| StartTime | int64 | `start_time` | 操作开始时间 | +| Elapsed | int64 | `elapsed_ms` | 操作耗时 | +| Error | error | `error,omitempty` | 操作执行错误 | +| SessionData | SessionData | - | 会话数据（内联） | **示例数据**: ```json { "action_name": "uixt__tap_xy", - "arguments": "{\"x\":1107.6,\"y\":232.4}", + "arguments": {"x": 1107.6, "y": 232.4}, "start_time": 1750657286274, "elapsed_ms": 319, "requests": [ /* 请求记录 */ ], @@ -412,26 +468,30 @@ AI规划中的工具调用信息： } ``` -### 22. Attachments (附件信息) +## 重要架构变更说明 -步骤执行过程中产生的附件，主要是截图： +### AI操作统一架构 -| 字段名 | 类型 | JSON标签 | 说明 | -|--------|------|----------|------| -| screen_results | []ScreenResult | `screen_results` | 屏幕截图列表 | +HttpRunner v5 引入了统一的AI操作架构： -## 数据类型层次关系 +1. **统一结果结构**: `AIExecutionResult` 作为所有AI操作的统一容器 +2. **类型区分**: 通过 `Type` 字段区分不同的AI操作类型 +3. **具体结果**: 根据操作类型，在对应的结果字段中存储具体数据 +4. **统一时间统计**: 所有AI操作都包含模型调用和截图的时间统计 +5. **统一错误处理**: 通过 `Error` 字段统一处理所有AI操作的错误 -### 时间戳格式 +### 数据类型和时间格式 + +#### 时间戳格式 - **Unix时间戳（毫秒）**: 用于 `start_time` 字段，如 `1750657267057` - **ISO时间格式**: 用于 `start_at` 和 `request_time` 字段，如 `"2025-06-23T13:41:06.150641+08:00"` -### 耗时统计 -- **毫秒级**: `elapsed_ms`, `response_duration(ms)`, `screenshot_elapsed_ms`, `model_call_elapsed_ms` +#### 耗时统计 +- **毫秒级**: `elapsed_ms`, `model_call_elapsed`, `screenshot_elapsed` 等 - **秒级**: `duration` 字段使用浮点数表示秒 -### 状态标识 -- **布尔值**: `success`, `Success` 表示操作或测试是否成功 +#### 状态标识 +- **布尔值**: `success`, `pass` 表示操作或测试是否成功 - **字符串**: `check_result` 使用 "pass"/"fail" 表示验证结果 -这个数据结构设计充分考虑了测试执行的各种场景，特别是UI自动化测试中的复杂交互和AI规划过程，为测试结果的分析和报告提供了完整的数据基础。通过详细的嵌套字段定义，开发者可以精确理解和使用每个数据元素，实现更强大的测试分析和报告功能。 \ No newline at end of file +这个数据结构设计充分考虑了现代UI自动化测试的需求，特别是AI驱动的测试场景。通过统一的AI操作架构和详细的嵌套字段定义，为测试结果的分析、报告生成和调试提供了完整的数据基础。 \ No newline at end of file diff --git a/internal/version/VERSION b/internal/version/VERSION index 0c419f8c..dffda2cc 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241601 +v5.0.0-beta-2506241700 From 54001363e4b5611ace3dd40d0e3ad7ed9fbc2d87 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 19:46:12 +0800 Subject: [PATCH 11/16] feat: save json case to results directory --- internal/config/config.go | 16 ++++++++++++++++ internal/version/VERSION | 2 +- runner.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index 1242f6a1..0f04e438 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ const ( SummaryFileName = "hrp_summary.json" // $PWD/results/20060102150405/hrp_summary.json LogFileName = "hrp.log" // $PWD/results/20060102150405/hrp.log ReportFileName = "report.html" // $PWD/results/20060102150405/report.html + CaseFileName = "case.json" // $PWD/results/20060102150405/case.json // mobile device path DeviceActionLogFilePath = "/sdcard/Android/data/io.appium.uiautomator2.server/files/hodor" @@ -36,6 +37,7 @@ type Config struct { summaryFilePath string logFilePath string reportFilePath string + caseFilePath string actionLogDirPath string mu sync.Mutex } @@ -179,3 +181,17 @@ func (c *Config) ReportFilePath() string { c.reportFilePath = filepath.Join(c.resultsPathUnlocked(), ReportFileName) return c.reportFilePath } + +// $PWD/results/20060102150405/case.json +func (c *Config) CaseFilePath() string { + c.mu.Lock() + defer c.mu.Unlock() + + if c.caseFilePath != "" { + return c.caseFilePath + } + + // Ensure directory creation and set cached path + c.caseFilePath = filepath.Join(c.resultsPathUnlocked(), CaseFileName) + return c.caseFilePath +} diff --git a/internal/version/VERSION b/internal/version/VERSION index dffda2cc..b1ed4424 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241700 +v5.0.0-beta-2506241946 diff --git a/runner.go b/runner.go index ed13eb05..370ed97c 100644 --- a/runner.go +++ b/runner.go @@ -26,6 +26,7 @@ import ( "github.com/httprunner/funplugin" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/builtin" + "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/sdk" "github.com/httprunner/httprunner/v5/internal/version" "github.com/httprunner/httprunner/v5/mcphost" @@ -677,6 +678,13 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) (summary *TestCa summary.InOut.ExportVars = exportVars summary.InOut.ConfigVars = config.Variables + // Save JSON case content to results directory + if config.Path != "" { + if err := saveJSONCase(config.Path); err != nil { + log.Warn().Err(err).Str("path", config.Path).Msg("save JSON case failed") + } + } + // TODO: move to mobile ui step // Collect logs from cached drivers for _, cached := range uixt.ListCachedDrivers() { @@ -1022,3 +1030,25 @@ func (r *SessionRunner) GetSessionVariables() map[string]interface{} { func (r *SessionRunner) GetTransactions() map[string]map[TransactionType]time.Time { return r.transactions } + +// saveJSONCase saves the original JSON case content to the results directory +func saveJSONCase(casePath string) error { + // Read the original JSON case content + path := TestCasePath(casePath) + testCase, err := path.GetTestCase() + if err != nil { + return errors.Wrap(err, "load JSON case failed") + } + + // remove environs from testcase config + tConfig := testCase.Config.(*TConfig) + tConfig.Environs = nil + + // save JSON case to results directory + jsonCasePath := config.GetConfig().CaseFilePath() + err = testCase.Dump2JSON(jsonCasePath) + if err != nil { + return errors.Wrap(err, "dump JSON case failed") + } + return nil +} From 084c28834c548304d7c236db3a5c3ff948ef669a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 21:08:59 +0800 Subject: [PATCH 12/16] fix: improve step execution handling with interrupt support and result logging --- internal/version/VERSION | 2 +- runner.go | 64 +++++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index b1ed4424..e2084c6b 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506241946 +v5.0.0-beta-2506242108 diff --git a/runner.go b/runner.go index 370ed97c..0aa7754f 100644 --- a/runner.go +++ b/runner.go @@ -865,23 +865,9 @@ func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) var stepResults []*StepResult - // execute with loops as outer iteration - for _, task := range tasks { - // execute step with merged variables - stepResult, err := r.executeStepWithVariables(step, task.stepName, task.parameters) - if err != nil { - if r.caseRunner.hrpRunner.failfast { - return nil, errors.Wrap(err, "execute step failed") - } - log.Error().Err(err).Str("step", task.stepName).Msg("execute step failed") - } - - stepResults = append(stepResults, stepResult) - } - - // return the last step result, or nil if no steps were executed - if len(stepResults) > 0 { - // add all step results to summary + // defer to save results regardless of how the function exits + defer func() { + // Save all completed results to summary for _, result := range stepResults { r.summary.AddStepResult(result) // update extracted variables from the last result @@ -891,22 +877,54 @@ func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) } // log final result - lastResult := stepResults[len(stepResults)-1] - if lastResult.Success { + if err == nil && stepResult.Success { log.Info().Str("step", stepName). Str("type", stepType). Bool("success", true). - Int64("elapsed(ms)", lastResult.Elapsed). - Interface("exportVars", lastResult.ExportVars). + Int64("elapsed(ms)", stepResult.Elapsed). + Interface("exportVars", stepResult.ExportVars). Msg(RUN_STEP_END) - } else { + } else if stepResult != nil { log.Error().Str("step", stepName). Str("type", stepType). Bool("success", false). - Int64("elapsed(ms)", lastResult.Elapsed). + Int64("elapsed(ms)", stepResult.Elapsed). + Int("completed_tasks", len(stepResults)). + Int("total_tasks", len(tasks)). Msg(RUN_STEP_END) } + }() + // execute with loops as outer iteration + for _, task := range tasks { + // Check for interrupt signal before each parameter iteration + select { + case <-r.caseRunner.hrpRunner.interruptSignal: + log.Warn().Int("completed_tasks", len(stepResults)). + Int("total_tasks", len(tasks)). + Msg("interrupted during parameter iteration") + return nil, errors.Wrap(code.InterruptError, "parameter iteration interrupted") + default: + } + + // execute step with merged variables + stepResult, stepErr := r.executeStepWithVariables(step, task.stepName, task.parameters) + if stepErr != nil { + if r.caseRunner.hrpRunner.failfast { + // failfast mode, abort running + return nil, errors.Wrap(stepErr, "execute step failed") + } + log.Error().Err(stepErr).Str("step", task.stepName).Msg("execute step failed") + } + + if stepResult != nil { + stepResults = append(stepResults, stepResult) + } + } + + // return last result + if len(stepResults) > 0 { + lastResult := stepResults[len(stepResults)-1] return lastResult, nil } From 43d990f8630339d41bb4a4bb2bc2a163c9209eb1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 22:22:34 +0800 Subject: [PATCH 13/16] fix: ensure extracted thought is not empty before displaying in report --- internal/version/VERSION | 2 +- report.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index e2084c6b..4e7a29cf 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506242108 +v5.0.0-beta-2506242222 diff --git a/report.go b/report.go index 9ab06436..a7aa5fe8 100644 --- a/report.go +++ b/report.go @@ -541,7 +541,7 @@ func (g *HTMLReportGenerator) GenerateReport(outputFile string) error { // Try to parse as JSON to extract thought field var data map[string]interface{} if err := json.Unmarshal([]byte(content), &data); err == nil { - if thought, ok := data["thought"].(string); ok { + if thought, ok := data["thought"].(string); ok && thought != "" { return thought } } @@ -2325,8 +2325,15 @@ const htmlTemplate = ` {{if $planning.Error}}Error: {{$planning.Error}}{{end}}
- {{if $planning.Thought}} -
{{$planning.Thought}}
+ {{$extractedThought := extractThought $planning.Content}} + {{if or $planning.Thought $extractedThought}} +
+ {{if $planning.Thought}} + {{$planning.Thought}} + {{else}} + {{$extractedThought}} + {{end}} +
{{end}} From 72a0915b04a55cbd5bb1db14bced0afa7ec0ecbc Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 22:54:26 +0800 Subject: [PATCH 14/16] fix: adb double tap --- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 4e7a29cf..e2491d84 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506242222 +v5.0.0-beta-2506242254 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index e883078e..55b6a7a1 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -339,15 +339,11 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error xStr := fmt.Sprintf("%.1f", x) yStr := fmt.Sprintf("%.1f", y) _, err = ad.runShellCommand( + "input", "tap", xStr, yStr, ";", + "sleep", "0.05", ";", "input", "tap", xStr, yStr) if err != nil { - return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) - } - time.Sleep(time.Duration(100) * time.Millisecond) - _, err = ad.runShellCommand( - "input", "tap", xStr, yStr) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) + return errors.Wrap(err, fmt.Sprintf("double tap <%s, %s> failed", xStr, yStr)) } return nil } From 53fad4edc50fac31203b7a7c78fa253cc46f4c4a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 24 Jun 2025 23:10:46 +0800 Subject: [PATCH 15/16] refactor: streamline AI assertion result handling by consolidating error management and improving result structure --- internal/version/VERSION | 2 +- step.go | 2 +- uixt/driver_ext_ai.go | 38 ++++++++++++++++---------------------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index e2491d84..16331be9 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506242254 +v5.0.0-beta-2506242310 diff --git a/step.go b/step.go index 266a937a..c82be894 100644 --- a/step.go +++ b/step.go @@ -65,7 +65,7 @@ type ActionResult struct { Error string `json:"error,omitempty"` // action execution result Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions AIResult *uixt.AIExecutionResult `json:"ai_result,omitempty"` // store unified AI execution result for ai_query/ai_action/ai_assert actions - uixt.SessionData // store session data for other actions besides start_to_goal & ai_query + uixt.SessionData // store session data for other actions besides start_to_goal } // one testcase contains one or multiple steps diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index bef8ac5f..1b11ca2c 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -424,15 +424,17 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* return nil, err } + assertResult := &AIExecutionResult{ + Type: "assert", + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, + } + screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize() if err != nil { - return &AIExecutionResult{ - Type: "assert", - ScreenshotElapsed: screenshotElapsed, - ImagePath: screenResult.ImagePath, - Resolution: &screenResult.Resolution, - Error: err.Error(), - }, err + assertResult.Error = err.Error() + return assertResult, err } // Step 2: Call model and measure time @@ -443,26 +445,18 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* Size: size, } result, err := dExt.LLMService.Assert(context.Background(), assertOpts) - modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() - - aiResult := &AIExecutionResult{ - Type: "assert", - ModelCallElapsed: modelCallElapsed, - ScreenshotElapsed: screenshotElapsed, - ImagePath: screenResult.ImagePath, - Resolution: &screenResult.Resolution, - AssertionResult: result, - } + assertResult.ModelCallElapsed = time.Since(modelCallStartTime).Milliseconds() + assertResult.AssertionResult = result if err != nil { - aiResult.Error = err.Error() - return aiResult, errors.Wrap(err, "AI assertion failed") + assertResult.Error = err.Error() + return assertResult, errors.Wrap(err, "AI assertion failed") } if !result.Pass { - aiResult.Error = result.Thought - return aiResult, errors.New(result.Thought) + assertResult.Error = result.Thought + return assertResult, errors.New(result.Thought) } - return aiResult, nil + return assertResult, nil } From 70471d2fb451856396bdd6cd6133580f4736689f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 25 Jun 2025 11:57:09 +0800 Subject: [PATCH 16/16] fix: enhance logging for interrupted processes and ensure step results are saved in failfast mode --- internal/version/VERSION | 2 +- runner.go | 17 ++++++++++++----- uixt/driver_ext_ai.go | 12 ++++++++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 16331be9..ffe8319b 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506242310 +v5.0.0-beta-2506251157 diff --git a/runner.go b/runner.go index 0aa7754f..88ebe740 100644 --- a/runner.go +++ b/runner.go @@ -909,17 +909,24 @@ func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) // execute step with merged variables stepResult, stepErr := r.executeStepWithVariables(step, task.stepName, task.parameters) + + // Always add stepResult to stepResults if it exists, even on error + // This ensures data is saved in defer function for summary generation + if stepResult != nil { + stepResults = append(stepResults, stepResult) + } + if stepErr != nil { if r.caseRunner.hrpRunner.failfast { - // failfast mode, abort running + // failfast mode, abort running but step result is already saved above + log.Error().Err(stepErr). + Str("step", task.stepName). + Int("completed_tasks", len(stepResults)). + Msg("execute step failed in failfast mode, step result saved") return nil, errors.Wrap(stepErr, "execute step failed") } log.Error().Err(stepErr).Str("step", task.stepName).Msg("execute step failed") } - - if stepResult != nil { - stepResults = append(stepResults, stepResult) - } } // return last result diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 1b11ca2c..c2d7042e 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -30,7 +30,10 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Check for context cancellation (interrupt signal) select { case <-ctx.Done(): - log.Warn().Msg("interrupted in StartToGoal") + log.Warn(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Msg("interrupted in StartToGoal") return allPlannings, errors.Wrap(code.InterruptError, "StartToGoal interrupted") default: } @@ -83,7 +86,12 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Check for context cancellation before each action select { case <-ctx.Done(): - log.Warn().Msg("interrupted in invokeToolCalls") + log.Warn(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Int("completed_tool_calls", len(planningResult.SubActions)). + Int("total_tool_calls", len(planningResult.ToolCalls)). + Msg("interrupted in invokeToolCalls") planningResult.Elapsed = time.Since(planningStartTime).Milliseconds() allPlannings = append(allPlannings, planningResult) return allPlannings, errors.Wrap(code.InterruptError, "invokeToolCalls interrupted")