diff --git a/config.go b/config.go index 34399576..cbb165b5 100644 --- a/config.go +++ b/config.go @@ -88,6 +88,20 @@ func (c *TConfig) WithParameters(parameters map[string]interface{}) *TConfig { return c } +// WithParametersSetting sets parameters setting for current testcase. +func (c *TConfig) WithParametersSetting(options ...ParametersOption) *TConfig { + if c.ParametersSetting == nil { + c.ParametersSetting = &TParamsConfig{} + } + + // apply all options + for _, option := range options { + option(c.ParametersSetting) + } + + return c +} + // SetThinkTime sets think time config for current testcase. func (c *TConfig) SetThinkTime(strategy ThinkTimeStrategy, cfg interface{}, limit float64) *TConfig { c.ThinkTimeSetting = &ThinkTimeConfig{strategy, cfg, limit} diff --git a/docs/parameters.md b/docs/parameters.md new file mode 100644 index 00000000..c8c9aeb3 --- /dev/null +++ b/docs/parameters.md @@ -0,0 +1,387 @@ +# HttpRunner 参数化功能 (Parameters) + +## 概述 + +HttpRunner 支持强大的**数据驱动测试**能力，允许用户在**测试用例（Testcase）**和**测试步骤（Step）**两个层级上进行参数化。这使得测试用例可以与外部数据文件解耦，实现更灵活、可维护性更高的自动化测试。 + +- **测试用例层级参数化**：对整个测试流程使用多组不同的数据重复执行。适用于需要验证完整业务流程的场景，例如使用不同用户登录并执行相同操作。 +- **测试步骤层级参数化**：仅在单个步骤内使用不同的参数重复执行。适用于需要验证单个功能点的场景，例如在搜索框中输入不同的关键词。 + +## 测试用例层级参数化 (TestCase-Level) + +当您需要使用不同的数据集完整地运行整个测试流程时，应使用测试用例层级的参数化。 + +### 使用方法 + +通过在 `hrp.TestCase` 的 `Parameters` 字段中定义参数，并可选择使用 `WithParametersSetting` 进行策略配置。 + +```go +// testcase_parameters_test.go +func TestTestcaseParameters(t *testing.T) { + testcase := &hrp.TestCase{ + Config: hrp.NewConfig("测试用例层级参数化"). + WithParameters(map[string]interface{}{ + "username-password": [][]interface{}{ + {"user1", "pass1"}, + {"user2", "pass2"}, + {"user3", "pass3"}, + }, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机选择参数 + hrp.WithLimit(2), // 只执行2次 + ), + TestSteps: []hrp.IStep{ + hrp.NewStep("登录"). + POST("/api/login"). + WithBody(map[string]interface{}{ + "username": "$username", + "password": "$password", + }), + hrp.NewStep("获取用户信息"). + GET("/api/user/info"), + }, + } + + err := hrp.NewRunner(t).Run(testcase) + assert.Nil(t, err) +} +``` + +### 执行结果 + +- 整个测试用例将运行两次。 +- 第一次运行时，`$username` 为 `user1`，`$password` 为 `pass1`。 +- 第二次运行时，`$username` 为 `user2`，`$password` 为 `pass2`。 +- 测试报告中会生成两个独立的测试结果，每个结果对应一组参数。 + +--- + +## 测试步骤层级参数化 (TestStep-Level) + +当您只需要在一个测试流程中对某个特定步骤使用不同参数进行多次测试时，应使用测试步骤层级的参数化。 + +### 使用场景 + +- 需要对同一个操作使用不同参数进行多次测试 +- 减少重复的步骤定义代码 +- 在 UI 自动化测试中，对同一个界面操作使用不同的输入数据 + +### 核心设计 + +#### 1. 架构改动 + +##### StepConfig 结构扩展 +```go +type StepConfig struct { + StepName string `json:"name" yaml:"name"` + Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` + Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` // 新增 + ParametersSetting *TParamsConfig `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` // 新增 + Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` // 已弃用，建议使用 Loop() 方法 + // ... 其他字段保持不变 +} +``` + +##### TParamsConfig 参数配置 +```go +type TParamsConfig struct { + PickOrder iteratorPickOrder `json:"pick_order,omitempty" yaml:"pick_order,omitempty"` + Strategies map[string]IteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` +} +``` + +#### 2. Limit 和 Loops 的关系 + +**重要**：`Limit` 和 `Loops` 不是替代关系，而是可以同时存在的不同控制机制： + +1. **Limit（总数限制）**： + - 作用：限制参数迭代的**总执行次数** + - 位置：`TParamsConfig.Limit` + - 示例：`Limit = 4` 表示最多执行4次，无论有多少参数 + +2. **Loops（循环次数）**： + - 作用：设置单个 parameter 的**循环使用次数** + - 位置：`StepConfig.Loops`（通过 `Loop()` 方法设置） + - 示例：`Loops = 2` 表示每个参数都循环执行2次 + +##### 执行顺序优化 + +用户期望的执行顺序（外层循环）： +```go +// 参数：["A", "B"]，Loops = 2 +// 执行顺序：A, B, A, B（先遍历所有参数，再进行下一轮循环） +// 步骤命名：[loop_1_params_1], [loop_1_params_2], [loop_2_params_1], [loop_2_params_2] +``` + +#### 3. 统一执行流程 + +``` +1. 收集所有参数组合 +2. 外层循环：循环次数 (1 到 loopTimes) +3. 内层循环：遍历所有参数 +4. 执行步骤并收集结果 +``` + +### API 设计 + +#### 统一的参数配置方法 + +为了避免代码重复和保持API一致性，参数配置方法只在 `StepRequest` 中定义： + +- `WithParameters()` - 设置步骤级参数 +- `WithParametersSetting()` - 设置参数选择策略（支持 option 模式） +- `Loop()` - 设置循环次数 +- `WithVariables()` - 设置步骤变量 + +这些方法返回 `*StepRequest`，可以与其他步骤类型（如 Mobile、HTTP 等）配合使用。 + +#### Option API（推荐） + +新的 option API 提供更简洁和灵活的参数配置方式： + +```go +// 单个 option +hrp.NewStep("测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京"}, + }). + WithParametersSetting(hrp.WithSequentialOrder()). + Loop(2). + Android() + +// 多个 options 组合 +hrp.NewStep("测试"). + WithParameters(...). + WithParametersSetting( + hrp.WithRandomOrder(), // 设置随机选择顺序 + hrp.WithLimit(4), // 限制总执行次数 + hrp.WithStrategy("param1", hrp.IteratorStrategy{ + Name: "custom", + PickOrder: "random", + }), + ). + Loop(2). + Android() +``` + +##### 可用的 Option 函数 + +###### 参数选择顺序 +- `hrp.WithSequentialOrder()` - 设置参数按顺序执行（默认） +- `hrp.WithRandomOrder()` - 设置参数随机选择 +- `hrp.WithUniqueOrder()` - 设置参数唯一选择，避免重复 + +###### 其他配置 +- `hrp.WithLimit(limit int)` - 设置参数迭代总数限制 +- `hrp.WithStrategy(paramName string, strategy IteratorStrategy)` - 为特定参数设置策略 + +#### 正确的调用顺序 + +**重要**：参数配置必须在指定步骤类型之前调用： + +```go +// ✅ 推荐：使用 Loop 方法 +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + Loop(3). // 使用 Loop 方法 + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ✅ 推荐：使用新的 Option API +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting( + hrp.WithSequentialOrder(), + hrp.WithLimit(3), + ). + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ✅ 兼容：使用传统 TParamsConfig +hrp.NewStep("搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting(&hrp.TParamsConfig{ + PickOrder: "sequential", + Limit: 3, + }). + Android(). + StartToGoal("进入搜索框，输入查询词「$query」") + +// ❌ 错误：在指定步骤类型后无法配置参数 +hrp.NewStep("搜索测试"). + Android(). + WithParameters(...) // 这里会编译错误 +``` + +### 基本用法 + +#### 1. 简单参数列表 + +```go +hrp.NewStep("搜索查询词"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + Android(). + StartToGoal("进入搜索框，输入 query「$query」，等待加载出 sug 提示词") +``` + +#### 2. 复合参数 + +```go +hrp.NewStep("登录测试"). + WithParameters(map[string]interface{}{ + "username-password": [][]interface{}{ + {"user1", "pass1"}, + {"user2", "pass2"}, + {"user3", "pass3"}, + }, + }). + POST("/api/login"). + WithBody(map[string]interface{}{ + "username": "$username", + "password": "$password", + }) +``` + +#### 3. 参数策略配置 + +```go +hrp.NewStep("随机测试数据"). + WithParameters(map[string]interface{}{ + "data": []interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机选择 + hrp.WithLimit(3), // 只执行3次 + ). + POST("/api/test"). + WithBody(map[string]interface{}{ + "value": "$data", + }) +``` + +#### 4. 循环配置 + +```go +// 推荐方式：使用 Loop 方法 +hrp.NewStep("循环测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京"}, + }). + Loop(3). // 每个参数重复执行3次 + Android(). + StartToGoal("搜索「$query」") + +// 总执行次数：2 × 3 = 6 次 +// 执行顺序：成都, 北京, 成都, 北京, 成都, 北京 +``` + +### 参数策略选项 + +- `sequential`: 顺序执行（默认） +- `random`: 随机选择 +- `unique`: 唯一值，避免重复 + +### 高级特性 + +#### 1. 与现有功能兼容 + +步骤级参数化与现有功能完全兼容： + +- ✅ 支持 Loop 循环 +- ✅ 支持变量提取和校验 +- ✅ 支持 SetupHooks 和 TeardownHooks +- ✅ 支持 failfast 控制 + +#### 2. 步骤命名规则 + +- 无参数，无循环：`步骤名称` +- 无参数，有循环：`步骤名称_loop_N` +- 有参数，无循环：`步骤名称 [params_N]` +- 有参数，有循环：`步骤名称 [loop_M_params_N]`（外层循环优化） + +#### 3. 变量恢复机制 + +每次执行完成后，会自动恢复原始的步骤变量，避免副作用影响后续执行。 + +### 完整示例 + +```go +func TestStepParametersExample(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("步骤级参数化示例"), + TestSteps: []hrp.IStep{ + hrp.NewStep("多查询词搜索测试"). + WithParameters(map[string]interface{}{ + "query": []interface{}{"成都", "北京", "重庆"}, + }). + WithParametersSetting( + hrp.WithSequentialOrder(), + hrp.WithLimit(3), + ). + Android(). + StartToGoal("搜索「$query」并等待结果加载"). + AIQuery("提取搜索结果"), + hrp.NewStep("循环执行测试"). + WithParameters(map[string]interface{}{ + "data": []interface{}{1, 2, 3}, + }). + Loop(2). // 每个参数执行2次 + GET("/api/test"). + WithParams(map[string]interface{}{ + "value": "$data", + }), + }, + } + + // 预期执行： + // 第一个步骤：成都, 北京, 重庆 = 3次 + // 第二个步骤：1, 2, 3, 1, 2, 3 = 6次 + + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} +``` + +## 对比与选择 + +| 特性 | 测试用例层级 (Testcase-Level) | 测试步骤层级 (Step-Level) | +| :--- | :--- | :--- | +| **适用场景** | 完整的业务流程验证 | 单个功能点或API的验证 | +| **数据作用域** | 整个 `TestCase` 中的所有 `Step` | 单个 `Step` | +| **报告结果** | 每组参数生成一个独立的测试报告 | 所有参数在同一个测试报告中 | +| **性能** | 开销较大，每次都需执行完整流程 | 开销较小，仅重复执行单个步骤 | +| **配置方法** | `hrp.TestCase{ Parameters: ... }` | `hrp.Step{}.WithParameters(...)` | + +**选择建议**： +- 当您需要用多组数据测试一个完整的端到端流程时（例如，不同的用户配置），请使用**测试用例层级**参数化。 +- 当您只需要在一个流程中，针对某个特定步骤进行多次数据测试时（例如，测试接口对不同输入的响应），请使用**测试步骤层级**参数化。 + +## 变量优先级 + +HttpRunner 中的变量优先级遵循覆盖原则，顺序如下（从高到低）： + +1. **步骤级参数化变量** (`WithParameters`) +2. **用例级参数化变量** (`TestCase.Parameters`) +3. **步骤级变量** (`WithVariables`) +4. **会话变量** (前面步骤提取的变量) +5. **配置变量** (`Config.Variables`) + +## 注意事项 + +1. **API 调用顺序**：步骤级参数化方法必须在步骤类型方法（如 `Android()`、`POST()` 等）之前调用 +2. **参数继承**：`StepMobile` 等步骤类型继承 `StepConfig`，参数会正确传递 +3. **编译时检查**：错误的调用顺序会在编译时报错，确保API使用正确性 +4. **变量覆盖**：Parameters 会覆盖同名的 Variables，但不会影响其他变量 +5. **变量恢复**：每次执行后自动恢复原始变量，避免副作用 +6. **执行顺序**：步骤级参数化采用外层循环优化，先遍历所有参数，再进行下一轮循环 +7. **Option API 推荐**：新项目建议使用 option 模式的 API，更加灵活和简洁 \ No newline at end of file diff --git a/docs/summary-structure.md b/docs/summary-structure.md new file mode 100644 index 00000000..041b56f6 --- /dev/null +++ b/docs/summary-structure.md @@ -0,0 +1,497 @@ +# Summary 数据结构说明文档 + +## 概述 + +HttpRunner 的 Summary 数据结构用于存储测试执行的完整汇总信息，包括测试结果、统计数据、时间信息、平台信息以及详细的测试步骤记录。本文档基于 `summary.go` 和相关代码的最新定义进行详细说明。 + +## 数据结构层次关系 + +``` +Summary (根结构) +├── Success (bool) +├── Stat (统计信息) +│ ├── TestCases (测试用例统计) +│ └── TestSteps (测试步骤统计) +├── Time (时间信息) +├── Platform (平台信息) +└── Details (测试用例详情列表) + └── TestCaseSummary (单个测试用例汇总) + ├── Stat (步骤统计) + ├── Time (用例时间) + ├── InOut (输入输出) + ├── Logs (日志) + └── Records (步骤记录) + └── StepResult (步骤结果) + ├── Data (步骤数据) + │ ├── ReqResps (请求响应) + │ └── Validators (验证器) + ├── Actions (操作列表) + │ ├── Requests (请求记录) + │ ├── Plannings (AI规划) + │ │ ├── ToolCalls (工具调用) + │ │ ├── Usage (模型使用统计) + │ │ ├── ScreenResult (屏幕结果) + │ │ └── SubActions (子操作) + │ ├── AIResult (统一AI操作结果) + │ │ ├── QueryResult (查询结果) + │ │ ├── PlanningResult (规划结果) + │ │ └── AssertionResult (断言结果) + │ └── ScreenResults (屏幕截图) + └── Attachments (附件信息) +``` + +## 详细数据结构说明 + +### 1. Summary (主汇总结构) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Success | bool | `success` | 整体测试执行是否成功 | +| Stat | *Stat | `stat` | 汇总统计信息 | +| Time | *TestCaseTime | `time` | 整体执行时间信息 | +| Platform | *Platform | `platform` | 平台和版本信息 | +| Details | []*TestCaseSummary | `details` | 各个测试用例的详细信息 | +| rootDir | string | - | 根目录路径（私有字段） | + +**示例数据**: +```json +{ + "success": true, + "stat": { /* 统计信息 */ }, + "time": { /* 时间信息 */ }, + "platform": { /* 平台信息 */ }, + "details": [ /* 测试用例详情 */ ] +} +``` + +### 2. Stat (统计信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| TestCases | TestCaseStat | `testcases` | 测试用例统计 | +| TestSteps | TestStepStat | `teststeps` | 测试步骤统计 | + +### 3. TestCaseStat (测试用例统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Total | int | `total` | 测试用例总数 | +| Success | int | `success` | 成功的测试用例数 | +| Fail | int | `fail` | 失败的测试用例数 | + +**示例数据**: +```json +{ + "testcases": { + "total": 1, + "success": 1, + "fail": 0 + } +} +``` + +### 4. TestStepStat (测试步骤统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Total | int | `total` | 测试步骤总数 | +| Successes | int | `successes` | 成功的步骤数 | +| Failures | int | `failures` | 失败的步骤数 | +| Actions | map[option.ActionName]int | `actions` | 各种操作的统计计数 | + +**示例数据**: +```json +{ + "teststeps": { + "total": 5, + "successes": 5, + "failures": 0, + "actions": {} + } +} +``` + +### 5. TestCaseTime (时间信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| StartAt | time.Time | `start_at,omitempty` | 开始时间 | +| Duration | float64 | `duration,omitempty` | 持续时间（秒） | + +**示例数据**: +```json +{ + "time": { + "start_at": "2025-06-23T13:41:06.150641+08:00", + "duration": 188.998332334 + } +} +``` + +### 6. Platform (平台信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| HttprunnerVersion | string | `httprunner_version` | HttpRunner 版本号 | +| GoVersion | string | `go_version` | Go 语言版本 | +| Platform | string | `platform` | 操作系统平台信息 | + +**示例数据**: +```json +{ + "platform": { + "httprunner_version": "v5.0.0-beta-2506222254", + "go_version": "go1.24.1", + "platform": "darwin-arm64" + } +} +``` + +### 7. TestCaseSummary (测试用例汇总) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Name | string | `name` | 测试用例名称 | +| Success | bool | `success` | 用例执行是否成功 | +| CaseId | string | `case_id,omitempty` | 用例ID（可选） | +| Stat | *TestStepStat | `stat` | 该用例的步骤统计 | +| Time | *TestCaseTime | `time` | 该用例的时间信息 | +| InOut | *TestCaseInOut | `in_out` | 输入输出信息 | +| Logs | []interface{} | `logs,omitempty` | 日志信息 | +| Records | []*StepResult | `records` | 步骤执行记录 | +| RootDir | string | `root_dir,omitempty` | 根目录路径 | + +### 8. TestCaseInOut (输入输出信息) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ConfigVars | map[string]interface{} | `config_vars` | 配置变量 | +| ExportVars | map[string]interface{} | `export_vars` | 导出变量 | + +**示例数据**: +```json +{ + "in_out": { + "config_vars": { + "OPENAI_API_KEY": "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", + "dramaName": "涂山赊刀", + "userName": "青榕小剧场" + }, + "export_vars": {} + } +} +``` + +### 9. StepResult (步骤结果) + +步骤结果是 Records 数组中的元素，包含每个测试步骤的详细执行信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Name | string | `name` | 步骤名称 | +| Identifier | string | `identifier,omitempty` | 步骤标识符 | +| StartTime | int64 | `start_time` | 开始时间（Unix时间戳，毫秒） | +| StepType | StepType | `step_type` | 步骤类型（如 "android_validation", "android"） | +| Success | bool | `success` | 步骤执行是否成功 | +| Elapsed | int64 | `elapsed_ms` | 执行耗时（毫秒） | +| HttpStat | map[string]int64 | `httpstat,omitempty` | HTTP统计信息（毫秒） | +| Data | interface{} | `data,omitempty` | 步骤相关数据 | +| ContentSize | int64 | `content_size,omitempty` | 响应体长度 | +| ExportVars | map[string]interface{} | `export_vars,omitempty` | 提取的变量 | +| Actions | []*ActionResult | `actions,omitempty` | 执行的操作列表 | +| Attachments | interface{} | `attachments,omitempty` | 附件信息（如截图等） | + +**示例数据**: +```json +{ + "name": "启动快手 app", + "start_time": 1750657267057, + "step_type": "android_validation", + "success": true, + "elapsed_ms": 8797, + "data": { /* 步骤数据 */ }, + "actions": [ /* 操作列表 */ ], + "attachments": { /* 附件信息 */ } +} +``` + +### 10. ActionResult (操作结果) + +每个步骤可能包含多个操作，每个操作的详细执行信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| MobileAction | option.MobileAction | `,inline` | 移动端操作信息（内联） | +| StartTime | int64 | `start_time` | 操作开始时间（Unix时间戳，毫秒） | +| Elapsed | int64 | `elapsed_ms` | 操作耗时（毫秒） | +| Error | string | `error,omitempty` | 操作执行错误信息 | +| Plannings | []*uixt.PlanningExecutionResult | `plannings,omitempty` | AI规划执行结果（用于start_to_goal操作） | +| AIResult | *uixt.AIExecutionResult | `ai_result,omitempty` | 统一AI执行结果（用于ai_query/ai_action/ai_assert操作） | +| SessionData | uixt.SessionData | - | 会话数据（内联，包含请求和屏幕截图信息） | + +### 11. AIExecutionResult (统一AI执行结果) + +这是所有AI操作（ai_query、ai_action、ai_assert）的统一结果结构： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Type | string | `type` | 操作类型："query"、"action"、"assert" | +| ModelCallElapsed | int64 | `model_call_elapsed` | 模型调用耗时（毫秒） | +| ScreenshotElapsed | int64 | `screenshot_elapsed` | 截图耗时（毫秒） | +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | *types.Size | `resolution` | 屏幕分辨率 | +| QueryResult | *ai.QueryResult | `query_result,omitempty` | 查询操作结果（仅query类型） | +| PlanningResult | *ai.PlanningResult | `planning_result,omitempty` | 规划操作结果（仅action类型） | +| AssertionResult | *ai.AssertionResult | `assertion_result,omitempty` | 断言操作结果（仅assert类型） | +| Error | string | `error,omitempty` | 操作失败的错误信息 | + +**示例数据**: +```json +{ + "type": "query", + "model_call_elapsed": 1234, + "screenshot_elapsed": 567, + "image_path": "/path/to/screenshot.png", + "resolution": {"width": 1080, "height": 1920}, + "query_result": { /* 查询结果详情 */ } +} +``` + +### 12. QueryResult (查询结果) + +用于ai_query操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Content | string | `content` | 提取的内容/信息 | +| Thought | string | `thought` | AI的推理过程 | +| Data | interface{} | `data,omitempty` | 结构化数据（当提供OutputSchema时） | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "content": "搜索框位于屏幕右上角", + "thought": "通过分析截图，我看到了页面右上角有一个搜索图标", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 1234, + "completion_tokens": 56, + "total_tokens": 1290 + } +} +``` + +### 13. PlanningResult (规划结果) + +用于ai_action操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ToolCalls | []schema.ToolCall | `tool_calls` | 工具调用列表 | +| Thought | string | `thought` | AI的思考过程 | +| Content | string | `content` | 模型的原始内容 | +| Error | string | `error,omitempty` | 规划错误信息 | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "tool_calls": [ + { + "id": "tap_xy_1750657286", + "type": "function", + "function": { + "name": "uixt__tap_xy", + "arguments": "{\"x\":1107.6,\"y\":232.4}" + } + } + ], + "thought": "点击页面右上角的搜索图标，打开搜索界面", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 2199, + "completion_tokens": 135, + "total_tokens": 2334 + } +} +``` + +### 14. AssertionResult (断言结果) + +用于ai_assert操作的具体结果： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Pass | bool | `pass` | 断言是否通过 | +| Thought | string | `thought` | AI的推理过程 | +| ModelName | string | `model_name` | 使用的模型名称 | +| Usage | *schema.TokenUsage | `usage,omitempty` | token使用统计 | + +**示例数据**: +```json +{ + "pass": true, + "thought": "根据截图分析，当前页面确实显示了搜索结果", + "model_name": "doubao-1.5-thinking-vision-pro-250428", + "usage": { + "prompt_tokens": 1500, + "completion_tokens": 45, + "total_tokens": 1545 + } +} +``` + +### 15. SessionData (会话数据) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ReqResps | *ReqResps | `req_resps` | 请求响应数据 | +| Address | *Address | `address,omitempty` | 网络地址信息 | +| Validators | []*ValidationResult | `validators,omitempty` | 验证结果列表 | + +### 16. ReqResps (请求响应) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Request | interface{} | `request` | 请求信息 | +| Response | interface{} | `response` | 响应信息 | + +### 17. ValidationResult (验证结果) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Check | string | `check` | 验证检查项 | +| Assert | string | `assert` | 断言类型 | +| Expect | interface{} | `expect` | 期望值 | +| Msg | string | `msg` | 验证消息 | +| CheckValue | interface{} | `check_value` | 实际检查值 | +| CheckResult | string | `check_result` | 检查结果（"pass"/"fail"） | + +**示例数据**: +```json +{ + "check": "ui_foreground_app", + "assert": "equal", + "expect": "com.smile.gifmaker", + "msg": "app [com.smile.gifmaker] should be in foreground", + "check_value": null, + "check_result": "pass" +} +``` + +### 18. PlanningExecutionResult (规划执行结果) + +用于复杂的start_to_goal操作，包含规划和执行的完整信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| PlanningResult | ai.PlanningResult | - | 继承的规划结果字段 | +| ScreenshotElapsed | int64 | `screenshot_elapsed_ms` | 截图耗时（毫秒） | +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | *types.Size | `resolution` | 图像分辨率 | +| ScreenResult | *ScreenResult | `screen_result` | 完整屏幕结果数据 | +| ModelCallElapsed | int64 | `model_call_elapsed_ms` | 模型调用耗时（毫秒） | +| ToolCallsCount | int | `tool_calls_count` | 生成的工具调用数量 | +| ActionNames | []string | `action_names` | 解析的操作名称列表 | +| StartTime | int64 | `start_time` | 规划开始时间 | +| Elapsed | int64 | `elapsed_ms` | 规划耗时（毫秒） | +| SubActions | []*SubActionResult | `sub_actions,omitempty` | 此规划生成的子操作 | + +### 19. ToolCall (工具调用) + +AI规划中的工具调用信息： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ID | string | `id` | 工具调用唯一标识 | +| Type | string | `type` | 调用类型（通常为 "function"） | +| Function | Function | `function` | 函数调用详情 | + +### 20. Function (函数调用) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Name | string | `name` | 函数名称（如 "uixt__tap_xy"） | +| Arguments | string | `arguments` | 函数参数（JSON字符串格式） | + +### 21. TokenUsage (token使用统计) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| PromptTokens | int | `prompt_tokens` | 输入token数量 | +| CompletionTokens | int | `completion_tokens` | 输出token数量 | +| TotalTokens | int | `total_tokens` | 总token数量 | + +### 22. Size (分辨率) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| Width | int | `width` | 屏幕宽度（像素） | +| Height | int | `height` | 屏幕高度（像素） | + +### 23. ScreenResult (屏幕结果) + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ImagePath | string | `image_path` | 截图文件路径 | +| Resolution | Size | `resolution` | 屏幕分辨率 | +| UploadedURL | string | `uploaded_url` | 上传后的URL（通常为空） | +| Texts | []Text | `texts` | 识别的文本信息（可为null） | +| Icons | []Icon | `icons` | 识别的图标信息（可为null） | +| Tags | []Tag | `tags` | 识别的标签信息（可为null） | + +### 24. SubActionResult (子操作结果) + +规划中实际执行的具体操作： + +| 字段名 | 类型 | JSON标签 | 说明 | +|--------|------|----------|------| +| ActionName | string | `action_name` | 操作名称（如 "uixt__tap_xy"） | +| Arguments | interface{} | `arguments,omitempty` | 操作参数 | +| StartTime | int64 | `start_time` | 操作开始时间 | +| Elapsed | int64 | `elapsed_ms` | 操作耗时 | +| Error | error | `error,omitempty` | 操作执行错误 | +| SessionData | SessionData | - | 会话数据（内联） | + +**示例数据**: +```json +{ + "action_name": "uixt__tap_xy", + "arguments": {"x": 1107.6, "y": 232.4}, + "start_time": 1750657286274, + "elapsed_ms": 319, + "requests": [ /* 请求记录 */ ], + "screen_results": [ /* 屏幕截图 */ ] +} +``` + +## 重要架构变更说明 + +### AI操作统一架构 + +HttpRunner v5 引入了统一的AI操作架构： + +1. **统一结果结构**: `AIExecutionResult` 作为所有AI操作的统一容器 +2. **类型区分**: 通过 `Type` 字段区分不同的AI操作类型 +3. **具体结果**: 根据操作类型，在对应的结果字段中存储具体数据 +4. **统一时间统计**: 所有AI操作都包含模型调用和截图的时间统计 +5. **统一错误处理**: 通过 `Error` 字段统一处理所有AI操作的错误 + +### 数据类型和时间格式 + +#### 时间戳格式 +- **Unix时间戳（毫秒）**: 用于 `start_time` 字段，如 `1750657267057` +- **ISO时间格式**: 用于 `start_at` 和 `request_time` 字段，如 `"2025-06-23T13:41:06.150641+08:00"` + +#### 耗时统计 +- **毫秒级**: `elapsed_ms`, `model_call_elapsed`, `screenshot_elapsed` 等 +- **秒级**: `duration` 字段使用浮点数表示秒 + +#### 状态标识 +- **布尔值**: `success`, `pass` 表示操作或测试是否成功 +- **字符串**: `check_result` 使用 "pass"/"fail" 表示验证结果 + +这个数据结构设计充分考虑了现代UI自动化测试的需求，特别是AI驱动的测试场景。通过统一的AI操作架构和详细的嵌套字段定义，为测试结果的分析、报告生成和调试提供了完整的数据基础。 \ No newline at end of file diff --git a/examples/game/llk/main.go b/examples/game/llk/main.go index 28eed8df..2e0bfafa 100644 --- a/examples/game/llk/main.go +++ b/examples/game/llk/main.go @@ -12,7 +12,6 @@ import ( "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/uixt" - "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/rs/zerolog/log" ) @@ -124,14 +123,19 @@ func (bot *LLKGameBot) AnalyzeGameInterface() (*GameElement, error) { return gameElement, nil } -// convertToGameElement converts AI query result to GameElement -func convertToGameElement(result *ai.QueryResult) (*GameElement, error) { +// convertToGameElement converts AI execution result to GameElement +func convertToGameElement(result *uixt.AIExecutionResult) (*GameElement, error) { if result == nil { - return nil, fmt.Errorf("query result is nil") + return nil, fmt.Errorf("AI execution result is nil") } + if result.QueryResult == nil { + return nil, fmt.Errorf("query result is nil in AI execution result") + } + queryResult := result.QueryResult + // Try direct conversion first - if gameElement, ok := result.Data.(*GameElement); ok { + if gameElement, ok := queryResult.Data.(*GameElement); ok { return gameElement, nil } @@ -140,11 +144,11 @@ func convertToGameElement(result *ai.QueryResult) (*GameElement, error) { var sourceData interface{} // Use Data if available, otherwise try Content - if result.Data != nil { - sourceData = result.Data - } else if result.Content != "" { + if queryResult.Data != nil { + sourceData = queryResult.Data + } else if queryResult.Content != "" { var contentData map[string]interface{} - if err := json.Unmarshal([]byte(result.Content), &contentData); err != nil { + if err := json.Unmarshal([]byte(queryResult.Content), &contentData); err != nil { return nil, fmt.Errorf("failed to parse JSON from Content: %w", err) } sourceData = contentData diff --git a/examples/game/llk/main_test.go b/examples/game/llk/main_test.go index cb2bf680..981318ba 100644 --- a/examples/game/llk/main_test.go +++ b/examples/game/llk/main_test.go @@ -2,6 +2,8 @@ package llk import ( "context" + "encoding/json" + "fmt" "os" "testing" @@ -13,6 +15,47 @@ import ( "github.com/stretchr/testify/require" ) +// convertToGameElementFromQueryResult converts AI query result to GameElement for testing +func convertToGameElementFromQueryResult(result *ai.QueryResult) (*GameElement, error) { + if result == nil { + return nil, fmt.Errorf("query result is nil") + } + + // Try direct conversion first + if gameElement, ok := result.Data.(*GameElement); ok { + return gameElement, nil + } + + // Convert to JSON and back for flexible parsing + var gameElement GameElement + var sourceData interface{} + + // Use Data if available, otherwise try Content + if result.Data != nil { + sourceData = result.Data + } else if result.Content != "" { + var contentData map[string]interface{} + if err := json.Unmarshal([]byte(result.Content), &contentData); err != nil { + return nil, fmt.Errorf("failed to parse JSON from Content: %w", err) + } + sourceData = contentData + } else { + return nil, fmt.Errorf("no data available in query result") + } + + // Convert via JSON marshaling/unmarshaling + jsonBytes, err := json.Marshal(sourceData) + if err != nil { + return nil, fmt.Errorf("failed to marshal result data: %w", err) + } + + if err := json.Unmarshal(jsonBytes, &gameElement); err != nil { + return nil, fmt.Errorf("failed to unmarshal to GameElement: %w", err) + } + + return &gameElement, nil +} + // hasRequiredEnvVars checks if the required environment variables are set for testing func hasRequiredEnvVars() bool { // Check for OpenAI environment variables @@ -83,7 +126,7 @@ Return JSON with: content, dimensions{rows,cols}, elements[{type,position{row,co require.NoError(t, err, "Failed to query AI model") // Convert result using enhanced compatibility logic - gameElement, err := convertToGameElement(result) + gameElement, err := convertToGameElementFromQueryResult(result) require.NoError(t, err, "Failed to convert query result to GameElement") require.NotNil(t, gameElement, "GameElement should not be nil") diff --git a/examples/parameters/main_test.go b/examples/parameters/main_test.go new file mode 100644 index 00000000..c18ce7fa --- /dev/null +++ b/examples/parameters/main_test.go @@ -0,0 +1,135 @@ +package parameters + +import ( + "testing" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" + "github.com/stretchr/testify/assert" +) + +// TestParametersExecutionScenarios 涵盖了参数化的核心执行场景， +// 包括纯参数驱动、参数与循环结合，以及使用随机和限制等设置。 +func TestParametersExecutionScenarios(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("测试参数化核心执行场景"). + WithVariables(map[string]interface{}{"loops": 2}), + TestSteps: []hrp.IStep{ + // 场景1: 纯参数驱动 + hrp.NewStep("API请求 - 纯参数"). + WithParameters(map[string]interface{}{ + "arg1": []int{10, 20}, + "arg2": []string{"a", "b"}, + }). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"p1": "$arg1", "p2": "$arg2"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + + // 场景2: 参数与 Loops 结合 + hrp.NewStep("API请求 - 参数与Loops结合"). + WithParameters(map[string]interface{}{ + "word": []string{"hello", "world"}, + }). + Loop(3). // 每个参数执行3次 + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"search": "$word"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + + // 场景3: 参数设置 (随机, 限制) + hrp.NewStep("API请求 - 参数设置"). + WithParameters(map[string]interface{}{ + "city": []string{"chengdu", "beijing", "shanghai", "guangzhou"}, + }). + WithParametersSetting( + hrp.WithRandomOrder(), // 随机顺序 + hrp.WithLimit(2), // 总共执行2次 + ). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{"city": "$city"}). + Validate(). + AssertEqual("status_code", 200, "check status code"), + }, + } + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} + +// TestParametersVariableOverride 用于验证参数(parameters)如何覆盖测试用例配置(config) +// 和步骤(step)中定义的同名变量(variables)。 +func TestParametersVariableOverride(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("测试参数覆盖变量的优先级"). + WithVariables(map[string]interface{}{ + "p1": "config_level", // 将被步骤级变量覆盖 + "p2": "config_level", // 将被参数覆盖 + }), + TestSteps: []hrp.IStep{ + hrp.NewStep("API请求 - 验证变量覆盖"). + WithVariables(map[string]interface{}{ + "p1": "step_level", // 不会被参数覆盖, 最终值为 "step_level" + "p2": "step_level", // 会被参数覆盖 + "p3": "step_level", // 新增的步骤级变量 + }). + WithParameters(map[string]interface{}{ + "p2-p4": [][]interface{}{ + {"param_level_2", "param_level_4_a"}, + {"param_level_2", "param_level_4_b"}, + }, + }). + GET("https://postman-echo.com/get"). + WithParams(map[string]interface{}{ + "param1": "$p1", // 预期: step_level + "param2": "$p2", // 预期: param_level_2 + "param3": "$p3", // 预期: step_level + "param4": "$p4", // 预期: param_level_4_a/b + }). + Validate(). + AssertEqual("status_code", 200, "check status code"). + AssertEqual("body.args.param1", "step_level", "p1 should be step_level"). + AssertEqual("body.args.param2", "param_level_2", "p2 should be param_level_2"). + AssertEqual("body.args.param3", "step_level", "p3 should be step_level"), + }, + } + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} + +// TestParametersForMobileUI 演示了如何在移动端UI测试中使用参数化来驱动测试。 +func TestParametersForMobileUI(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("小红书UI参数化搜索"). + SetAIOptions(option.WithLLMConfig( + option.NewLLMServiceConfig(option.DOUBAO_1_5_UI_TARS_250328), + )), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动APP"). + Android(). + AppLaunch("com.xingin.xhs"). + Sleep(5). + Validate(). + AssertAppInForeground("com.xingin.xhs"), + hrp.NewStep("UI搜索 - 单参数"). + WithParameters(map[string]interface{}{ + "query": []string{"成都", "北京"}, + }). + Android(). + StartToGoal("进入搜索框，输入「$query」，等待搜索建议出现"). + Sleep(2), + hrp.NewStep("UI搜索 - 复合参数"). + WithParameters(map[string]interface{}{ + "query-category": [][]string{ + {"美食", "食物"}, + {"旅游", "地点"}, + }, + }). + Android(). + StartToGoal("进入搜索框，输入「$query」，并确认其类别为「$category」，等待搜索建议出现"). + Sleep(2), + }, + } + + err := hrp.NewRunner(t).Run(testCase) + assert.Nil(t, err) +} diff --git a/internal/config/config.go b/internal/config/config.go index 1242f6a1..0f04e438 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ const ( SummaryFileName = "hrp_summary.json" // $PWD/results/20060102150405/hrp_summary.json LogFileName = "hrp.log" // $PWD/results/20060102150405/hrp.log ReportFileName = "report.html" // $PWD/results/20060102150405/report.html + CaseFileName = "case.json" // $PWD/results/20060102150405/case.json // mobile device path DeviceActionLogFilePath = "/sdcard/Android/data/io.appium.uiautomator2.server/files/hodor" @@ -36,6 +37,7 @@ type Config struct { summaryFilePath string logFilePath string reportFilePath string + caseFilePath string actionLogDirPath string mu sync.Mutex } @@ -179,3 +181,17 @@ func (c *Config) ReportFilePath() string { c.reportFilePath = filepath.Join(c.resultsPathUnlocked(), ReportFileName) return c.reportFilePath } + +// $PWD/results/20060102150405/case.json +func (c *Config) CaseFilePath() string { + c.mu.Lock() + defer c.mu.Unlock() + + if c.caseFilePath != "" { + return c.caseFilePath + } + + // Ensure directory creation and set cached path + c.caseFilePath = filepath.Join(c.resultsPathUnlocked(), CaseFileName) + return c.caseFilePath +} diff --git a/internal/version/VERSION b/internal/version/VERSION index 61281e5a..54d05ab6 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-beta-2506251113 +v5.0.0-beta-2506251207 diff --git a/parameters.go b/parameters.go index 1de3d21c..c698dd57 100644 --- a/parameters.go +++ b/parameters.go @@ -11,10 +11,51 @@ import ( "github.com/rs/zerolog/log" ) +// ParametersOption defines options for parameters configuration +type ParametersOption func(*TParamsConfig) + +// WithSequentialOrder sets parameters to be picked in sequential order +func WithSequentialOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("sequential") + } +} + +// WithRandomOrder sets parameters to be picked in random order +func WithRandomOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("random") + } +} + +// WithUniqueOrder sets parameters to be picked uniquely (no duplicates) +func WithUniqueOrder() ParametersOption { + return func(config *TParamsConfig) { + config.PickOrder = iteratorPickOrder("unique") + } +} + +// WithLimit sets the limit count for parameter iteration +func WithLimit(limit int) ParametersOption { + return func(config *TParamsConfig) { + config.Limit = limit + } +} + +// WithStrategy sets individual strategy for a specific parameter +func WithStrategy(paramName string, strategy IteratorStrategy) ParametersOption { + return func(config *TParamsConfig) { + if config.Strategies == nil { + config.Strategies = make(map[string]IteratorStrategy) + } + config.Strategies[paramName] = strategy + } +} + type TParamsConfig struct { PickOrder iteratorPickOrder `json:"pick_order,omitempty" yaml:"pick_order,omitempty"` // overall pick-order strategy Strategies map[string]IteratorStrategy `json:"strategies,omitempty" yaml:"strategies,omitempty"` // individual strategies for each parameters - Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` + Limit int `json:"limit,omitempty" yaml:"limit,omitempty"` // limit count for parameter iteration } type iteratorPickOrder string diff --git a/report.go b/report.go index eb27c714..a7aa5fe8 100644 --- a/report.go +++ b/report.go @@ -221,8 +221,9 @@ func (g *HTMLReportGenerator) getStepLogs(stepName string, startTime int64, elap for _, logEntry := range g.LogData { // Check for step boundaries to control inclusion if logEntry.Message == RUN_STEP_START { - if stepFieldValue, exists := logEntry.Fields["step"]; exists { - if stepFieldValue == stepName { + if stepFieldValue, exists := logEntry.Fields["step"].(string); exists { + // use prefix matching for parameterized steps + if strings.HasPrefix(stepName, stepFieldValue) { inCurrentStep = true stepLogs = append(stepLogs, logEntry) continue @@ -234,8 +235,9 @@ func (g *HTMLReportGenerator) getStepLogs(stepName string, startTime int64, elap } if logEntry.Message == RUN_STEP_END { - if stepFieldValue, exists := logEntry.Fields["step"]; exists { - if stepFieldValue == stepName { + if stepFieldValue, exists := logEntry.Fields["step"].(string); exists { + // use prefix matching for parameterized steps + if strings.HasPrefix(stepName, stepFieldValue) { stepLogs = append(stepLogs, logEntry) inCurrentStep = false continue @@ -415,16 +417,53 @@ func (g *HTMLReportGenerator) calculateTotalUsage() map[string]interface{} { continue } for _, action := range step.Actions { - if action.Plannings == nil { - continue - } - for _, planning := range action.Plannings { - if planning.Usage == nil { - continue + // Calculate planning usage + if action.Plannings != nil { + for _, planning := range action.Plannings { + if planning.Usage != nil { + totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + planning.Usage.PromptTokens + totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + planning.Usage.CompletionTokens + totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + planning.Usage.TotalTokens + } + } + } + + // Calculate AI operations usage (ai_query, ai_action, ai_assert) + if action.AIResult != nil { + var usage *map[string]interface{} + + switch action.AIResult.Type { + case "query": + if action.AIResult.QueryResult != nil && action.AIResult.QueryResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.QueryResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.QueryResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.QueryResult.Usage.TotalTokens, + } + } + case "action": + if action.AIResult.PlanningResult != nil && action.AIResult.PlanningResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.PlanningResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.PlanningResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.PlanningResult.Usage.TotalTokens, + } + } + case "assert": + if action.AIResult.AssertionResult != nil && action.AIResult.AssertionResult.Usage != nil { + usage = &map[string]interface{}{ + "prompt_tokens": action.AIResult.AssertionResult.Usage.PromptTokens, + "completion_tokens": action.AIResult.AssertionResult.Usage.CompletionTokens, + "total_tokens": action.AIResult.AssertionResult.Usage.TotalTokens, + } + } + } + + if usage != nil { + totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + (*usage)["prompt_tokens"].(int) + totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + (*usage)["completion_tokens"].(int) + totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + (*usage)["total_tokens"].(int) } - totalUsage["prompt_tokens"] = totalUsage["prompt_tokens"].(int) + planning.Usage.PromptTokens - totalUsage["completion_tokens"] = totalUsage["completion_tokens"].(int) + planning.Usage.CompletionTokens - totalUsage["total_tokens"] = totalUsage["total_tokens"].(int) + planning.Usage.TotalTokens } } } @@ -489,6 +528,12 @@ func (g *HTMLReportGenerator) GenerateReport(outputFile string) error { "add": func(a, b int) int { return a + b }, "base": filepath.Base, "index": func(m map[string]any, key string) any { return m[key] }, + "title": func(s string) string { + if s == "" { + return "" + } + return strings.ToUpper(s[:1]) + s[1:] + }, "extractThought": func(content string) string { if content == "" { return "" @@ -496,7 +541,7 @@ func (g *HTMLReportGenerator) GenerateReport(outputFile string) error { // Try to parse as JSON to extract thought field var data map[string]interface{} if err := json.Unmarshal([]byte(content), &data); err == nil { - if thought, ok := data["thought"].(string); ok { + if thought, ok := data["thought"].(string); ok && thought != "" { return thought } } @@ -655,9 +700,9 @@ const htmlTemplate = ` .summary h2 { color: #2c3e50; - margin-bottom: 20px; - border-bottom: 2px solid #3498db; - padding-bottom: 10px; + margin: 0; + padding: 0; + border: none; } .summary-grid { @@ -667,6 +712,32 @@ const htmlTemplate = ` margin-bottom: 20px; } + .summary-title-bar { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 20px; + border-bottom: 2px solid #3498db; + padding-bottom: 10px; + } + + .toggle-all-btn { + background-color: #ffc107; + color: #212529; + border: none; + padding: 8px 16px; + border-radius: 5px; + cursor: pointer; + font-size: 0.9em; + font-weight: 500; + transition: background-color 0.2s ease; + flex-shrink: 0; + } + + .toggle-all-btn:hover { + background-color: #e0a800; + } + .summary-item { text-align: center; padding: 15px; @@ -1284,6 +1355,21 @@ const htmlTemplate = ` color: #495057; } + .structured-data { + background: #f8f9fa; + border: 1px solid #28a745; + border-radius: 6px; + padding: 10px 12px; + margin: 8px 0; + font-size: 0.85em; + color: #495057; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + white-space: pre-wrap; + word-wrap: break-word; + max-height: 200px; + overflow-y: auto; + } + @media screen and (max-width: 768px) { .planning-three-columns { flex-direction: column; @@ -2118,7 +2204,10 @@ const htmlTemplate = `
-

📊 Test Summary

+
+

📊 Test Summary

+ +
{{.Stat.TestCases.Success}}
@@ -2236,8 +2325,15 @@ const htmlTemplate = ` {{if $planning.Error}}Error: {{$planning.Error}}{{end}}
- {{if $planning.Thought}} -
{{$planning.Thought}}
+ {{$extractedThought := extractThought $planning.Content}} + {{if or $planning.Thought $extractedThought}} +
+ {{if $planning.Thought}} + {{$planning.Thought}} + {{else}} + {{$extractedThought}} + {{end}} +
{{end}} @@ -2348,59 +2444,44 @@ const htmlTemplate = ` {{end}} {{end}} - {{/* Handle special case: ai_query needs enhanced display even when not in planning */}} - {{if eq $action.Method "ai_query"}} + {{/* Enhanced AI Operations Display - using unified AIResult data structure */}} + {{if or (eq $action.Method "ai_query") (eq $action.Method "ai_action") (eq $action.Method "ai_assert")}} + {{if $action.AIResult}}
-
- - {{$stepLogs := getStepLogs $step}} - {{$queryThought := ""}} - {{$queryModel := ""}} - {{$queryUsage := ""}} - {{$queryScreenshot := ""}} - {{$queryResult := ""}} - {{range $logEntry := $stepLogs}} - {{if and (eq $logEntry.Message "log response message") (index $logEntry.Fields "content")}} - {{$content := index $logEntry.Fields "content"}} - {{if $content}} - {{$queryResult = $content}} - {{end}} + + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.Thought}} +
{{$action.AIResult.QueryResult.Thought}}
{{end}} - {{if and (eq $logEntry.Message "call model service for query") (index $logEntry.Fields "model")}} - {{$queryModel = index $logEntry.Fields "model"}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.Thought}} +
{{$action.AIResult.PlanningResult.Thought}}
{{end}} - {{if and (eq $logEntry.Message "usage statistics") (index $logEntry.Fields "input_tokens")}} - {{$inputTokens := index $logEntry.Fields "input_tokens"}} - {{$outputTokens := index $logEntry.Fields "output_tokens"}} - {{$totalTokens := index $logEntry.Fields "total_tokens"}} - {{$queryUsage = printf "📊 Tokens: %v in / %v out / %v total" $inputTokens $outputTokens $totalTokens}} - {{end}} - {{if and (eq $logEntry.Message "log screenshot") (index $logEntry.Fields "imagePath")}} - {{$queryScreenshot = index $logEntry.Fields "imagePath"}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.Thought}} +
{{$action.AIResult.AssertionResult.Thought}}
{{end}} {{end}} - - {{if $queryResult}} -
{{$queryResult}}
- {{end}} - - +
- {{if $queryScreenshot}} + {{if $action.AIResult.ImagePath}}
- 📸 Query Screenshot + 📸 {{title $action.AIResult.Type}} Screenshot + {{if $action.AIResult.ScreenshotElapsed}} + {{formatDuration $action.AIResult.ScreenshotElapsed}} + {{end}}
- {{$base64Image := encodeImageBase64 $queryScreenshot}} + {{$base64Image := encodeImageBase64 $action.AIResult.ImagePath}} {{if $base64Image}}
- Query Screenshot + AI {{title $action.AIResult.Type}} Screenshot
{{end}} @@ -2409,18 +2490,60 @@ const htmlTemplate = `
{{end}} - +
- 🤖 AI Query + 🤖 AI {{title $action.AIResult.Type}} Analysis + {{if $action.AIResult.ModelCallElapsed}} + {{formatDuration $action.AIResult.ModelCallElapsed}} + {{end}}
- {{if $queryModel}} -
🤖 Model: {{$queryModel}}
+ {{/* Model name and usage from specific result types */}} + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.ModelName}} +
🤖 Model: {{$action.AIResult.QueryResult.ModelName}}
+ {{end}} + {{if $action.AIResult.QueryResult.Usage}} +
📊 Tokens: {{$action.AIResult.QueryResult.Usage.PromptTokens}} in / {{$action.AIResult.QueryResult.Usage.CompletionTokens}} out / {{$action.AIResult.QueryResult.Usage.TotalTokens}} total
+ {{end}} + {{/* Display structured data for query results */}} + {{if $action.AIResult.QueryResult.Data}} +
📥 Structured Data:
+
{{safeHTML (toJSON $action.AIResult.QueryResult.Data)}}
+ {{end}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.ModelName}} +
🤖 Model: {{$action.AIResult.PlanningResult.ModelName}}
+ {{end}} + {{if $action.AIResult.PlanningResult.Usage}} +
📊 Tokens: {{$action.AIResult.PlanningResult.Usage.PromptTokens}} in / {{$action.AIResult.PlanningResult.Usage.CompletionTokens}} out / {{$action.AIResult.PlanningResult.Usage.TotalTokens}} total
+ {{end}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.ModelName}} +
🤖 Model: {{$action.AIResult.AssertionResult.ModelName}}
+ {{end}} + {{if $action.AIResult.AssertionResult.Usage}} +
📊 Tokens: {{$action.AIResult.AssertionResult.Usage.PromptTokens}} in / {{$action.AIResult.AssertionResult.Usage.CompletionTokens}} out / {{$action.AIResult.AssertionResult.Usage.TotalTokens}} total
+ {{end}} {{end}} - {{if $queryUsage}} -
{{$queryUsage}}
+ {{if $action.AIResult.Resolution}} +
📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}
+ {{end}} + {{/* Display Content from specific result types */}} + {{if eq $action.AIResult.Type "query"}} + {{if $action.AIResult.QueryResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.QueryResult.Content}}
+ {{end}} + {{else if eq $action.AIResult.Type "action"}} + {{if $action.AIResult.PlanningResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.PlanningResult.Content}}
+ {{end}} + {{else if eq $action.AIResult.Type "assert"}} + {{if $action.AIResult.AssertionResult.Content}} +
💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.AssertionResult.Content}}
+ {{end}} {{end}}
@@ -2429,6 +2552,7 @@ const htmlTemplate = `
{{end}} + {{end}} {{/* Handle SessionData: display requests and screen results for non-planning actions */}} {{if not $action.Plannings}} @@ -2473,9 +2597,6 @@ const htmlTemplate = `
{{base $screenshot.ImagePath}} - {{if $screenshot.Resolution}} - {{$screenshot.Resolution.Width}}x{{$screenshot.Resolution.Height}} - {{end}}
Screenshot @@ -2512,84 +2633,6 @@ const htmlTemplate = ` {{if and $validator.msg (ne $validator.check_result "pass")}}
{{$validator.msg}}
{{end}} - - - {{if or (eq $validator.check "ui_ai") (eq $validator.assert "ai_assert")}} -
- - {{$stepLogs := getStepLogs $step}} - {{$validationThought := ""}} - {{$validationModel := ""}} - {{$validationUsage := ""}} - {{$validationScreenshot := ""}} - {{range $logEntry := $stepLogs}} - {{if and (eq $logEntry.Message "log response message") (index $logEntry.Fields "content")}} - {{$content := index $logEntry.Fields "content"}} - {{if $content}} - {{$validationThought = $content}} - {{end}} - {{end}} - {{if and (eq $logEntry.Message "call model service for assertion") (index $logEntry.Fields "model")}} - {{$validationModel = index $logEntry.Fields "model"}} - {{end}} - {{if and (eq $logEntry.Message "usage statistics") (index $logEntry.Fields "input_tokens")}} - {{$inputTokens := index $logEntry.Fields "input_tokens"}} - {{$outputTokens := index $logEntry.Fields "output_tokens"}} - {{$totalTokens := index $logEntry.Fields "total_tokens"}} - {{$validationUsage = printf "📊 Tokens: %v in / %v out / %v total" $inputTokens $outputTokens $totalTokens}} - {{end}} - {{if and (eq $logEntry.Message "log screenshot") (index $logEntry.Fields "imagePath")}} - {{$validationScreenshot = index $logEntry.Fields "imagePath"}} - {{end}} - {{end}} - - - {{if $validationThought}} -
{{extractThought $validationThought}}
- {{end}} - - -
- - {{if $validationScreenshot}} -
-
-
- 📸 Validation Screenshot -
-
- {{$base64Image := encodeImageBase64 $validationScreenshot}} - {{if $base64Image}} -
-
- Validation Screenshot -
-
- {{end}} -
-
-
- {{end}} - - -
-
-
- 🤖 AI Analysis -
-
- {{if $validationModel}} -
🤖 Model: {{$validationModel}}
- {{end}} - {{if $validationUsage}} -
{{$validationUsage}}
- {{end}} -
-
-
-
-
- {{end}}
{{end}}
@@ -2616,12 +2659,6 @@ const htmlTemplate = `
{{base $imagePath}} - {{if $screenshot.Resolution}} - {{$screenshot.Resolution.Width}}x{{$screenshot.Resolution.Height}} - {{else if index $screenshot "resolution"}} - {{$resolution := index $screenshot "resolution"}} - {{index $resolution "width"}}x{{index $resolution "height"}} - {{end}}
Screenshot @@ -2816,6 +2853,35 @@ const htmlTemplate = ` contents.forEach(content => content.classList.add('show')); icons.forEach(icon => icon.classList.add('rotated')); }); + + function toggleAllSteps() { + const contents = document.querySelectorAll('.step-content'); + const icons = document.querySelectorAll('.toggle-icon'); + const btn = document.getElementById('toggle-all-steps-btn'); + + if (!contents || contents.length === 0) { + return; + } + + // If any step is expanded, collapse all. Otherwise expand all. + let isAnyExpanded = false; + for (let i = 0; i < contents.length; i++) { + if (contents[i].classList.contains('show')) { + isAnyExpanded = true; + break; + } + } + + if (isAnyExpanded) { + contents.forEach(content => content.classList.remove('show')); + icons.forEach(icon => icon.classList.remove('rotated')); + btn.textContent = 'Expand All Steps'; + } else { + contents.forEach(content => content.classList.add('show')); + icons.forEach(icon => icon.classList.add('rotated')); + btn.textContent = 'Collapse All Steps'; + } + } ` diff --git a/runner.go b/runner.go index 34d0d5cb..88ebe740 100644 --- a/runner.go +++ b/runner.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "reflect" + "sort" "strings" "syscall" "testing" @@ -25,6 +26,7 @@ import ( "github.com/httprunner/funplugin" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/builtin" + "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/sdk" "github.com/httprunner/httprunner/v5/internal/version" "github.com/httprunner/httprunner/v5/mcphost" @@ -676,6 +678,13 @@ func (r *SessionRunner) Start(givenVars map[string]interface{}) (summary *TestCa summary.InOut.ExportVars = exportVars summary.InOut.ConfigVars = config.Variables + // Save JSON case content to results directory + if config.Path != "" { + if err := saveJSONCase(config.Path); err != nil { + log.Warn().Err(err).Str("path", config.Path).Msg("save JSON case failed") + } + } + // TODO: move to mobile ui step // Collect logs from cached drivers for _, cached := range uixt.ListCachedDrivers() { @@ -734,6 +743,98 @@ const ( RUN_STEP_END = "run step end" ) +// executionTask holds the necessary information for a single step execution. +type executionTask struct { + stepName string + parameters map[string]interface{} +} + +// formatParameters formats parameter values into a string for display in step names. +// e.g. {"foo": "bar", "age": 18} -> "bar-18" +func formatParameters(params map[string]interface{}) string { + if len(params) == 0 { + return "" + } + + // sort keys to ensure consistent order + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + var values []string + for _, k := range keys { + values = append(values, fmt.Sprintf("%v", params[k])) + } + return strings.Join(values, "-") +} + +// generateExecutionTasks generates a list of execution tasks based on step parameters and loops. +func (r *SessionRunner) generateExecutionTasks(step IStep) ([]executionTask, error) { + stepConfig := step.Config() + stepName := step.Name() + + // determine effective loop times + loopTimes := stepConfig.Loops + if loopTimes <= 0 { + loopTimes = 1 // default to 1 if not set + } + + // initialize parameters iterator + parametersIterator, err := r.caseRunner.parser.InitParametersIterator(&TConfig{ + Parameters: stepConfig.Parameters, + ParametersSetting: stepConfig.ParametersSetting, + Variables: stepConfig.Variables, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize parameters iterator") + } + + // collect all parameter combinations first + var allParameters []map[string]interface{} + if parametersIterator != nil { + for parametersIterator.HasNext() { + allParameters = append(allParameters, parametersIterator.Next()) + } + } + + // if no parameters are specified, but loop times are set, + // we should run the step loopTimes with empty parameters. + if len(allParameters) == 0 && loopTimes > 0 { + allParameters = append(allParameters, make(map[string]interface{})) + } + + // generate execution tasks + var tasks []executionTask + for loopIndex := 1; loopIndex <= loopTimes; loopIndex++ { + for _, params := range allParameters { + // determine step name based on parameters and loops + currentStepName := stepName + hasParameters := len(params) > 0 + hasLoops := loopTimes > 1 + + if hasParameters { + paramStr := formatParameters(params) + if hasLoops { + currentStepName = fmt.Sprintf("%s [loop_%d_params_%s]", stepName, loopIndex, paramStr) + } else { + currentStepName = fmt.Sprintf("%s [params_%s]", stepName, paramStr) + } + } else if hasLoops { + currentStepName = fmt.Sprintf("%s_loop_%d", stepName, loopIndex) + } + + tasks = append(tasks, executionTask{ + stepName: currentStepName, + parameters: params, + }) + } + } + + return tasks, nil +} + func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) { // check for interrupt signal before running step select { @@ -753,56 +854,115 @@ func (r *SessionRunner) RunStep(step IStep) (stepResult *StepResult, err error) stepName := step.Name() stepType := string(step.Type()) + log.Info().Str("step", stepName).Str("type", stepType).Msg(RUN_STEP_START) - // run times of step - loopTimes := step.Config().Loops - if loopTimes == 0 { - loopTimes = 1 // default run once - } else if loopTimes > 1 { - log.Info().Int("loops", loopTimes).Msg("set multiple loop times") + // execute step with parameters iterator + tasks, err := r.generateExecutionTasks(step) + if err != nil { + return nil, errors.Wrap(err, "failed to generate execution tasks") } - // run step with specified loop times - for i := 1; i <= loopTimes; i++ { - var loopIndex string - if loopTimes > 1 { - log.Info().Int("index", i).Msg("start running step in loop") - loopIndex = fmt.Sprintf("_loop_%d", i) + var stepResults []*StepResult + + // defer to save results regardless of how the function exits + defer func() { + // Save all completed results to summary + for _, result := range stepResults { + r.summary.AddStepResult(result) + // update extracted variables from the last result + for k, v := range result.ExportVars { + r.sessionVariables[k] = v + } } - // run step - stepResult, err = step.Run(r) - stepResult.Name = stepName + loopIndex - - // add step result to summary - r.summary.AddStepResult(stepResult) - - // update extracted variables - for k, v := range stepResult.ExportVars { - r.sessionVariables[k] = v - } - - // run step success - if err == nil { + // log final result + if err == nil && stepResult.Success { log.Info().Str("step", stepName). Str("type", stepType). Bool("success", true). Int64("elapsed(ms)", stepResult.Elapsed). Interface("exportVars", stepResult.ExportVars). Msg(RUN_STEP_END) - continue + } else if stepResult != nil { + log.Error().Str("step", stepName). + Str("type", stepType). + Bool("success", false). + Int64("elapsed(ms)", stepResult.Elapsed). + Int("completed_tasks", len(stepResults)). + Int("total_tasks", len(tasks)). + Msg(RUN_STEP_END) + } + }() + + // execute with loops as outer iteration + for _, task := range tasks { + // Check for interrupt signal before each parameter iteration + select { + case <-r.caseRunner.hrpRunner.interruptSignal: + log.Warn().Int("completed_tasks", len(stepResults)). + Int("total_tasks", len(tasks)). + Msg("interrupted during parameter iteration") + return nil, errors.Wrap(code.InterruptError, "parameter iteration interrupted") + default: + } + + // execute step with merged variables + stepResult, stepErr := r.executeStepWithVariables(step, task.stepName, task.parameters) + + // Always add stepResult to stepResults if it exists, even on error + // This ensures data is saved in defer function for summary generation + if stepResult != nil { + stepResults = append(stepResults, stepResult) + } + + if stepErr != nil { + if r.caseRunner.hrpRunner.failfast { + // failfast mode, abort running but step result is already saved above + log.Error().Err(stepErr). + Str("step", task.stepName). + Int("completed_tasks", len(stepResults)). + Msg("execute step failed in failfast mode, step result saved") + return nil, errors.Wrap(stepErr, "execute step failed") + } + log.Error().Err(stepErr).Str("step", task.stepName).Msg("execute step failed") } - // run step failed - log.Error().Err(err).Str("step", stepName). - Str("type", stepType). - Bool("success", false). - Int64("elapsed(ms)", stepResult.Elapsed). - Msg(RUN_STEP_END) - return stepResult, err } - return stepResult, nil + // return last result + if len(stepResults) > 0 { + lastResult := stepResults[len(stepResults)-1] + return lastResult, nil + } + + return nil, errors.New("no steps were executed") +} + +// executeStepWithVariables executes a single step with given parameters +// parameters will override step variables with the same name +func (r *SessionRunner) executeStepWithVariables(step IStep, stepName string, parameters map[string]interface{}) (stepResult *StepResult, err error) { + stepConfig := step.Config() + + // backup original variables + originalVariables := make(map[string]interface{}) + for k, v := range stepConfig.Variables { + originalVariables[k] = v + } + + // merge parameters into step variables + // parameters have higher priority than variables + for k, v := range parameters { + stepConfig.Variables[k] = v + } + + // execute step + stepResult, err = step.Run(r) + stepResult.Name = stepName + + // restore original variables to avoid side effects + stepConfig.Variables = originalVariables + + return stepResult, err } func (r *SessionRunner) GetSummary() *TestCaseSummary { @@ -895,3 +1055,25 @@ func (r *SessionRunner) GetSessionVariables() map[string]interface{} { func (r *SessionRunner) GetTransactions() map[string]map[TransactionType]time.Time { return r.transactions } + +// saveJSONCase saves the original JSON case content to the results directory +func saveJSONCase(casePath string) error { + // Read the original JSON case content + path := TestCasePath(casePath) + testCase, err := path.GetTestCase() + if err != nil { + return errors.Wrap(err, "load JSON case failed") + } + + // remove environs from testcase config + tConfig := testCase.Config.(*TConfig) + tConfig.Environs = nil + + // save JSON case to results directory + jsonCasePath := config.GetConfig().CaseFilePath() + err = testCase.Dump2JSON(jsonCasePath) + if err != nil { + return errors.Wrap(err, "dump JSON case failed") + } + return nil +} diff --git a/step.go b/step.go index 3b2c866f..c82be894 100644 --- a/step.go +++ b/step.go @@ -27,15 +27,17 @@ const ( ) type StepConfig struct { - StepName string `json:"name" yaml:"name"` // required - Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` - SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` - TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` - Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` - Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` - StepExport []string `json:"export,omitempty" yaml:"export,omitempty"` - Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` - AutoPopupHandler bool `json:"auto_popup_handler,omitempty" yaml:"auto_popup_handler,omitempty"` // enable auto popup handler for this step + StepName string `json:"name" yaml:"name"` // required + Variables map[string]interface{} `json:"variables,omitempty" yaml:"variables,omitempty"` + Parameters map[string]interface{} `json:"parameters,omitempty" yaml:"parameters,omitempty"` + ParametersSetting *TParamsConfig `json:"parameters_setting,omitempty" yaml:"parameters_setting,omitempty"` + SetupHooks []string `json:"setup_hooks,omitempty" yaml:"setup_hooks,omitempty"` + TeardownHooks []string `json:"teardown_hooks,omitempty" yaml:"teardown_hooks,omitempty"` + Extract map[string]string `json:"extract,omitempty" yaml:"extract,omitempty"` + Validators []interface{} `json:"validate,omitempty" yaml:"validate,omitempty"` + StepExport []string `json:"export,omitempty" yaml:"export,omitempty"` + Loops int `json:"loops,omitempty" yaml:"loops,omitempty"` + AutoPopupHandler bool `json:"auto_popup_handler,omitempty" yaml:"auto_popup_handler,omitempty"` // enable auto popup handler for this step } // define struct for teststep @@ -62,6 +64,7 @@ type ActionResult struct { Elapsed int64 `json:"elapsed_ms"` // action elapsed time(ms) Error string `json:"error,omitempty"` // action execution result Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions + AIResult *uixt.AIExecutionResult `json:"ai_result,omitempty"` // store unified AI execution result for ai_query/ai_action/ai_assert actions uixt.SessionData // store session data for other actions besides start_to_goal } diff --git a/step_request.go b/step_request.go index 25b88666..8f9bc23f 100644 --- a/step_request.go +++ b/step_request.go @@ -543,6 +543,26 @@ func (s *StepRequest) WithVariables(variables map[string]interface{}) *StepReque return s } +// WithParameters sets parameters for step-level data driven +func (s *StepRequest) WithParameters(parameters map[string]interface{}) *StepRequest { + s.Parameters = parameters + return s +} + +// WithParametersSetting sets parameters setting for step-level data driven +func (s *StepRequest) WithParametersSetting(options ...ParametersOption) *StepRequest { + if s.ParametersSetting == nil { + s.ParametersSetting = &TParamsConfig{} + } + + // apply all options + for _, option := range options { + option(s.ParametersSetting) + } + + return s +} + // SetupHook adds a setup hook for current teststep. func (s *StepRequest) SetupHook(hook string) *StepRequest { s.SetupHooks = append(s.SetupHooks, hook) @@ -557,7 +577,7 @@ func (s *StepRequest) HTTP2() *StepRequest { return s } -// Loop specify running times for the current step +// Loop sets loop count for step execution. func (s *StepRequest) Loop(times int) *StepRequest { s.Loops = times return s diff --git a/step_ui.go b/step_ui.go index 426621cb..9971ca08 100644 --- a/step_ui.go +++ b/step_ui.go @@ -935,7 +935,7 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err } }() - // handle start_to_goal action + // handle start_to_goal AI action if action.Method == option.ACTION_StartToGoal { planningResults, err := uiDriver.StartToGoal(ctx, action.Params.(string), action.GetOptions()...) @@ -943,6 +943,7 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err actionResult.Plannings = planningResults stepResult.Actions = append(stepResult.Actions, actionResult) if err != nil { + actionResult.Error = err.Error() if !code.IsErrorPredefined(err) { err = errors.Wrap(code.MobileUIDriverError, err.Error()) } @@ -951,7 +952,35 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err continue } - // handle other actions + // handle AI operations (ai_action, ai_query, ai_assert) with unified result storage + if action.Method == option.ACTION_AIAction || action.Method == option.ACTION_Query || action.Method == option.ACTION_AIAssert { + var aiResult *uixt.AIExecutionResult + var err error + + prompt := action.Params.(string) + switch action.Method { + case option.ACTION_AIAction: + aiResult, err = uiDriver.AIAction(ctx, prompt, action.GetOptions()...) + case option.ACTION_Query: + aiResult, err = uiDriver.AIQuery(prompt, action.GetOptions()...) + case option.ACTION_AIAssert: + aiResult, err = uiDriver.AIAssert(prompt, action.GetOptions()...) + } + + actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() + actionResult.AIResult = aiResult + stepResult.Actions = append(stepResult.Actions, actionResult) + if err != nil { + actionResult.Error = err.Error() + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.MobileUIDriverError, err.Error()) + } + return stepResult, err + } + continue + } + + // handle other non-AI actions sessionData, err := uiDriver.ExecuteAction(ctx, action) actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() actionResult.SessionData = sessionData diff --git a/uixt/ai/asserter.go b/uixt/ai/asserter.go index 58880c51..2760f6ff 100644 --- a/uixt/ai/asserter.go +++ b/uixt/ai/asserter.go @@ -30,8 +30,10 @@ type AssertOptions struct { // AssertionResult represents the response from an AI assertion type AssertionResult struct { - Pass bool `json:"pass"` - Thought string `json:"thought"` + Pass bool `json:"pass"` + Thought string `json:"thought"` + ModelName string `json:"model_name"` // model name used for assertion + Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics } // Asserter handles assertion using different AI models @@ -85,7 +87,7 @@ func NewAsserter(ctx context.Context, modelConfig *ModelConfig) (*Asserter, erro } // Assert performs the assertion check on the screenshot -func (a *Asserter) Assert(ctx context.Context, opts *AssertOptions) (*AssertionResult, error) { +func (a *Asserter) Assert(ctx context.Context, opts *AssertOptions) (result *AssertionResult, err error) { // Validate input parameters if err := validateAssertionInput(opts); err != nil { return nil, errors.Wrap(err, "validate assertion parameters failed") @@ -132,8 +134,15 @@ Here is the assertion. Please tell whether it is truthy according to the screens return nil, errors.Wrap(code.LLMRequestServiceError, err.Error()) } + defer func() { + // Extract usage information if available + if message.ResponseMeta != nil && message.ResponseMeta.Usage != nil { + result.Usage = message.ResponseMeta.Usage + } + }() + // Parse result - result, err := parseAssertionResult(message.Content) + result, err = parseAssertionResult(message.Content, a.modelConfig.ModelType) if err != nil { return nil, errors.Wrap(code.LLMParseAssertionResponseError, err.Error()) } @@ -159,7 +168,7 @@ func validateAssertionInput(opts *AssertOptions) error { } // parseAssertionResult parses the model response into AssertionResponse -func parseAssertionResult(content string) (*AssertionResult, error) { +func parseAssertionResult(content string, modelType option.LLMServiceType) (*AssertionResult, error) { var result AssertionResult // Use the generic structured response parser @@ -170,5 +179,6 @@ func parseAssertionResult(content string) (*AssertionResult, error) { return nil, errors.Wrap(code.LLMParseAssertionResponseError, err.Error()) } + result.ModelName = string(modelType) return &result, nil } diff --git a/uixt/ai/asserter_test.go b/uixt/ai/asserter_test.go index d9bdaaba..18fd3cdd 100644 --- a/uixt/ai/asserter_test.go +++ b/uixt/ai/asserter_test.go @@ -136,7 +136,7 @@ func TestParseAssertionResult(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := parseAssertionResult(tt.input) + result, err := parseAssertionResult(tt.input, option.DOUBAO_1_5_UI_TARS_250328) if tt.shouldSucceed { require.NoError(t, err) assert.NotNil(t, result) diff --git a/uixt/ai/querier.go b/uixt/ai/querier.go index 75d5dbc3..54194fad 100644 --- a/uixt/ai/querier.go +++ b/uixt/ai/querier.go @@ -32,9 +32,11 @@ type QueryOptions struct { // QueryResult represents the response from an AI query type QueryResult struct { - Content string `json:"content"` // The extracted content/information - Thought string `json:"thought"` // The reasoning process - Data interface{} `json:"data,omitempty"` // Structured data when OutputSchema is provided + Content string `json:"content"` // The extracted content/information + Thought string `json:"thought"` // The reasoning process + Data interface{} `json:"data,omitempty"` // Structured data when OutputSchema is provided + ModelName string `json:"model_name"` // model name used for query + Usage *schema.TokenUsage `json:"usage,omitempty"` // token usage statistics } // Querier handles query operations using different AI models @@ -89,7 +91,7 @@ func NewQuerier(ctx context.Context, modelConfig *ModelConfig) (*Querier, error) // callModelWithLogging calls the model with automatic logging and timing // Query performs the information extraction from the screenshot -func (q *Querier) Query(ctx context.Context, opts *QueryOptions) (*QueryResult, error) { +func (q *Querier) Query(ctx context.Context, opts *QueryOptions) (result *QueryResult, err error) { // Validate input parameters if err := validateQueryInput(opts); err != nil { return nil, errors.Wrap(err, "validate query parameters failed") @@ -141,8 +143,15 @@ Here is the query. Please extract the requested information from the screenshot. return nil, errors.Wrap(code.LLMRequestServiceError, err.Error()) } + defer func() { + // Extract usage information if available + if message.ResponseMeta != nil && message.ResponseMeta.Usage != nil { + result.Usage = message.ResponseMeta.Usage + } + }() + // Parse result - result, err := parseQueryResult(message.Content) + result, err = parseQueryResult(message.Content, q.modelConfig.ModelType) if err != nil { return nil, errors.Wrap(code.LLMParseQueryResponseError, err.Error()) } @@ -168,18 +177,20 @@ func validateQueryInput(opts *QueryOptions) error { } // parseQueryResult parses the model response into QueryResult -func parseQueryResult(content string) (*QueryResult, error) { +func parseQueryResult(content string, modelType option.LLMServiceType) (*QueryResult, error) { var result QueryResult // Use the generic structured response parser with enhanced error recovery if err := parseStructuredResponse(content, &result); err != nil { // If parseStructuredResponse fails completely, treat content as plain text return &QueryResult{ - Content: content, - Thought: "Failed to parse response, returning raw content", + Content: content, + Thought: "Failed to parse response, returning raw content", + ModelName: string(modelType), }, nil } + result.ModelName = string(modelType) return &result, nil } diff --git a/uixt/ai/querier_test.go b/uixt/ai/querier_test.go index 3793f7bf..99a0fed1 100644 --- a/uixt/ai/querier_test.go +++ b/uixt/ai/querier_test.go @@ -87,7 +87,6 @@ func loadTestImage(t *testing.T, path string) (string, types.Size) { } // Test functions - func TestParseQueryResult(t *testing.T) { tests := []struct { name string @@ -130,7 +129,7 @@ func TestParseQueryResult(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := parseQueryResult(tt.content) + result, err := parseQueryResult(tt.content, option.DOUBAO_1_5_UI_TARS_250328) assert.NoError(t, err) assert.Equal(t, tt.expected.Content, result.Content) assert.Equal(t, tt.expected.Thought, result.Thought) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index e883078e..55b6a7a1 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -339,15 +339,11 @@ func (ad *ADBDriver) DoubleTap(x, y float64, opts ...option.ActionOption) error xStr := fmt.Sprintf("%.1f", x) yStr := fmt.Sprintf("%.1f", y) _, err = ad.runShellCommand( + "input", "tap", xStr, yStr, ";", + "sleep", "0.05", ";", "input", "tap", xStr, yStr) if err != nil { - return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) - } - time.Sleep(time.Duration(100) * time.Millisecond) - _, err = ad.runShellCommand( - "input", "tap", xStr, yStr) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("tap <%s, %s> failed", xStr, yStr)) + return errors.Wrap(err, fmt.Sprintf("double tap <%s, %s> failed", xStr, yStr)) } return nil } diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 773e4c1c..c2d7042e 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -30,7 +30,10 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Check for context cancellation (interrupt signal) select { case <-ctx.Done(): - log.Warn().Msg("interrupted in StartToGoal") + log.Warn(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Msg("interrupted in StartToGoal") return allPlannings, errors.Wrap(code.InterruptError, "StartToGoal interrupted") default: } @@ -83,7 +86,12 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Check for context cancellation before each action select { case <-ctx.Done(): - log.Warn().Msg("interrupted in invokeToolCalls") + log.Warn(). + Int("attempt", attempt). + Int("completed_plannings", len(allPlannings)). + Int("completed_tool_calls", len(planningResult.SubActions)). + Int("total_tool_calls", len(planningResult.ToolCalls)). + Msg("interrupted in invokeToolCalls") planningResult.Elapsed = time.Since(planningStartTime).Milliseconds() allPlannings = append(allPlannings, planningResult) return allPlannings, errors.Wrap(code.InterruptError, "invokeToolCalls interrupted") @@ -131,24 +139,48 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op } } -func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) error { +// AIAction performs AI-driven action and returns detailed execution result +func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...option.ActionOption) (*AIExecutionResult, error) { log.Info().Str("prompt", prompt).Msg("performing AI action") - // plan next action - planningResult, err := dExt.PlanNextAction(ctx, prompt, opts...) + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() if err != nil { - return err + return nil, err } - // Invoke tool calls + // Step 2: Plan next action and measure time + modelCallStartTime := time.Now() + planningResult, err := dExt.PlanNextAction(ctx, prompt, opts...) + modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() + aiExecutionResult := &AIExecutionResult{ + Type: "action", + ModelCallElapsed: modelCallElapsed, + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, + PlanningResult: &planningResult.PlanningResult, + } + + if err != nil { + aiExecutionResult.Error = err.Error() + return aiExecutionResult, errors.Wrap(err, "get next action failed") + } + + // Step 3: Execute tool calls for _, toolCall := range planningResult.ToolCalls { err = dExt.invokeToolCall(ctx, toolCall) if err != nil { - return err + aiExecutionResult.Error = err.Error() + return aiExecutionResult, errors.Wrap(err, "invoke tool call failed") } } - return nil + return aiExecutionResult, nil } // PlanNextAction performs planning and returns unified planning information @@ -301,6 +333,23 @@ type PlanningExecutionResult struct { SubActions []*SubActionResult `json:"sub_actions,omitempty"` // sub-actions generated from this planning } +// AIExecutionResult represents a unified result structure for all AI operations +type AIExecutionResult struct { + Type string `json:"type"` // operation type: "query", "action", "assert" + ModelCallElapsed int64 `json:"model_call_elapsed"` // model call elapsed time in milliseconds + ScreenshotElapsed int64 `json:"screenshot_elapsed"` // screenshot elapsed time in milliseconds + ImagePath string `json:"image_path"` // path to screenshot used for operation + Resolution *types.Size `json:"resolution"` // screen resolution + + // Operation-specific results (only one will be populated based on Type) + QueryResult *ai.QueryResult `json:"query_result,omitempty"` // for ai_query operations + PlanningResult *ai.PlanningResult `json:"planning_result,omitempty"` // for ai_action operations + AssertionResult *ai.AssertionResult `json:"assertion_result,omitempty"` // for ai_assert operations + + // Common fields + Error string `json:"error,omitempty"` // error message if operation failed +} + // SubActionResult represents a sub-action within a start_to_goal action type SubActionResult struct { ActionName string `json:"action_name"` // name of the sub-action (e.g., "tap", "input") @@ -316,11 +365,21 @@ type SessionData struct { ScreenResults []*ScreenResult `json:"screen_results,omitempty"` // store sub-action specific screen_results } -func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.QueryResult, error) { +func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*AIExecutionResult, error) { if dExt.LLMService == nil { return nil, errors.New("LLM service is not initialized") } + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() + if err != nil { + return nil, err + } + screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize() if err != nil { return nil, err @@ -329,6 +388,9 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.Que // parse action options to extract OutputSchema actionOptions := option.NewActionOptions(opts...) + // Step 2: Call model and measure time + modelCallStartTime := time.Now() + // execute query queryOpts := &ai.QueryOptions{ Query: text, @@ -337,37 +399,72 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*ai.Que OutputSchema: actionOptions.OutputSchema, } result, err := dExt.LLMService.Query(context.Background(), queryOpts) + modelCallElapsed := time.Since(modelCallStartTime).Milliseconds() if err != nil { return nil, errors.Wrap(err, "AI query failed") } - return result, nil + // Create AIExecutionResult with all timing and metadata + aiResult := &AIExecutionResult{ + Type: "query", + ModelCallElapsed: modelCallElapsed, // model call timing + ScreenshotElapsed: screenshotElapsed, // screenshot timing + ImagePath: screenResult.ImagePath, // screenshot path + Resolution: &screenResult.Resolution, // screen resolution + QueryResult: result, // query-specific result + } + return aiResult, nil } -func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) error { +// AIAssert performs AI-driven assertion and returns detailed execution result +func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*AIExecutionResult, error) { if dExt.LLMService == nil { - return errors.New("LLM service is not initialized") + return nil, errors.New("LLM service is not initialized") + } + + // Step 1: Take screenshot and measure time + screenshotStartTime := time.Now() + screenResult, err := dExt.createScreenshotWithSession( + option.WithScreenShotFileName(builtin.GenNameWithTimestamp("%d_screenshot")), + ) + screenshotElapsed := time.Since(screenshotStartTime).Milliseconds() + if err != nil { + return nil, err + } + + assertResult := &AIExecutionResult{ + Type: "assert", + ScreenshotElapsed: screenshotElapsed, + ImagePath: screenResult.ImagePath, + Resolution: &screenResult.Resolution, } screenShotBase64, size, err := dExt.GetScreenshotBase64WithSize() if err != nil { - return err + assertResult.Error = err.Error() + return assertResult, err } - // execute assertion + // Step 2: Call model and measure time + modelCallStartTime := time.Now() assertOpts := &ai.AssertOptions{ Assertion: assertion, Screenshot: screenShotBase64, Size: size, } result, err := dExt.LLMService.Assert(context.Background(), assertOpts) + assertResult.ModelCallElapsed = time.Since(modelCallStartTime).Milliseconds() + assertResult.AssertionResult = result + if err != nil { - return errors.Wrap(err, "AI assertion failed") + assertResult.Error = err.Error() + return assertResult, errors.Wrap(err, "AI assertion failed") } if !result.Pass { - return errors.New(result.Thought) + assertResult.Error = result.Thought + return assertResult, errors.New(result.Thought) } - return nil + return assertResult, nil } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 5db77431..3167c6cf 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -15,10 +15,10 @@ import ( func TestDriverExt_TapByLLM(t *testing.T) { driver := setupDriverExt(t) - err := driver.AIAction(context.Background(), "点击第一个帖子的作者头像") + _, err := driver.AIAction(context.Background(), "点击第一个帖子的作者头像") assert.Nil(t, err) - err = driver.AIAssert("当前在个人介绍页") + _, err = driver.AIAssert("当前在个人介绍页") assert.Nil(t, err) } diff --git a/uixt/driver_utils.go b/uixt/driver_utils.go index 787f6f9b..c24abcc6 100644 --- a/uixt/driver_utils.go +++ b/uixt/driver_utils.go @@ -195,7 +195,7 @@ func (dExt *XTDriver) DoValidation(check, assert, expected string, message ...st case option.SelectorOCR: err = dExt.assertOCR(expected, assert) case option.SelectorAI: - err = dExt.AIAssert(expected) + _, err = dExt.AIAssert(expected) case option.SelectorForegroundApp: err = dExt.assertForegroundApp(expected, assert) case option.SelectorSelector: diff --git a/uixt/mcp_tools_ai.go b/uixt/mcp_tools_ai.go index 26168732..c932f43f 100644 --- a/uixt/mcp_tools_ai.go +++ b/uixt/mcp_tools_ai.go @@ -102,7 +102,7 @@ func (t *ToolAIAction) Implement() server.ToolHandlerFunc { } // AI action logic - err = driverExt.AIAction(ctx, unifiedReq.Prompt) + _, err = driverExt.AIAction(ctx, unifiedReq.Prompt) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("AI action failed: %s", err.Error())), err } @@ -166,7 +166,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc { opts := unifiedReq.Options() // AI query logic with options - result, err := driverExt.AIQuery(unifiedReq.Prompt, opts...) + queryResult, err := driverExt.AIQuery(unifiedReq.Prompt, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("AI query failed: %s", err.Error())), err } @@ -174,7 +174,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc { message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt) returnData := ToolAIQuery{ Prompt: unifiedReq.Prompt, - Result: result.Content, + Result: queryResult.QueryResult.Content, } return NewMCPSuccessResponse(message, &returnData), nil