mirror of
https://github.com/httprunner/httprunner.git
synced 2026-09-07 00:17:37 +08:00
feat: 支持wings断言和自动化
This commit is contained in:
余泓铮
+267
-28
@@ -7,10 +7,11 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/uixt/ai"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/httprunner/httprunner/v5/uixt/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDriverExt_TapByLLM(t *testing.T) {
|
||||
@@ -22,33 +23,33 @@ func TestDriverExt_TapByLLM(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestDriverExt_StartToGoal(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
|
||||
1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
|
||||
2. 连接规则:
|
||||
- 两个相同的图案可以通过不超过三条直线连接。
|
||||
- 连接线可以水平或垂直，但不能斜线，也不能跨过其他图案。
|
||||
- 连接线的转折次数不能超过两次。
|
||||
3. 游戏界面:
|
||||
- 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
|
||||
- 图案或图标在未选中状态下背景为白色，选中状态下背景为绿色。
|
||||
4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
|
||||
5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
|
||||
6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。
|
||||
|
||||
注意事项：
|
||||
1、当连接错误时，顶部的红心会减少一个，需及时调整策略，避免红心变为0个后游戏失败
|
||||
2、不要连续 2 次点击同一个图案
|
||||
3、不要犯重复的错误
|
||||
`
|
||||
|
||||
userInstruction += "\n\n请严格按照以上游戏规则，开始游戏；注意，请只做点击操作"
|
||||
|
||||
_, err := driver.StartToGoal(context.Background(), userInstruction)
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
//func TestDriverExt_StartToGoal(t *testing.T) {
|
||||
// driver := setupDriverExt(t)
|
||||
//
|
||||
// userInstruction := `连连看是一款经典的益智消除类小游戏，通常以图案或图标为主要元素。以下是连连看的基本规则说明：
|
||||
// 1. 游戏目标: 玩家需要在规定时间内，通过连接相同的图案或图标，将它们从游戏界面中消除。
|
||||
// 2. 连接规则:
|
||||
// - 两个相同的图案可以通过不超过三条直线连接。
|
||||
// - 连接线可以水平或垂直，但不能斜线，也不能跨过其他图案。
|
||||
// - 连接线的转折次数不能超过两次。
|
||||
// 3. 游戏界面:
|
||||
// - 游戏界面通常是一个矩形区域，内含多个图案或图标，排列成行和列。
|
||||
// - 图案或图标在未选中状态下背景为白色，选中状态下背景为绿色。
|
||||
// 4. 时间限制: 游戏通常设有时间限制，玩家需要在时间耗尽前完成所有图案的消除。
|
||||
// 5. 得分机制: 每成功连接并消除一对图案，玩家会获得相应的分数。完成游戏后，根据剩余时间和消除效率计算总分。
|
||||
// 6. 关卡设计: 游戏可能包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。
|
||||
//
|
||||
// 注意事项：
|
||||
// 1、当连接错误时，顶部的红心会减少一个，需及时调整策略，避免红心变为0个后游戏失败
|
||||
// 2、不要连续 2 次点击同一个图案
|
||||
// 3、不要犯重复的错误
|
||||
// `
|
||||
//
|
||||
// userInstruction += "\n\n请严格按照以上游戏规则，开始游戏；注意，请只做点击操作"
|
||||
//
|
||||
// //_, err := driver.StartToGoal(context.Background(), userInstruction)
|
||||
// //assert.Nil(t, err)
|
||||
//}
|
||||
|
||||
func TestDriverExt_PlanNextAction(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
@@ -244,3 +245,241 @@ func TestPlanningOptions_ResetHistory(t *testing.T) {
|
||||
assert.True(t, opts.ResetHistory)
|
||||
assert.Equal(t, "test instruction", opts.UserInstruction)
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAction tests the AIAction method integration with real driver
|
||||
func TestDriverExt_AIAction(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
// Test AIAction with search button click prompt
|
||||
result, err := driver.AIAction(context.Background(), "冷启动抖音app")
|
||||
|
||||
// Verify no error occurred
|
||||
assert.Nil(t, err, "AIAction should execute without error")
|
||||
|
||||
// Verify result is not nil
|
||||
assert.NotNil(t, result, "AIAction should return a result")
|
||||
|
||||
// Verify result has correct type
|
||||
assert.Equal(t, "action", result.Type, "Result type should be 'action'")
|
||||
|
||||
// Verify timing information is captured
|
||||
assert.Greater(t, result.ModelCallElapsed, int64(0), "Model call should have elapsed time")
|
||||
assert.Greater(t, result.ScreenshotElapsed, int64(0), "Screenshot should have elapsed time")
|
||||
|
||||
// Verify screenshot information is captured
|
||||
assert.NotEmpty(t, result.ImagePath, "Image path should not be empty")
|
||||
assert.NotNil(t, result.Resolution, "Resolution should not be nil")
|
||||
assert.Greater(t, result.Resolution.Width, 0, "Width should be greater than 0")
|
||||
assert.Greater(t, result.Resolution.Height, 0, "Height should be greater than 0")
|
||||
|
||||
// Verify planning result is captured
|
||||
assert.NotNil(t, result.PlanningResult, "Planning result should not be nil")
|
||||
assert.Equal(t, "wings-api", result.PlanningResult.ModelName, "Model name should be 'wings-api'")
|
||||
// Log result for debugging
|
||||
t.Logf("AIAction executed successfully:")
|
||||
t.Logf(" Type: %s", result.Type)
|
||||
t.Logf(" Model Call Elapsed: %d ms", result.ModelCallElapsed)
|
||||
t.Logf(" Screenshot Elapsed: %d ms", result.ScreenshotElapsed)
|
||||
t.Logf(" Image Path: %s", result.ImagePath)
|
||||
t.Logf(" Resolution: %dx%d", result.Resolution.Width, result.Resolution.Height)
|
||||
|
||||
if result.Error != "" {
|
||||
t.Logf(" Error: %s", result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAction_CompareWithAIAction compares AIAction with AIAction
|
||||
func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
prompt := "点击搜索按钮"
|
||||
|
||||
// Test both methods with the same prompt
|
||||
wingsResult, wingsErr := driver.AIAction(context.Background(), prompt)
|
||||
aiResult, aiErr := driver.AIAction(context.Background(), prompt)
|
||||
|
||||
// Both should execute without critical errors (may have different implementations)
|
||||
t.Logf("AIAction error: %v", wingsErr)
|
||||
t.Logf("AIAction error: %v", aiErr)
|
||||
|
||||
// If both succeed, compare results
|
||||
if wingsResult != nil && aiResult != nil {
|
||||
assert.Equal(t, "action", wingsResult.Type, "AIAction result type should be 'action'")
|
||||
assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'")
|
||||
|
||||
// Both should have timing information
|
||||
assert.Greater(t, wingsResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time")
|
||||
assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time")
|
||||
|
||||
// Both should have screenshot information
|
||||
assert.NotEmpty(t, wingsResult.ImagePath, "AIAction should have image path")
|
||||
assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path")
|
||||
|
||||
// Compare model names
|
||||
if wingsResult.PlanningResult != nil && aiResult.PlanningResult != nil {
|
||||
t.Logf("AIAction model: %s", wingsResult.PlanningResult.ModelName)
|
||||
t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName)
|
||||
|
||||
assert.Equal(t, "wings-api", wingsResult.PlanningResult.ModelName, "AIAction should use wings-api")
|
||||
assert.NotEqual(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should not use wings-api")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAction_ErrorHandling tests AIAction error handling
|
||||
func TestDriverExt_AIAction_ErrorHandling(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
// Test with empty prompt
|
||||
result, err := driver.AIAction(context.Background(), "")
|
||||
|
||||
// Should handle empty prompt gracefully
|
||||
if err != nil {
|
||||
t.Logf("Empty prompt error (expected): %v", err)
|
||||
assert.NotNil(t, result, "Result should still be returned even on error")
|
||||
if result != nil {
|
||||
assert.NotEmpty(t, result.Error, "Result should contain error message")
|
||||
}
|
||||
} else {
|
||||
t.Logf("Empty prompt handled successfully")
|
||||
assert.NotNil(t, result, "Result should be returned")
|
||||
}
|
||||
|
||||
// Test with very long prompt
|
||||
longPrompt := "这是一个非常长的提示词，用来测试AIAction是否能够正确处理长文本输入。" +
|
||||
"我们需要确保API能够处理各种长度的输入，包括这种可能超出某些限制的文本。" +
|
||||
"请在当前界面中寻找任何可能的搜索相关的按钮或输入框，然后进行点击操作。"
|
||||
|
||||
result2, err2 := driver.AIAction(context.Background(), longPrompt)
|
||||
|
||||
// Should handle long prompt
|
||||
if err2 != nil {
|
||||
t.Logf("Long prompt error: %v", err2)
|
||||
} else {
|
||||
t.Logf("Long prompt handled successfully")
|
||||
assert.NotNil(t, result2, "Result should be returned for long prompt")
|
||||
assert.Equal(t, "action", result2.Type, "Result type should be 'action'")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAssert tests the AIAssert method integration with real driver
|
||||
func TestDriverExt_AIAssert(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
// Test AIAssert with assertion about search button
|
||||
result, err := driver.AIAssert("屏幕中存在搜索按钮")
|
||||
|
||||
// Verify no error occurred (or error is captured in result)
|
||||
if err != nil {
|
||||
t.Logf("AIAssert error: %v", err)
|
||||
// For assertion failures, error is expected, but result should still be returned
|
||||
assert.NotNil(t, result, "AIAssert should return a result even on assertion failure")
|
||||
} else {
|
||||
assert.NotNil(t, result, "AIAssert should return a result")
|
||||
}
|
||||
|
||||
// Verify result has correct type
|
||||
assert.Equal(t, "assert", result.Type, "Result type should be 'assert'")
|
||||
|
||||
// Verify timing information is captured
|
||||
assert.Greater(t, result.ModelCallElapsed, int64(0), "Model call should have elapsed time")
|
||||
assert.Greater(t, result.ScreenshotElapsed, int64(0), "Screenshot should have elapsed time")
|
||||
|
||||
// Verify screenshot information is captured
|
||||
assert.NotEmpty(t, result.ImagePath, "Image path should not be empty")
|
||||
assert.NotNil(t, result.Resolution, "Resolution should not be nil")
|
||||
assert.Greater(t, result.Resolution.Width, 0, "Width should be greater than 0")
|
||||
assert.Greater(t, result.Resolution.Height, 0, "Height should be greater than 0")
|
||||
|
||||
// Verify assertion result is captured
|
||||
assert.NotNil(t, result.AssertionResult, "Assertion result should not be nil")
|
||||
assert.NotEmpty(t, result.AssertionResult.Thought, "Assertion result thought should not be empty")
|
||||
|
||||
// Log result for debugging
|
||||
t.Logf("AIAssert executed:")
|
||||
t.Logf(" Type: %s", result.Type)
|
||||
t.Logf(" Model Call Elapsed: %d ms", result.ModelCallElapsed)
|
||||
t.Logf(" Screenshot Elapsed: %d ms", result.ScreenshotElapsed)
|
||||
t.Logf(" Image Path: %s", result.ImagePath)
|
||||
t.Logf(" Resolution: %dx%d", result.Resolution.Width, result.Resolution.Height)
|
||||
t.Logf(" Assertion Pass: %t", result.AssertionResult.Pass)
|
||||
t.Logf(" Assertion Thought: %s", result.AssertionResult.Thought)
|
||||
|
||||
if result.Error != "" {
|
||||
t.Logf(" Error: %s", result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAssert_CompareWithAIAssert compares AIAssert with AIAssert
|
||||
func TestDriverExt_AIAssert_CompareWithAIAssert(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
assertion := "屏幕中存在搜索按钮"
|
||||
|
||||
// Test both methods with the same assertion
|
||||
wingsResult, wingsErr := driver.AIAssert(assertion)
|
||||
aiResult, aiErr := driver.AIAssert(assertion)
|
||||
|
||||
// Both should execute (may have different results)
|
||||
t.Logf("AIAssert error: %v", wingsErr)
|
||||
t.Logf("AIAssert error: %v", aiErr)
|
||||
|
||||
// If both succeed, compare results
|
||||
if wingsResult != nil && aiResult != nil {
|
||||
assert.Equal(t, "assert", wingsResult.Type, "AIAssert result type should be 'assert'")
|
||||
assert.Equal(t, "assert", aiResult.Type, "AIAssert result type should be 'assert'")
|
||||
|
||||
// Both should have timing information
|
||||
assert.Greater(t, wingsResult.ModelCallElapsed, int64(0), "AIAssert should have model call elapsed time")
|
||||
assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAssert should have model call elapsed time")
|
||||
|
||||
// Both should have screenshot information
|
||||
assert.NotEmpty(t, wingsResult.ImagePath, "AIAssert should have image path")
|
||||
assert.NotEmpty(t, aiResult.ImagePath, "AIAssert should have image path")
|
||||
|
||||
// Both should have assertion results
|
||||
assert.NotNil(t, wingsResult.AssertionResult, "AIAssert should have assertion result")
|
||||
assert.NotNil(t, aiResult.AssertionResult, "AIAssert should have assertion result")
|
||||
|
||||
// Log comparison
|
||||
t.Logf("AIAssert Pass: %t, Thought: %s", wingsResult.AssertionResult.Pass, wingsResult.AssertionResult.Thought)
|
||||
t.Logf("AIAssert Pass: %t, Thought: %s", aiResult.AssertionResult.Pass, aiResult.AssertionResult.Thought)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDriverExt_AIAssert_ErrorHandling tests AIAssert error handling
|
||||
func TestDriverExt_AIAssert_ErrorHandling(t *testing.T) {
|
||||
driver := setupDriverExt(t)
|
||||
|
||||
// Test with empty assertion
|
||||
result, err := driver.AIAssert("")
|
||||
|
||||
// Should handle empty assertion gracefully
|
||||
if err != nil {
|
||||
t.Logf("Empty assertion error (may be expected): %v", err)
|
||||
assert.NotNil(t, result, "Result should still be returned even on error")
|
||||
if result != nil {
|
||||
assert.NotEmpty(t, result.Error, "Result should contain error message")
|
||||
}
|
||||
} else {
|
||||
t.Logf("Empty assertion handled successfully")
|
||||
assert.NotNil(t, result, "Result should be returned")
|
||||
}
|
||||
|
||||
// Test with complex assertion
|
||||
complexAssertion := "断言：当前屏幕显示的是主页面，包含用户头像、搜索框、导航栏等关键元素，并且没有任何错误提示信息"
|
||||
|
||||
result2, err2 := driver.AIAssert(complexAssertion)
|
||||
|
||||
// Should handle complex assertion
|
||||
if err2 != nil {
|
||||
t.Logf("Complex assertion result: %v", err2)
|
||||
} else {
|
||||
t.Logf("Complex assertion handled successfully")
|
||||
assert.NotNil(t, result2, "Result should be returned for complex assertion")
|
||||
assert.Equal(t, "assert", result2.Type, "Result type should be 'assert'")
|
||||
if result2.AssertionResult != nil {
|
||||
t.Logf("Assertion passed: %t", result2.AssertionResult.Pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user