Merge branch 'dev' into 'master'

针对 StartToGoal 增加 timeout 机制

See merge request iesqa/httprunner!111
This commit is contained in:
李隆
2025-06-30 08:46:48 +00:00
11 changed files with 246 additions and 61 deletions

View File

@@ -0,0 +1 @@
package game_sudoku

View File

@@ -0,0 +1,73 @@
{
"config": {
"name": "羊了个羊小游戏自动化测试",
"ai_options": {
"llm_service": "doubao-1.5-thinking-vision-pro-250428"
}
},
"teststeps": [
{
"name": "启动抖音 app",
"android": {
"os_type": "android",
"actions": [
{
"method": "app_launch",
"params": "com.ss.android.ugc.aweme"
},
{
"method": "sleep",
"params": 5
}
]
},
"validate": [
{
"check": "ui_foreground_app",
"assert": "equal",
"expect": "com.ss.android.ugc.aweme",
"msg": "app [com.ss.android.ugc.aweme] should be in foreground"
}
]
},
{
"name": "进入「羊了个羊」小游戏",
"android": {
"os_type": "android",
"actions": [
{
"method": "start_to_goal",
"params": "搜索「羊了个羊星球」,进入小程序,加入羊群进入游戏",
"options": {
"pre_mark_operation": true
}
}
]
},
"validate": [
{
"check": "ui_ai",
"assert": "ai_assert",
"expect": "当前页面底部包含「移出」「撤回」「洗牌」按钮",
"msg": "assert ai prompt [当前页面底部包含「移出」「撤回」「洗牌」按钮] failed"
}
]
},
{
"name": "开始游戏",
"android": {
"os_type": "android",
"actions": [
{
"method": "start_to_goal",
"params": "羊了个羊是一款热门的消除类小游戏,玩法简单但具有挑战性。以下是游戏的基本规则说明:\n1. 游戏目标: 玩家需要通过消除图案来完成关卡,最终目标是清空所有图案。\n2. 消除规则:\n- 游戏界面中会出现多个图案,玩家需要点击图案将其放入底部的槽中。\n- 图案存在多层堆叠的情况,只能点击最上层的完整图案。\n- 当槽中有三个相同的图案时,这三个图案会自动消除。\n- 玩家需要尽量避免槽中积累过多不同的图案,以免无法继续消除。\n- 严禁点击收集槽里的图案,严禁观看广告和使用道具(移出、撤回、洗牌)。\n- 请持续推进游戏进程,游戏通关后继续下一关,游戏失败后重新开始。\n3. 游戏界面: 图案通常以堆叠的方式呈现,玩家需要逐层消除。\n4. 关卡设计: 游戏包含多个关卡,随着关卡的推进,图案的复杂度和数量会增加。\n5. 策略性: 玩家需要规划消除顺序,以避免槽中积累过多无法消除的图案。\n\n请严格按照以上游戏规则开始游戏\n",
"options": {
"timeout": 300,
"pre_mark_operation": true
}
}
]
}
}
]
}

View File

@@ -0,0 +1,57 @@
package game_yanglegeyang
import (
"testing"
"github.com/stretchr/testify/require"
hrp "github.com/httprunner/httprunner/v5"
"github.com/httprunner/httprunner/v5/uixt/option"
)
func TestGameYanglegeyang(t *testing.T) {
userInstruction := `羊了个羊是一款热门的消除类小游戏,玩法简单但具有挑战性。以下是游戏的基本规则说明:
1. 游戏目标: 玩家需要通过消除图案来完成关卡,最终目标是清空所有图案。
2. 消除规则:
- 游戏界面中会出现多个图案,玩家需要点击图案将其放入底部的槽中。
- 图案存在多层堆叠的情况,只能点击最上层的完整图案。
- 当槽中有三个相同的图案时,这三个图案会自动消除。
- 玩家需要尽量避免槽中积累过多不同的图案,以免无法继续消除。
- 严禁点击收集槽里的图案,严禁观看广告和使用道具(移出、撤回、洗牌)。
- 请持续推进游戏进程,游戏通关后继续下一关,游戏失败后重新开始。
3. 游戏界面: 图案通常以堆叠的方式呈现,玩家需要逐层消除。
4. 关卡设计: 游戏包含多个关卡,随着关卡的推进,图案的复杂度和数量会增加。
5. 策略性: 玩家需要规划消除顺序,以避免槽中积累过多无法消除的图案。
请严格按照以上游戏规则,开始游戏
`
testCase := &hrp.TestCase{
Config: hrp.NewConfig("羊了个羊小游戏自动化测试").
SetLLMService(option.DOUBAO_1_5_THINKING_VISION_PRO_250428),
TestSteps: []hrp.IStep{
hrp.NewStep("启动抖音 app").
Android().
AppLaunch("com.ss.android.ugc.aweme").
Sleep(5).
Validate().
AssertAppInForeground("com.ss.android.ugc.aweme"),
hrp.NewStep("进入「羊了个羊」小游戏").
Android().
StartToGoal("搜索「羊了个羊星球」,进入小程序,加入羊群进入游戏",
option.WithPreMarkOperation(true)).
Validate().
AssertAI("当前页面底部包含「移出」「撤回」「洗牌」按钮"),
hrp.NewStep("开始游戏").
Android().
StartToGoal(userInstruction,
option.WithPreMarkOperation(true),
option.WithTimeout(300)), // 5 minutes
},
}
err := testCase.Dump2JSON("game_yanglegeyang.json")
require.Nil(t, err)
// err = hrp.NewRunner(t).Run(testCase)
// assert.Nil(t, err)
}

2
go.mod
View File

@@ -157,6 +157,4 @@ require (
software.sslmate.com/src/go-pkcs12 v0.2.0 // indirect
)
replace github.com/danielpaulus/go-ios => code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398
// replace github.com/httprunner/funplugin => ../funplugin

4
go.sum
View File

@@ -1,6 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398 h1:oDMPs9vRnMn1ZAT2SJSejoyV5BwiiWw1JPvifrLlOl4=
code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398/go.mod h1:ZkUcaC59yNba47j/+ULKsCi3dYPFwY9r39PxdmVmLHE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
@@ -85,6 +83,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/danielpaulus/go-ios v1.0.161 h1:HhQO/GqINde9Xrvge5ksHxLQk5hQmUAxE7CcS2bIc4A=
github.com/danielpaulus/go-ios v1.0.161/go.mod h1:ZkUcaC59yNba47j/+ULKsCi3dYPFwY9r39PxdmVmLHE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

View File

@@ -851,10 +851,10 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err
for _, action := range mobileStep.Actions {
select {
case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C:
log.Warn().Msg("timeout in mobile UI runner")
return stepResult, errors.Wrap(code.TimeoutError, "mobile UI runner timeout")
log.Warn().Msg("case timeout in mobile UI runner, abort running")
return stepResult, errors.Wrap(code.TimeoutError, "mobile UI runner case timeout")
case <-s.caseRunner.hrpRunner.interruptSignal:
log.Warn().Msg("interrupted in mobile UI runner")
log.Warn().Msg("interrupted in mobile UI runner, abort running")
return stepResult, errors.Wrap(code.InterruptError, "mobile UI runner interrupted")
default:
actionStartTime := time.Now()
@@ -925,15 +925,18 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err
}
// call MCP tool to execute action with cancellable context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx, cancel := context.WithCancelCause(context.Background())
defer cancel(nil)
// Create a goroutine to monitor for interrupt signals
// Create a goroutine to monitor for interrupt signals and timeouts
go func() {
select {
case <-s.caseRunner.hrpRunner.interruptSignal:
log.Warn().Msg("cancelling action due to interrupt signal")
cancel()
cancel(code.InterruptError)
case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C:
log.Warn().Msg("cancelling action due to case timeout")
cancel(code.TimeoutError)
case <-ctx.Done():
// Context already cancelled
}

View File

@@ -18,7 +18,22 @@ import (
func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...option.ActionOption) ([]*PlanningExecutionResult, error) {
options := option.NewActionOptions(opts...)
log.Info().Int("max_retry_times", options.MaxRetryTimes).Msg("StartToGoal")
logger := log.Info().Str("prompt", prompt)
if options.MaxRetryTimes > 0 {
logger = logger.Int("max_retry_times", options.MaxRetryTimes)
}
if options.Timeout > 0 {
logger = logger.Int("timeout_seconds", options.Timeout)
}
logger.Msg("StartToGoal")
// Create timeout context for entire StartToGoal process if Timeout is specified
if options.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second)
defer cancel()
log.Info().Int("timeout_seconds", options.Timeout).Msg("StartToGoal timeout configured for entire process")
}
var allPlannings []*PlanningExecutionResult
var attempt int
@@ -26,14 +41,17 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op
attempt++
log.Info().Int("attempt", attempt).Msg("planning attempt")
// Check for context cancellation (interrupt signal)
// Check for context cancellation (interrupt signal or timeout)
select {
case <-ctx.Done():
cause := context.Cause(ctx)
log.Warn().
Int("attempt", attempt).
Int("completed_plannings", len(allPlannings)).
Msg("interrupted in StartToGoal")
return allPlannings, errors.Wrap(code.InterruptError, "StartToGoal interrupted")
Err(cause).
Msg("StartToGoal cancelled")
// Return the specific error type based on the cancellation cause
return allPlannings, errors.Wrap(cause, "StartToGoal cancelled")
default:
}
@@ -85,15 +103,18 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op
// Check for context cancellation before each action
select {
case <-ctx.Done():
cause := context.Cause(ctx)
log.Warn().
Int("attempt", attempt).
Int("completed_plannings", len(allPlannings)).
Int("completed_tool_calls", len(planningResult.SubActions)).
Int("total_tool_calls", len(planningResult.ToolCalls)).
Msg("interrupted in invokeToolCalls")
Err(cause).
Msg("invokeToolCalls cancelled")
planningResult.Elapsed = time.Since(planningStartTime).Milliseconds()
allPlannings = append(allPlannings, planningResult)
return allPlannings, errors.Wrap(code.InterruptError, "invokeToolCalls interrupted")
// Return the specific error type based on the cancellation cause
return allPlannings, errors.Wrap(cause, "invokeToolCalls cancelled")
default:
}
@@ -113,8 +134,10 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op
planningResult.SubActions = append(planningResult.SubActions, subActionResult)
}()
// Execute the tool call
if err := dExt.invokeToolCall(ctx, toolCall, opts...); err != nil {
log.Error().Err(err).
Str("action", toolCall.Function.Name).
Msg("invoke tool call failed")
subActionResult.Error = err
return err
}
@@ -176,6 +199,9 @@ func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...optio
for _, toolCall := range planningResult.ToolCalls {
err = dExt.invokeToolCall(ctx, toolCall, opts...)
if err != nil {
log.Error().Err(err).
Str("action", toolCall.Function.Name).
Msg("invoke tool call failed")
aiExecutionResult.Error = err.Error()
return aiExecutionResult, errors.Wrap(err, "invoke tool call failed")
}

View File

@@ -19,6 +19,7 @@ import (
"github.com/danielpaulus/go-ios/ios/instruments"
"github.com/danielpaulus/go-ios/ios/testmanagerd"
"github.com/danielpaulus/go-ios/ios/tunnel"
"github.com/danielpaulus/go-ios/ios/zipconduit"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
@@ -141,6 +142,7 @@ type DeviceDetail struct {
WiFiAddress string `json:"wifiAddress,omitempty"`
BuildVersion string `json:"buildVersion,omitempty"`
}
type ApplicationType string
const (
@@ -238,11 +240,21 @@ func (dev *IOSDevice) NewDriver() (driver IDriver, err error) {
}
func (dev *IOSDevice) Install(appPath string, opts ...option.InstallOption) (err error) {
conn, err := installationproxy.New(dev.DeviceEntry)
if err != nil {
return err
installOpts := option.NewInstallOptions(opts...)
for i := 0; i <= installOpts.RetryTimes; i++ {
var conn *zipconduit.Connection
conn, err = zipconduit.New(dev.DeviceEntry)
if err != nil {
return errors.Wrap(err, "failed to create zipconduit connection")
}
defer conn.Close()
err = conn.SendFile(appPath)
if err != nil {
log.Error().Err(err).Int("retry_times", i).Msg("failed to install app")
continue
}
return nil
}
err = conn.Install(dev.DeviceEntry, appPath)
return err
}

View File

@@ -4,13 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
)
// ToolSleep implements the sleep tool call.
@@ -98,7 +100,8 @@ func (t *ToolSleep) ConvertActionToCallToolRequest(action option.MobileAction) (
// ToolSleepMS implements the sleep_ms tool call.
type ToolSleepMS struct {
// Return data fields - these define the structure of data returned by this tool
Milliseconds int64 `json:"milliseconds" desc:"Duration in milliseconds that was slept"`
Milliseconds int64 `json:"milliseconds" desc:"Duration in milliseconds that was slept"`
Duration string `json:"duration" desc:"Human-readable duration string"`
}
func (t *ToolSleepMS) Name() option.ActionName {
@@ -110,26 +113,44 @@ func (t *ToolSleepMS) Description() string {
}
func (t *ToolSleepMS) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SleepMS)
return []mcp.ToolOption{
mcp.WithNumber("milliseconds", mcp.Description("Number of milliseconds to sleep")),
}
}
func (t *ToolSleepMS) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
unifiedReq, err := parseActionOptions(request.Params.Arguments)
if err != nil {
return nil, err
}
// Validate required parameters
if unifiedReq.Milliseconds == 0 {
return nil, fmt.Errorf("milliseconds is required")
milliseconds, ok := request.Params.Arguments["milliseconds"]
if !ok {
log.Warn().Msg("milliseconds parameter is required, using default value 1000 milliseconds")
milliseconds = 1000
}
// Sleep MS action logic
log.Info().Int64("milliseconds", unifiedReq.Milliseconds).Msg("sleeping in milliseconds")
log.Info().Interface("milliseconds", milliseconds).Msg("sleeping in milliseconds")
duration := time.Duration(unifiedReq.Milliseconds) * time.Millisecond
var duration time.Duration
var actualMilliseconds int64
switch v := milliseconds.(type) {
case float64:
actualMilliseconds = int64(v)
duration = time.Duration(v) * time.Millisecond
case int:
actualMilliseconds = int64(v)
duration = time.Duration(v) * time.Millisecond
case int64:
actualMilliseconds = v
duration = time.Duration(v) * time.Millisecond
case string:
ms, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid sleep duration: %v", v)
}
actualMilliseconds = ms
duration = time.Duration(ms) * time.Millisecond
default:
return nil, fmt.Errorf("unsupported sleep duration type: %T", v)
}
// Use context-aware sleep instead of blocking time.Sleep
select {
@@ -141,8 +162,11 @@ func (t *ToolSleepMS) Implement() server.ToolHandlerFunc {
return nil, fmt.Errorf("sleep interrupted: %w", ctx.Err())
}
message := fmt.Sprintf("Successfully slept for %d milliseconds", unifiedReq.Milliseconds)
returnData := ToolSleepMS{Milliseconds: unifiedReq.Milliseconds}
message := fmt.Sprintf("Successfully slept for %d milliseconds", actualMilliseconds)
returnData := ToolSleepMS{
Milliseconds: actualMilliseconds,
Duration: duration.String(),
}
return NewMCPSuccessResponse(message, &returnData), nil
}

View File

@@ -191,10 +191,6 @@ type ActionOptions struct {
ResetHistory bool `json:"reset_history,omitempty" yaml:"reset_history,omitempty" desc:"Whether to reset conversation history before AI planning"`
OutputSchema interface{} `json:"output_schema,omitempty" yaml:"output_schema,omitempty" desc:"Custom output schema for structured AI query response"`
// Time related
Seconds float64 `json:"seconds,omitempty" yaml:"seconds,omitempty" desc:"Sleep duration in seconds"`
Milliseconds int64 `json:"milliseconds,omitempty" yaml:"milliseconds,omitempty" desc:"Sleep duration in milliseconds"`
// Control options
Context context.Context `json:"-" yaml:"-"`
Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty" desc:"Action identifier for logging"`
@@ -204,7 +200,7 @@ type ActionOptions struct {
PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"`
Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"`
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"`
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds"`
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"`
Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"`
ScreenOptions
@@ -368,7 +364,8 @@ func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) {
}
func (o *ActionOptions) ApplySwipeOffset(absFromX, absFromY, absToX, absToY float64) (
float64, float64, float64, float64) {
float64, float64, float64, float64,
) {
if len(o.SwipeOffset) == 4 {
absFromX += float64(o.SwipeOffset[0])
absFromY += float64(o.SwipeOffset[1])
@@ -536,9 +533,9 @@ func WithMaxRetryTimes(maxRetryTimes int) ActionOption {
}
}
func WithTimeout(timeout int) ActionOption {
func WithTimeout(seconds int) ActionOption {
return func(o *ActionOptions) {
o.Timeout = timeout
o.Timeout = seconds
}
}

View File

@@ -140,16 +140,14 @@ func TestUnifiedActionRequest_CustomOptions(t *testing.T) {
func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) {
// Test basic type fields (no longer pointers)
unifiedReq := &ActionOptions{
Platform: "android",
Serial: "device123",
Count: 5,
Keycode: 123,
Delta: 10,
Width: 800,
Height: 600,
Seconds: 2.5,
Milliseconds: 1500,
TabIndex: 3,
Platform: "android",
Serial: "device123",
Count: 5,
Keycode: 123,
Delta: 10,
Width: 800,
Height: 600,
TabIndex: 3,
}
// Test direct field access (no need for Getter methods)
@@ -158,8 +156,6 @@ func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) {
assert.Equal(t, 10, unifiedReq.Delta)
assert.Equal(t, 800, unifiedReq.Width)
assert.Equal(t, 600, unifiedReq.Height)
assert.Equal(t, 2.5, unifiedReq.Seconds)
assert.Equal(t, int64(1500), unifiedReq.Milliseconds)
assert.Equal(t, 3, unifiedReq.TabIndex)
// Test zero value detection
@@ -169,7 +165,5 @@ func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) {
assert.Equal(t, 0, emptyReq.Delta)
assert.Equal(t, 0, emptyReq.Width)
assert.Equal(t, 0, emptyReq.Height)
assert.Equal(t, 0.0, emptyReq.Seconds)
assert.Equal(t, int64(0), emptyReq.Milliseconds)
assert.Equal(t, 0, emptyReq.TabIndex)
}