From 224f3af5581ae7f16c6a6cc82b038d28ea26ad5a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 12:05:26 +0800 Subject: [PATCH 1/8] example: add yanglegeyang --- examples/game/sudoku/main_test.go | 1 + .../game/yanglegeyang/game_yanglegeyang.json | 73 +++++++++++++++++++ examples/game/yanglegeyang/main_test.go | 57 +++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 examples/game/sudoku/main_test.go create mode 100644 examples/game/yanglegeyang/game_yanglegeyang.json create mode 100644 examples/game/yanglegeyang/main_test.go diff --git a/examples/game/sudoku/main_test.go b/examples/game/sudoku/main_test.go new file mode 100644 index 00000000..348e29a7 --- /dev/null +++ b/examples/game/sudoku/main_test.go @@ -0,0 +1 @@ +package game_sudoku diff --git a/examples/game/yanglegeyang/game_yanglegeyang.json b/examples/game/yanglegeyang/game_yanglegeyang.json new file mode 100644 index 00000000..95ce0f3b --- /dev/null +++ b/examples/game/yanglegeyang/game_yanglegeyang.json @@ -0,0 +1,73 @@ +{ + "config": { + "name": "羊了个羊小游戏自动化测试", + "ai_options": { + "llm_service": "doubao-1.5-ui-tars-250328" + } + }, + "teststeps": [ + { + "name": "启动抖音 app", + "android": { + "os_type": "android", + "actions": [ + { + "method": "app_launch", + "params": "com.ss.android.ugc.aweme" + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_foreground_app", + "assert": "equal", + "expect": "com.ss.android.ugc.aweme", + "msg": "app [com.ss.android.ugc.aweme] should be in foreground" + } + ] + }, + { + "name": "进入「羊了个羊」小游戏", + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "搜索「羊了个羊星球」，进入小程序，加入羊群进入游戏", + "options": { + "pre_mark_operation": true + } + } + ] + }, + "validate": [ + { + "check": "ui_ai", + "assert": "ai_assert", + "expect": "当前位于抖音「羊了个羊」小游戏页面", + "msg": "assert ai prompt [当前位于抖音「羊了个羊」小游戏页面] failed" + } + ] + }, + { + "name": "开始游戏", + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "羊了个羊是一款热门的消除类小游戏，玩法简单但具有挑战性。以下是游戏的基本规则说明：\n1. 游戏目标: 玩家需要通过消除图案来完成关卡，最终目标是清空所有图案。\n2. 消除规则:\n- 游戏界面中会出现多个图案，玩家需要点击图案将其放入底部的槽中。\n- 图案存在多层堆叠的情况，只能点击最上层的完整图案。\n- 当槽中有三个相同的图案时，这三个图案会自动消除。\n- 玩家需要尽量避免槽中积累过多不同的图案，以免无法继续消除。\n- 严禁点击收集槽里的图案，严禁观看广告和使用道具（移出、撤回、洗牌）。\n- 请持续推进游戏进程，游戏通关后继续下一关，游戏失败后重新开始。\n3. 游戏界面: 图案通常以堆叠的方式呈现，玩家需要逐层消除。\n4. 关卡设计: 游戏包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。\n5. 策略性: 玩家需要规划消除顺序，以避免槽中积累过多无法消除的图案。\n\n请严格按照以上游戏规则，开始游戏\n", + "options": { + "max_retry_times": 50, + "pre_mark_operation": true + } + } + ] + } + } + ] +} diff --git a/examples/game/yanglegeyang/main_test.go b/examples/game/yanglegeyang/main_test.go new file mode 100644 index 00000000..2604da14 --- /dev/null +++ b/examples/game/yanglegeyang/main_test.go @@ -0,0 +1,57 @@ +package game_yanglegeyang + +import ( + "testing" + + "github.com/stretchr/testify/require" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" +) + +func TestGameYanglegeyang(t *testing.T) { + userInstruction := `羊了个羊是一款热门的消除类小游戏，玩法简单但具有挑战性。以下是游戏的基本规则说明： +1. 游戏目标: 玩家需要通过消除图案来完成关卡，最终目标是清空所有图案。 +2. 消除规则: +- 游戏界面中会出现多个图案，玩家需要点击图案将其放入底部的槽中。 +- 图案存在多层堆叠的情况，只能点击最上层的完整图案。 +- 当槽中有三个相同的图案时，这三个图案会自动消除。 +- 玩家需要尽量避免槽中积累过多不同的图案，以免无法继续消除。 +- 严禁点击收集槽里的图案，严禁观看广告和使用道具（移出、撤回、洗牌）。 +- 请持续推进游戏进程，游戏通关后继续下一关，游戏失败后重新开始。 +3. 游戏界面: 图案通常以堆叠的方式呈现，玩家需要逐层消除。 +4. 关卡设计: 游戏包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。 +5. 策略性: 玩家需要规划消除顺序，以避免槽中积累过多无法消除的图案。 + +请严格按照以上游戏规则，开始游戏 +` + + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("羊了个羊小游戏自动化测试"). + SetLLMService(option.DOUBAO_1_5_UI_TARS_250328), + TestSteps: []hrp.IStep{ + hrp.NewStep("启动抖音 app"). + Android(). + AppLaunch("com.ss.android.ugc.aweme"). + Sleep(5). + Validate(). + AssertAppInForeground("com.ss.android.ugc.aweme"), + hrp.NewStep("进入「羊了个羊」小游戏"). + Android(). + StartToGoal("搜索「羊了个羊星球」，进入小程序，加入羊群进入游戏", + option.WithPreMarkOperation(true)). + Validate(). + AssertAI("当前位于抖音「羊了个羊」小游戏页面"), + hrp.NewStep("开始游戏"). + Android(). + StartToGoal(userInstruction, + option.WithPreMarkOperation(true), + option.WithMaxRetryTimes(50)), + }, + } + err := testCase.Dump2JSON("game_yanglegeyang.json") + require.Nil(t, err) + + // err = hrp.NewRunner(t).Run(testCase) + // assert.Nil(t, err) +} From e5823bba0eed83b9c838067612c0159c6bdb4d57 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 12:07:16 +0800 Subject: [PATCH 2/8] fix: cancel UI action when case timeout --- step_ui.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/step_ui.go b/step_ui.go index c499321b..ac13b4be 100644 --- a/step_ui.go +++ b/step_ui.go @@ -851,10 +851,10 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err for _, action := range mobileStep.Actions { select { case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C: - log.Warn().Msg("timeout in mobile UI runner") - return stepResult, errors.Wrap(code.TimeoutError, "mobile UI runner timeout") + log.Warn().Msg("case timeout in mobile UI runner, abort running") + return stepResult, errors.Wrap(code.TimeoutError, "mobile UI runner case timeout") case <-s.caseRunner.hrpRunner.interruptSignal: - log.Warn().Msg("interrupted in mobile UI runner") + log.Warn().Msg("interrupted in mobile UI runner, abort running") return stepResult, errors.Wrap(code.InterruptError, "mobile UI runner interrupted") default: actionStartTime := time.Now() @@ -934,6 +934,9 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err case <-s.caseRunner.hrpRunner.interruptSignal: log.Warn().Msg("cancelling action due to interrupt signal") cancel() + case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C: + log.Warn().Msg("cancelling action due to case timeout") + cancel() case <-ctx.Done(): // Context already cancelled } From 0b6e764c9f5e5f5cf443eebfd27eeb7b63e7b4bc Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 13:04:00 +0800 Subject: [PATCH 3/8] refactor: enhance context cancellation handling in mobile UI and driver extensions --- step_ui.go | 10 +++++----- uixt/driver_ext_ai.go | 16 +++++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/step_ui.go b/step_ui.go index ac13b4be..bc729c48 100644 --- a/step_ui.go +++ b/step_ui.go @@ -925,18 +925,18 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err } // call MCP tool to execute action with cancellable context - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) - // Create a goroutine to monitor for interrupt signals + // Create a goroutine to monitor for interrupt signals and timeouts go func() { select { case <-s.caseRunner.hrpRunner.interruptSignal: log.Warn().Msg("cancelling action due to interrupt signal") - cancel() + cancel(code.InterruptError) case <-s.caseRunner.hrpRunner.caseTimeoutTimer.C: log.Warn().Msg("cancelling action due to case timeout") - cancel() + cancel(code.TimeoutError) case <-ctx.Done(): // Context already cancelled } diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 90fc13fc..f6c4c542 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -26,14 +26,17 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op attempt++ log.Info().Int("attempt", attempt).Msg("planning attempt") - // Check for context cancellation (interrupt signal) + // Check for context cancellation (interrupt signal or timeout) select { case <-ctx.Done(): + cause := context.Cause(ctx) log.Warn(). Int("attempt", attempt). Int("completed_plannings", len(allPlannings)). - Msg("interrupted in StartToGoal") - return allPlannings, errors.Wrap(code.InterruptError, "StartToGoal interrupted") + Err(cause). + Msg("StartToGoal cancelled") + // Return the specific error type based on the cancellation cause + return allPlannings, errors.Wrap(cause, "StartToGoal cancelled") default: } @@ -85,15 +88,18 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op // Check for context cancellation before each action select { case <-ctx.Done(): + cause := context.Cause(ctx) log.Warn(). Int("attempt", attempt). Int("completed_plannings", len(allPlannings)). Int("completed_tool_calls", len(planningResult.SubActions)). Int("total_tool_calls", len(planningResult.ToolCalls)). - Msg("interrupted in invokeToolCalls") + Err(cause). + Msg("invokeToolCalls cancelled") planningResult.Elapsed = time.Since(planningStartTime).Milliseconds() allPlannings = append(allPlannings, planningResult) - return allPlannings, errors.Wrap(code.InterruptError, "invokeToolCalls interrupted") + // Return the specific error type based on the cancellation cause + return allPlannings, errors.Wrap(cause, "invokeToolCalls cancelled") default: } From 16bb91a098db942d5892a19298e792a36754ee33 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 13:26:11 +0800 Subject: [PATCH 4/8] feat: add action timeout for StartToGoal --- uixt/driver_ext_ai.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index f6c4c542..aacde7dd 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -119,8 +119,25 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op planningResult.SubActions = append(planningResult.SubActions, subActionResult) }() - // Execute the tool call - if err := dExt.invokeToolCall(ctx, toolCall, opts...); err != nil { + // Create action context with timeout if specified + actionCtx := ctx + if options.Timeout > 0 { + var cancel context.CancelFunc + actionCtx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) + defer cancel() + } + + // Execute the tool call with timeout + if err := dExt.invokeToolCall(actionCtx, toolCall, opts...); err != nil { + // Check if the error is due to timeout + if errors.Is(err, context.DeadlineExceeded) { + log.Warn(). + Str("action", toolCall.Function.Name). + Int("timeout_seconds", options.Timeout). + Msg("action timeout exceeded, continuing to next action") + subActionResult.Error = errors.New("action timeout exceeded") + return nil // Continue to next action instead of failing the entire StartToGoal + } subActionResult.Error = err return err } From f332f4e304dd7a831c5852ac2bd9ff15cc32c655 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 13:36:44 +0800 Subject: [PATCH 5/8] fix: ToolSleepMS tool call --- uixt/mcp_tools_utility.go | 58 +++++++++++++++++++++++++++----------- uixt/option/action.go | 7 ++--- uixt/option/action_test.go | 22 ++++++--------- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/uixt/mcp_tools_utility.go b/uixt/mcp_tools_utility.go index 4aa2a267..9246709d 100644 --- a/uixt/mcp_tools_utility.go +++ b/uixt/mcp_tools_utility.go @@ -4,13 +4,15 @@ import ( "context" "encoding/json" "fmt" + "strconv" "time" - "github.com/httprunner/httprunner/v5/internal/builtin" - "github.com/httprunner/httprunner/v5/uixt/option" "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v5/internal/builtin" + "github.com/httprunner/httprunner/v5/uixt/option" ) // ToolSleep implements the sleep tool call. @@ -98,7 +100,8 @@ func (t *ToolSleep) ConvertActionToCallToolRequest(action option.MobileAction) ( // ToolSleepMS implements the sleep_ms tool call. type ToolSleepMS struct { // Return data fields - these define the structure of data returned by this tool - Milliseconds int64 `json:"milliseconds" desc:"Duration in milliseconds that was slept"` + Milliseconds int64 `json:"milliseconds" desc:"Duration in milliseconds that was slept"` + Duration string `json:"duration" desc:"Human-readable duration string"` } func (t *ToolSleepMS) Name() option.ActionName { @@ -110,26 +113,44 @@ func (t *ToolSleepMS) Description() string { } func (t *ToolSleepMS) Options() []mcp.ToolOption { - unifiedReq := &option.ActionOptions{} - return unifiedReq.GetMCPOptions(option.ACTION_SleepMS) + return []mcp.ToolOption{ + mcp.WithNumber("milliseconds", mcp.Description("Number of milliseconds to sleep")), + } } func (t *ToolSleepMS) Implement() server.ToolHandlerFunc { return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { - unifiedReq, err := parseActionOptions(request.Params.Arguments) - if err != nil { - return nil, err - } - - // Validate required parameters - if unifiedReq.Milliseconds == 0 { - return nil, fmt.Errorf("milliseconds is required") + milliseconds, ok := request.Params.Arguments["milliseconds"] + if !ok { + log.Warn().Msg("milliseconds parameter is required, using default value 1000 milliseconds") + milliseconds = 1000 } // Sleep MS action logic - log.Info().Int64("milliseconds", unifiedReq.Milliseconds).Msg("sleeping in milliseconds") + log.Info().Interface("milliseconds", milliseconds).Msg("sleeping in milliseconds") - duration := time.Duration(unifiedReq.Milliseconds) * time.Millisecond + var duration time.Duration + var actualMilliseconds int64 + switch v := milliseconds.(type) { + case float64: + actualMilliseconds = int64(v) + duration = time.Duration(v) * time.Millisecond + case int: + actualMilliseconds = int64(v) + duration = time.Duration(v) * time.Millisecond + case int64: + actualMilliseconds = v + duration = time.Duration(v) * time.Millisecond + case string: + ms, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid sleep duration: %v", v) + } + actualMilliseconds = ms + duration = time.Duration(ms) * time.Millisecond + default: + return nil, fmt.Errorf("unsupported sleep duration type: %T", v) + } // Use context-aware sleep instead of blocking time.Sleep select { @@ -141,8 +162,11 @@ func (t *ToolSleepMS) Implement() server.ToolHandlerFunc { return nil, fmt.Errorf("sleep interrupted: %w", ctx.Err()) } - message := fmt.Sprintf("Successfully slept for %d milliseconds", unifiedReq.Milliseconds) - returnData := ToolSleepMS{Milliseconds: unifiedReq.Milliseconds} + message := fmt.Sprintf("Successfully slept for %d milliseconds", actualMilliseconds) + returnData := ToolSleepMS{ + Milliseconds: actualMilliseconds, + Duration: duration.String(), + } return NewMCPSuccessResponse(message, &returnData), nil } diff --git a/uixt/option/action.go b/uixt/option/action.go index f811562e..f23ecad8 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -191,10 +191,6 @@ type ActionOptions struct { ResetHistory bool `json:"reset_history,omitempty" yaml:"reset_history,omitempty" desc:"Whether to reset conversation history before AI planning"` OutputSchema interface{} `json:"output_schema,omitempty" yaml:"output_schema,omitempty" desc:"Custom output schema for structured AI query response"` - // Time related - Seconds float64 `json:"seconds,omitempty" yaml:"seconds,omitempty" desc:"Sleep duration in seconds"` - Milliseconds int64 `json:"milliseconds,omitempty" yaml:"milliseconds,omitempty" desc:"Sleep duration in milliseconds"` - // Control options Context context.Context `json:"-" yaml:"-"` Identifier string `json:"identifier,omitempty" yaml:"identifier,omitempty" desc:"Action identifier for logging"` @@ -368,7 +364,8 @@ func (o *ActionOptions) ApplyTapOffset(absX, absY float64) (float64, float64) { } func (o *ActionOptions) ApplySwipeOffset(absFromX, absFromY, absToX, absToY float64) ( - float64, float64, float64, float64) { + float64, float64, float64, float64, +) { if len(o.SwipeOffset) == 4 { absFromX += float64(o.SwipeOffset[0]) absFromY += float64(o.SwipeOffset[1]) diff --git a/uixt/option/action_test.go b/uixt/option/action_test.go index 9d7bb2e6..c15fdc15 100644 --- a/uixt/option/action_test.go +++ b/uixt/option/action_test.go @@ -140,16 +140,14 @@ func TestUnifiedActionRequest_CustomOptions(t *testing.T) { func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) { // Test basic type fields (no longer pointers) unifiedReq := &ActionOptions{ - Platform: "android", - Serial: "device123", - Count: 5, - Keycode: 123, - Delta: 10, - Width: 800, - Height: 600, - Seconds: 2.5, - Milliseconds: 1500, - TabIndex: 3, + Platform: "android", + Serial: "device123", + Count: 5, + Keycode: 123, + Delta: 10, + Width: 800, + Height: 600, + TabIndex: 3, } // Test direct field access (no need for Getter methods) @@ -158,8 +156,6 @@ func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) { assert.Equal(t, 10, unifiedReq.Delta) assert.Equal(t, 800, unifiedReq.Width) assert.Equal(t, 600, unifiedReq.Height) - assert.Equal(t, 2.5, unifiedReq.Seconds) - assert.Equal(t, int64(1500), unifiedReq.Milliseconds) assert.Equal(t, 3, unifiedReq.TabIndex) // Test zero value detection @@ -169,7 +165,5 @@ func TestUnifiedActionRequest_BasicTypeFields(t *testing.T) { assert.Equal(t, 0, emptyReq.Delta) assert.Equal(t, 0, emptyReq.Width) assert.Equal(t, 0, emptyReq.Height) - assert.Equal(t, 0.0, emptyReq.Seconds) - assert.Equal(t, int64(0), emptyReq.Milliseconds) assert.Equal(t, 0, emptyReq.TabIndex) } From ae0d28c26deebbab9331c1479955d3e18be3426f Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 13:55:17 +0800 Subject: [PATCH 6/8] feat: add action timeout for StartToGoal --- uixt/driver_ext_ai.go | 36 ++++++++++++++++-------------------- uixt/option/action.go | 6 +++--- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index aacde7dd..68a4303c 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -18,7 +18,17 @@ import ( func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...option.ActionOption) ([]*PlanningExecutionResult, error) { options := option.NewActionOptions(opts...) - log.Info().Int("max_retry_times", options.MaxRetryTimes).Msg("StartToGoal") + log.Info().Int("max_retry_times", options.MaxRetryTimes). + Int("timeout_seconds", options.Timeout). + Msg("StartToGoal") + + // Create timeout context for entire StartToGoal process if Timeout is specified + if options.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) + defer cancel() + log.Info().Int("timeout_seconds", options.Timeout).Msg("StartToGoal timeout configured for entire process") + } var allPlannings []*PlanningExecutionResult var attempt int @@ -119,25 +129,11 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op planningResult.SubActions = append(planningResult.SubActions, subActionResult) }() - // Create action context with timeout if specified - actionCtx := ctx - if options.Timeout > 0 { - var cancel context.CancelFunc - actionCtx, cancel = context.WithTimeout(ctx, time.Duration(options.Timeout)*time.Second) - defer cancel() - } - - // Execute the tool call with timeout - if err := dExt.invokeToolCall(actionCtx, toolCall, opts...); err != nil { - // Check if the error is due to timeout - if errors.Is(err, context.DeadlineExceeded) { - log.Warn(). - Str("action", toolCall.Function.Name). - Int("timeout_seconds", options.Timeout). - Msg("action timeout exceeded, continuing to next action") - subActionResult.Error = errors.New("action timeout exceeded") - return nil // Continue to next action instead of failing the entire StartToGoal - } + if err := dExt.invokeToolCall(ctx, toolCall, opts...); err != nil { + log.Warn(). + Str("action", toolCall.Function.Name). + Err(err). + Msg("invoke tool call failed") subActionResult.Error = err return err } diff --git a/uixt/option/action.go b/uixt/option/action.go index f23ecad8..15305edf 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -200,7 +200,7 @@ type ActionOptions struct { PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"` Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds"` + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` ScreenOptions @@ -533,9 +533,9 @@ func WithMaxRetryTimes(maxRetryTimes int) ActionOption { } } -func WithTimeout(timeout int) ActionOption { +func WithTimeout(seconds int) ActionOption { return func(o *ActionOptions) { - o.Timeout = timeout + o.Timeout = seconds } } From 3cd4e5b836e28b31b54af7a13c0ee6d4a69ade06 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 14:03:36 +0800 Subject: [PATCH 7/8] change: update examples --- .../game/yanglegeyang/game_yanglegeyang.json | 8 ++++---- examples/game/yanglegeyang/main_test.go | 6 +++--- uixt/driver_ext_ai.go | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/examples/game/yanglegeyang/game_yanglegeyang.json b/examples/game/yanglegeyang/game_yanglegeyang.json index 95ce0f3b..9c72a02a 100644 --- a/examples/game/yanglegeyang/game_yanglegeyang.json +++ b/examples/game/yanglegeyang/game_yanglegeyang.json @@ -2,7 +2,7 @@ "config": { "name": "羊了个羊小游戏自动化测试", "ai_options": { - "llm_service": "doubao-1.5-ui-tars-250328" + "llm_service": "doubao-1.5-thinking-vision-pro-250428" } }, "teststeps": [ @@ -48,8 +48,8 @@ { "check": "ui_ai", "assert": "ai_assert", - "expect": "当前位于抖音「羊了个羊」小游戏页面", - "msg": "assert ai prompt [当前位于抖音「羊了个羊」小游戏页面] failed" + "expect": "当前页面底部包含「移出」「撤回」「洗牌」按钮", + "msg": "assert ai prompt [当前页面底部包含「移出」「撤回」「洗牌」按钮] failed" } ] }, @@ -62,7 +62,7 @@ "method": "start_to_goal", "params": "羊了个羊是一款热门的消除类小游戏，玩法简单但具有挑战性。以下是游戏的基本规则说明：\n1. 游戏目标: 玩家需要通过消除图案来完成关卡，最终目标是清空所有图案。\n2. 消除规则:\n- 游戏界面中会出现多个图案，玩家需要点击图案将其放入底部的槽中。\n- 图案存在多层堆叠的情况，只能点击最上层的完整图案。\n- 当槽中有三个相同的图案时，这三个图案会自动消除。\n- 玩家需要尽量避免槽中积累过多不同的图案，以免无法继续消除。\n- 严禁点击收集槽里的图案，严禁观看广告和使用道具（移出、撤回、洗牌）。\n- 请持续推进游戏进程，游戏通关后继续下一关，游戏失败后重新开始。\n3. 游戏界面: 图案通常以堆叠的方式呈现，玩家需要逐层消除。\n4. 关卡设计: 游戏包含多个关卡，随着关卡的推进，图案的复杂度和数量会增加。\n5. 策略性: 玩家需要规划消除顺序，以避免槽中积累过多无法消除的图案。\n\n请严格按照以上游戏规则，开始游戏\n", "options": { - "max_retry_times": 50, + "timeout": 300, "pre_mark_operation": true } } diff --git a/examples/game/yanglegeyang/main_test.go b/examples/game/yanglegeyang/main_test.go index 2604da14..a6372347 100644 --- a/examples/game/yanglegeyang/main_test.go +++ b/examples/game/yanglegeyang/main_test.go @@ -28,7 +28,7 @@ func TestGameYanglegeyang(t *testing.T) { testCase := &hrp.TestCase{ Config: hrp.NewConfig("羊了个羊小游戏自动化测试"). - SetLLMService(option.DOUBAO_1_5_UI_TARS_250328), + SetLLMService(option.DOUBAO_1_5_THINKING_VISION_PRO_250428), TestSteps: []hrp.IStep{ hrp.NewStep("启动抖音 app"). Android(). @@ -41,12 +41,12 @@ func TestGameYanglegeyang(t *testing.T) { StartToGoal("搜索「羊了个羊星球」，进入小程序，加入羊群进入游戏", option.WithPreMarkOperation(true)). Validate(). - AssertAI("当前位于抖音「羊了个羊」小游戏页面"), + AssertAI("当前页面底部包含「移出」「撤回」「洗牌」按钮"), hrp.NewStep("开始游戏"). Android(). StartToGoal(userInstruction, option.WithPreMarkOperation(true), - option.WithMaxRetryTimes(50)), + option.WithTimeout(300)), // 5 minutes }, } err := testCase.Dump2JSON("game_yanglegeyang.json") diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 68a4303c..e44f5020 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -18,9 +18,14 @@ import ( func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...option.ActionOption) ([]*PlanningExecutionResult, error) { options := option.NewActionOptions(opts...) - log.Info().Int("max_retry_times", options.MaxRetryTimes). - Int("timeout_seconds", options.Timeout). - Msg("StartToGoal") + logger := log.Info().Str("prompt", prompt) + if options.MaxRetryTimes > 0 { + logger = logger.Int("max_retry_times", options.MaxRetryTimes) + } + if options.Timeout > 0 { + logger = logger.Int("timeout_seconds", options.Timeout) + } + logger.Msg("StartToGoal") // Create timeout context for entire StartToGoal process if Timeout is specified if options.Timeout > 0 { @@ -130,9 +135,8 @@ func (dExt *XTDriver) StartToGoal(ctx context.Context, prompt string, opts ...op }() if err := dExt.invokeToolCall(ctx, toolCall, opts...); err != nil { - log.Warn(). + log.Error().Err(err). Str("action", toolCall.Function.Name). - Err(err). Msg("invoke tool call failed") subActionResult.Error = err return err @@ -195,6 +199,9 @@ func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...optio for _, toolCall := range planningResult.ToolCalls { err = dExt.invokeToolCall(ctx, toolCall, opts...) if err != nil { + log.Error().Err(err). + Str("action", toolCall.Function.Name). + Msg("invoke tool call failed") aiExecutionResult.Error = err.Error() return aiExecutionResult, errors.Wrap(err, "invoke tool call failed") } From 89cf550b17dab7688aad801ed6499c82621553b1 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Mon, 30 Jun 2025 16:27:30 +0800 Subject: [PATCH 8/8] change: remove byted go-ios --- go.mod | 2 -- go.sum | 4 ++-- uixt/ios_device.go | 20 ++++++++++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index ea2db9fe..2365d357 100644 --- a/go.mod +++ b/go.mod @@ -157,6 +157,4 @@ require ( software.sslmate.com/src/go-pkcs12 v0.2.0 // indirect ) -replace github.com/danielpaulus/go-ios => code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398 - // replace github.com/httprunner/funplugin => ../funplugin diff --git a/go.sum b/go.sum index fd762816..8d381d89 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398 h1:oDMPs9vRnMn1ZAT2SJSejoyV5BwiiWw1JPvifrLlOl4= -code.byted.org/yuhongzheng/go-ios v0.0.0-20250619061606-bbfa2c208398/go.mod h1:ZkUcaC59yNba47j/+ULKsCi3dYPFwY9r39PxdmVmLHE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -85,6 +83,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danielpaulus/go-ios v1.0.161 h1:HhQO/GqINde9Xrvge5ksHxLQk5hQmUAxE7CcS2bIc4A= +github.com/danielpaulus/go-ios v1.0.161/go.mod h1:ZkUcaC59yNba47j/+ULKsCi3dYPFwY9r39PxdmVmLHE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/uixt/ios_device.go b/uixt/ios_device.go index 9bde7fde..17caddf2 100644 --- a/uixt/ios_device.go +++ b/uixt/ios_device.go @@ -19,6 +19,7 @@ import ( "github.com/danielpaulus/go-ios/ios/instruments" "github.com/danielpaulus/go-ios/ios/testmanagerd" "github.com/danielpaulus/go-ios/ios/tunnel" + "github.com/danielpaulus/go-ios/ios/zipconduit" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -141,6 +142,7 @@ type DeviceDetail struct { WiFiAddress string `json:"wifiAddress,omitempty"` BuildVersion string `json:"buildVersion,omitempty"` } + type ApplicationType string const ( @@ -238,11 +240,21 @@ func (dev *IOSDevice) NewDriver() (driver IDriver, err error) { } func (dev *IOSDevice) Install(appPath string, opts ...option.InstallOption) (err error) { - conn, err := installationproxy.New(dev.DeviceEntry) - if err != nil { - return err + installOpts := option.NewInstallOptions(opts...) + for i := 0; i <= installOpts.RetryTimes; i++ { + var conn *zipconduit.Connection + conn, err = zipconduit.New(dev.DeviceEntry) + if err != nil { + return errors.Wrap(err, "failed to create zipconduit connection") + } + defer conn.Close() + err = conn.SendFile(appPath) + if err != nil { + log.Error().Err(err).Int("retry_times", i).Msg("failed to install app") + continue + } + return nil } - err = conn.Install(dev.DeviceEntry, appPath) return err }