From 059332e39f49d4a98d13e74f8953be15474c202d Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 3 Aug 2025 00:46:50 +0800 Subject: [PATCH 01/27] fix: claude code actions --- .github/workflows/claude-code.yml | 6 +++--- internal/version/VERSION | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/claude-code.yml b/.github/workflows/claude-code.yml index 5c1c9bd9..6b6b740d 100644 --- a/.github/workflows/claude-code.yml +++ b/.github/workflows/claude-code.yml @@ -2,11 +2,11 @@ name: Claude Code on: issue_comment: - types: [created] + types: [created, edited] pull_request_review_comment: - types: [created] + types: [created, edited] issues: - types: [opened, assigned] + types: [opened, assigned, edited] pull_request_review: types: [submitted] diff --git a/internal/version/VERSION b/internal/version/VERSION index a36de551..0abffd80 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250802 +v5.0.0-250803 From c6f358ffca0e7ba9702aa8644779648d7d72f029 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sun, 3 Aug 2025 11:43:21 +0800 Subject: [PATCH 02/27] fix: convert AI tests from skip statements to build tags (#1783) * fix: convert AI tests from skip statements to build tags - Add //go:build localtest tag to uixt/ai/ai_test.go and examples/game/llk/main_test.go - Remove environment-based skip statements and hasRequiredEnvVars functions - Maintain consistency with existing build tag approach for mobile/device tests - Prevents CI/CD failures when external AI services are not available Co-authored-by: debugtalk * fix: add missing BoundBox type and field to Element struct - Add BoundBox struct with X, Y, Width, Height fields - Update Element struct to include BoundBox field - Fix structural mismatch between test expectations and Go code - Resolves CI compilation failures Co-authored-by: debugtalk --------- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --- examples/game/llk/main.go | 9 +++++++++ examples/game/llk/main_test.go | 18 ++---------------- uixt/ai/ai_test.go | 24 ++---------------------- 3 files changed, 13 insertions(+), 38 deletions(-) diff --git a/examples/game/llk/main.go b/examples/game/llk/main.go index 2e0bfafa..c8e5f201 100644 --- a/examples/game/llk/main.go +++ b/examples/game/llk/main.go @@ -34,6 +34,15 @@ type Dimensions struct { type Element struct { Type string `json:"type"` // Element type/name Position Position `json:"position"` // Position in grid + BoundBox BoundBox `json:"boundBox"` // Bounding box coordinates +} + +// BoundBox represents bounding box coordinates +type BoundBox struct { + X float64 `json:"x"` // X coordinate + Y float64 `json:"y"` // Y coordinate + Width float64 `json:"width"` // Box width + Height float64 `json:"height"` // Box height } // Position represents grid position diff --git a/examples/game/llk/main_test.go b/examples/game/llk/main_test.go index d55667ed..74421801 100644 --- a/examples/game/llk/main_test.go +++ b/examples/game/llk/main_test.go @@ -1,10 +1,11 @@ +//go:build localtest + package llk import ( "context" "encoding/json" "fmt" - "os" "testing" "github.com/stretchr/testify/assert" @@ -97,18 +98,6 @@ func convertToGameElementFromQueryResult(result *ai.QueryResult) (*GameElement, return &gameElement, nil } -// hasRequiredEnvVars checks if the required environment variables are set for testing -func hasRequiredEnvVars() bool { - // Check for OpenAI environment variables - if os.Getenv("OPENAI_BASE_URL") != "" && os.Getenv("OPENAI_API_KEY") != "" { - return true - } - // Check for GPT-4O specific environment variables - if os.Getenv("OPENAI_GPT_4O_BASE_URL") != "" && os.Getenv("OPENAI_GPT_4O_API_KEY") != "" { - return true - } - return false -} // loadTestImage loads the test image from testdata func loadTestImage(t *testing.T) (string, types.Size) { @@ -129,9 +118,6 @@ func createAIQueryer(t *testing.T) *ai.Querier { // TestLLKGameBot_AnalyzeGameInterface comprehensive test for game interface analysis func TestLLKGameBot_AnalyzeGameInterface(t *testing.T) { - if !hasRequiredEnvVars() { - t.Skip("Skipping test: required environment variables not set") - } t.Run("AnalyzeWithTestImage", func(t *testing.T) { // Create test bot and load test image diff --git a/uixt/ai/ai_test.go b/uixt/ai/ai_test.go index 2035c047..0a387784 100644 --- a/uixt/ai/ai_test.go +++ b/uixt/ai/ai_test.go @@ -1,8 +1,9 @@ +//go:build localtest + package ai import ( "context" - "os" "testing" "github.com/httprunner/httprunner/v5/internal/builtin" @@ -11,24 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -// hasRequiredEnvVars checks if the required environment variables are set for testing -func hasRequiredEnvVars() bool { - // Check for OpenAI environment variables - if os.Getenv("OPENAI_BASE_URL") != "" && os.Getenv("OPENAI_API_KEY") != "" { - return true - } - // Check for GPT-4O specific environment variables - if os.Getenv("OPENAI_GPT_4O_BASE_URL") != "" && os.Getenv("OPENAI_GPT_4O_API_KEY") != "" { - return true - } - return false -} - func TestILLMServiceQuery(t *testing.T) { - // Skip test if required environment variables are not set - if !hasRequiredEnvVars() { - t.Skip("Skipping test: required environment variables not set") - } // Create LLM service service, err := NewLLMService(option.OPENAI_GPT_4O) @@ -96,10 +80,6 @@ func TestILLMServiceQuery(t *testing.T) { } func TestILLMServiceIntegration(t *testing.T) { - // Skip test if required environment variables are not set - if !hasRequiredEnvVars() { - t.Skip("Skipping test: required environment variables not set") - } // Create LLM service service, err := NewLLMService(option.OPENAI_GPT_4O) From b25599aba4cfb99eba5b1c285227222b8cafb56a Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 3 Aug 2025 11:48:22 +0800 Subject: [PATCH 03/27] feat: add CLAUDE.md --- CLAUDE.md | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..ec539501 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,128 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +HttpRunner v5 is a comprehensive testing framework written in Go that supports API testing, load testing, and UI automation across multiple platforms (Android/iOS/Harmony/Browser). The framework integrates LLM technology for intelligent test automation and uses a pure visual-driven approach (OCR/CV/VLM) for UI testing. + +## Development Commands + +### Building +- `make build` - Build the hrp CLI tool with static linking and embedded version info +- `go build -o output/hrp ./cmd/cli` - Alternative build command +- `make test` - Run unit tests with race detection + +### Testing +- `go test -race -v ./...` - Run all tests with race detection +- `go test -v ./tests/...` - Run test suite only +- `go test -v ./uixt/...` - Run UI automation tests +- `go test -v ./cmd/...` - Run CLI command tests + +### Code Quality +- `go mod tidy` - Clean up dependencies +- `gofmt -w .` - Format code +- Pre-commit hooks are available in `scripts/` directory + +## Core Architecture + +### Main Components + +**Core Testing Engine** +- `runner.go` - Main test runner (HRPRunner, CaseRunner, SessionRunner) +- `testcase.go` - Test case definitions and loading (ITestCase interface) +- `step.go` - Step definitions and configurations +- `step_*.go` - Specific step implementations (request, api, testcase, ui, etc.) + +**Step Types** +- `step_request.go` - HTTP/HTTPS requests +- `step_api.go` - API calls with parameters +- `step_testcase.go` - Nested test cases +- `step_websocket.go` - WebSocket communication +- `step_ui.go` - UI automation steps +- `step_transaction.go` - Transaction grouping +- `step_rendezvous.go` - Synchronization points +- `step_shell.go` - Shell command execution +- `step_function.go` - Custom function calls + +**UI Automation (uixt/)** +- `device.go` - Device abstraction interface (IDevice) +- `driver.go` - Driver interface and session management +- `android_*.go` - Android platform implementation (ADB/UIAutomator2) +- `ios_*.go` - iOS platform implementation (WDA) +- `harmony_*.go` - HarmonyOS implementation (HDC) +- `browser_*.go` - Web browser automation +- `ai/` - AI-powered UI interaction (OCR/VLM) + +**CLI Interface (cmd/)** +- `root.go` - Root command and global configuration +- `run.go` - Test execution +- `server.go` - HTTP server mode +- `convert.go` - Format conversion utilities +- `build.go` - Plugin building +- `adb/` - Android device management +- `ios/` - iOS device management + +### Plugin System + +The framework supports both Go and Python plugins: +- `build.go` - Plugin compilation system +- `plugin.go` - Plugin interface definitions +- Templates in `internal/scaffold/templates/plugin/` + +### Configuration Management + +- `config.go` - Global configuration +- `internal/config/` - Environment and settings management +- Environment variables and .env file support + +## Key Design Patterns + +### Interface-Driven Architecture +- `ITestCase` interface for different test case sources +- `IDevice` interface for multi-platform support +- `IDriver` interface for different automation drivers + +### Step-Based Testing +- Each test consists of configurable steps +- Steps support setup/teardown hooks +- Variables and parameters flow between steps + +### Plugin Architecture +- Hashicorp go-plugin for Go plugins +- Python plugin support via funplugin +- Template-based plugin generation + +## Testing Approach + +### Test Formats Supported +- YAML/JSON test cases +- Go test files +- Python pytest integration +- HAR, Postman, cURL conversion + +### UI Testing Strategy +- Pure visual-driven (no element locators) +- OCR/VLM for text recognition +- Cross-platform unified API +- AI-powered interaction planning + +## Development Guidelines + +### Code Structure +- Core framework logic in root directory +- Platform-specific implementations in `uixt/` +- CLI commands in `cmd/` +- Internal utilities in `internal/` +- Examples in `examples/` + +### Dependencies +- Go 1.23+ required +- Uses Cobra for CLI +- Integrates with multiple automation frameworks +- LLM integration via CloudWeGo Eino + +### Build Configuration +- Static linking for deployment +- Version info embedded via ldflags +- Cross-platform builds supported \ No newline at end of file From 758ece299885654b17d50f59d6c1cad630ab77e6 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sun, 3 Aug 2025 14:29:34 +0800 Subject: [PATCH 04/27] change: update docs --- docs/cmd/hrp.md | 2 +- docs/cmd/hrp_adb.md | 2 +- docs/cmd/hrp_adb_devices.md | 2 +- docs/cmd/hrp_adb_install.md | 2 +- docs/cmd/hrp_adb_screencap.md | 2 +- docs/cmd/hrp_build.md | 2 +- docs/cmd/hrp_convert.md | 2 +- docs/cmd/hrp_ios.md | 2 +- docs/cmd/hrp_ios_apps.md | 2 +- docs/cmd/hrp_ios_devices.md | 2 +- docs/cmd/hrp_ios_install.md | 2 +- docs/cmd/hrp_ios_mount.md | 2 +- docs/cmd/hrp_ios_ps.md | 2 +- docs/cmd/hrp_ios_reboot.md | 2 +- docs/cmd/hrp_ios_tunnel.md | 2 +- docs/cmd/hrp_ios_uninstall.md | 2 +- docs/cmd/hrp_ios_xctest.md | 2 +- docs/cmd/hrp_mcp-server.md | 2 +- docs/cmd/hrp_mcphost.md | 2 +- docs/cmd/hrp_pytest.md | 2 +- docs/cmd/hrp_report.md | 2 +- docs/cmd/hrp_run.md | 2 +- docs/cmd/hrp_server.md | 2 +- docs/cmd/hrp_startproject.md | 2 +- docs/cmd/hrp_wiki.md | 2 +- examples/game/llk/main_test.go | 2 - examples/game/llk/solver_test.go | 5 +- examples/uitest/android_e2e_delay_test.go | 61 --- examples/uitest/android_e2e_delay_test.json | 151 ------- examples/uitest/android_expert_test.json | 409 ------------------ .../uitest/android_swipe_tap_loadmore.json | 171 ++++++++ examples/uitest/harmony_e2e_delay_test.go | 66 --- examples/uitest/harmony_e2e_delay_test.json | 146 ------- examples/uitest/ios_expert_test.json | 388 ----------------- examples/uitest/sph_search.json | 206 +++++++++ examples/uitest/sph_search_test.go | 64 +++ 36 files changed, 470 insertions(+), 1249 deletions(-) delete mode 100644 examples/uitest/android_e2e_delay_test.go delete mode 100644 examples/uitest/android_e2e_delay_test.json delete mode 100644 examples/uitest/android_expert_test.json create mode 100644 examples/uitest/android_swipe_tap_loadmore.json delete mode 100644 examples/uitest/harmony_e2e_delay_test.go delete mode 100644 examples/uitest/harmony_e2e_delay_test.json delete mode 100644 examples/uitest/ios_expert_test.json create mode 100644 examples/uitest/sph_search.json create mode 100644 examples/uitest/sph_search_test.go diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 22b7a5a2..817e9676 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -63,4 +63,4 @@ Copyright © 2017-present debugtalk. Apache-2.0 License. * [hrp startproject](hrp_startproject.md) - Create a scaffold project * [hrp wiki](hrp_wiki.md) - visit https://httprunner.com -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_adb.md b/docs/cmd/hrp_adb.md index 1d75c96a..b5567af5 100644 --- a/docs/cmd/hrp_adb.md +++ b/docs/cmd/hrp_adb.md @@ -23,4 +23,4 @@ simple utils for android device management * [hrp adb install](hrp_adb_install.md) - push package to the device and install them automatically * [hrp adb screencap](hrp_adb_screencap.md) - Start android screen capture -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_adb_devices.md b/docs/cmd/hrp_adb_devices.md index a0aa341b..48656aab 100644 --- a/docs/cmd/hrp_adb_devices.md +++ b/docs/cmd/hrp_adb_devices.md @@ -24,4 +24,4 @@ hrp adb devices [flags] * [hrp adb](hrp_adb.md) - simple utils for android device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_adb_install.md b/docs/cmd/hrp_adb_install.md index 16c1e90a..420b66df 100644 --- a/docs/cmd/hrp_adb_install.md +++ b/docs/cmd/hrp_adb_install.md @@ -28,4 +28,4 @@ hrp adb install [flags] PACKAGE * [hrp adb](hrp_adb.md) - simple utils for android device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_adb_screencap.md b/docs/cmd/hrp_adb_screencap.md index 22179c91..defddfb6 100644 --- a/docs/cmd/hrp_adb_screencap.md +++ b/docs/cmd/hrp_adb_screencap.md @@ -25,4 +25,4 @@ hrp adb screencap [flags] * [hrp adb](hrp_adb.md) - simple utils for android device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_build.md b/docs/cmd/hrp_build.md index 109582be..a884bbfd 100644 --- a/docs/cmd/hrp_build.md +++ b/docs/cmd/hrp_build.md @@ -36,4 +36,4 @@ hrp build $path ... [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_convert.md b/docs/cmd/hrp_convert.md index 1d34c9c9..b4bab2a0 100644 --- a/docs/cmd/hrp_convert.md +++ b/docs/cmd/hrp_convert.md @@ -34,4 +34,4 @@ hrp convert $path... [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios.md b/docs/cmd/hrp_ios.md index d91fab1b..f9eeebf1 100644 --- a/docs/cmd/hrp_ios.md +++ b/docs/cmd/hrp_ios.md @@ -29,4 +29,4 @@ simple utils for ios device management * [hrp ios uninstall](hrp_ios_uninstall.md) - uninstall package automatically * [hrp ios xctest](hrp_ios_xctest.md) - run xctest -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_apps.md b/docs/cmd/hrp_ios_apps.md index eb02b1fc..ee92d84b 100644 --- a/docs/cmd/hrp_ios_apps.md +++ b/docs/cmd/hrp_ios_apps.md @@ -26,4 +26,4 @@ hrp ios apps [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_devices.md b/docs/cmd/hrp_ios_devices.md index fbd2a6ee..d86bfcd3 100644 --- a/docs/cmd/hrp_ios_devices.md +++ b/docs/cmd/hrp_ios_devices.md @@ -24,4 +24,4 @@ hrp ios devices [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_install.md b/docs/cmd/hrp_ios_install.md index 74bcc464..4d06ea83 100644 --- a/docs/cmd/hrp_ios_install.md +++ b/docs/cmd/hrp_ios_install.md @@ -25,4 +25,4 @@ hrp ios install [flags] PACKAGE * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_mount.md b/docs/cmd/hrp_ios_mount.md index b08ceb11..280c47b5 100644 --- a/docs/cmd/hrp_ios_mount.md +++ b/docs/cmd/hrp_ios_mount.md @@ -28,4 +28,4 @@ hrp ios mount [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_ps.md b/docs/cmd/hrp_ios_ps.md index 7e44e89a..675a4603 100644 --- a/docs/cmd/hrp_ios_ps.md +++ b/docs/cmd/hrp_ios_ps.md @@ -26,4 +26,4 @@ hrp ios ps [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_reboot.md b/docs/cmd/hrp_ios_reboot.md index 0ff1628f..641a1022 100644 --- a/docs/cmd/hrp_ios_reboot.md +++ b/docs/cmd/hrp_ios_reboot.md @@ -25,4 +25,4 @@ hrp ios reboot [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_tunnel.md b/docs/cmd/hrp_ios_tunnel.md index 07698c8d..18dd3bd0 100644 --- a/docs/cmd/hrp_ios_tunnel.md +++ b/docs/cmd/hrp_ios_tunnel.md @@ -24,4 +24,4 @@ hrp ios tunnel [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_uninstall.md b/docs/cmd/hrp_ios_uninstall.md index 4674cf29..4c4efcb5 100644 --- a/docs/cmd/hrp_ios_uninstall.md +++ b/docs/cmd/hrp_ios_uninstall.md @@ -26,4 +26,4 @@ hrp ios uninstall [flags] PACKAGE * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_ios_xctest.md b/docs/cmd/hrp_ios_xctest.md index 57807f93..65032225 100644 --- a/docs/cmd/hrp_ios_xctest.md +++ b/docs/cmd/hrp_ios_xctest.md @@ -28,4 +28,4 @@ hrp ios xctest [flags] * [hrp ios](hrp_ios.md) - simple utils for ios device management -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_mcp-server.md b/docs/cmd/hrp_mcp-server.md index 28e05be4..e79e353a 100644 --- a/docs/cmd/hrp_mcp-server.md +++ b/docs/cmd/hrp_mcp-server.md @@ -28,4 +28,4 @@ hrp mcp-server [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_mcphost.md b/docs/cmd/hrp_mcphost.md index dce4bb76..6a0834b2 100644 --- a/docs/cmd/hrp_mcphost.md +++ b/docs/cmd/hrp_mcphost.md @@ -31,4 +31,4 @@ hrp mcphost [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index 20f2acca..b7dff5e7 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -24,4 +24,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_report.md b/docs/cmd/hrp_report.md index c1a63415..b5ef2375 100644 --- a/docs/cmd/hrp_report.md +++ b/docs/cmd/hrp_report.md @@ -33,4 +33,4 @@ hrp report [result_folder] [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index 3bfca80f..01dacdf9 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -46,4 +46,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_server.md b/docs/cmd/hrp_server.md index cd8c3e21..1dfe0a6a 100644 --- a/docs/cmd/hrp_server.md +++ b/docs/cmd/hrp_server.md @@ -30,4 +30,4 @@ hrp server start [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 64784a0e..40a3ef8e 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -29,4 +29,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/docs/cmd/hrp_wiki.md b/docs/cmd/hrp_wiki.md index 643ae924..94ea3c70 100644 --- a/docs/cmd/hrp_wiki.md +++ b/docs/cmd/hrp_wiki.md @@ -24,4 +24,4 @@ hrp wiki [flags] * [hrp](hrp.md) - All-in-One Testing Framework for API, UI and Performance -###### Auto generated by spf13/cobra on 28-Jun-2025 +###### Auto generated by spf13/cobra on 3-Aug-2025 diff --git a/examples/game/llk/main_test.go b/examples/game/llk/main_test.go index 74421801..348cfe3e 100644 --- a/examples/game/llk/main_test.go +++ b/examples/game/llk/main_test.go @@ -98,7 +98,6 @@ func convertToGameElementFromQueryResult(result *ai.QueryResult) (*GameElement, return &gameElement, nil } - // loadTestImage loads the test image from testdata func loadTestImage(t *testing.T) (string, types.Size) { screenshot, size, err := builtin.LoadImage("../../../uixt/ai/testdata/llk_1.png") @@ -118,7 +117,6 @@ func createAIQueryer(t *testing.T) *ai.Querier { // TestLLKGameBot_AnalyzeGameInterface comprehensive test for game interface analysis func TestLLKGameBot_AnalyzeGameInterface(t *testing.T) { - t.Run("AnalyzeWithTestImage", func(t *testing.T) { // Create test bot and load test image querier := createAIQueryer(t) diff --git a/examples/game/llk/solver_test.go b/examples/game/llk/solver_test.go index b6b7acc3..ecfe3430 100644 --- a/examples/game/llk/solver_test.go +++ b/examples/game/llk/solver_test.go @@ -1,3 +1,5 @@ +//go:build localtest + package llk import ( @@ -7,10 +9,11 @@ import ( "os" "testing" - "github.com/httprunner/httprunner/v5/uixt/ai" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/httprunner/httprunner/v5/uixt/ai" ) // TestLLKSolver tests the LianLianKan solver functionality diff --git a/examples/uitest/android_e2e_delay_test.go b/examples/uitest/android_e2e_delay_test.go deleted file mode 100644 index 6b113c4c..00000000 --- a/examples/uitest/android_e2e_delay_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package uitest - -import ( - "testing" - - hrp "github.com/httprunner/httprunner/v5" - "github.com/httprunner/httprunner/v5/uixt/option" -) - -func TestAndroidDouyinE2E(t *testing.T) { - testCase := &hrp.TestCase{ - Config: hrp.NewConfig("直播_抖音_端到端时延_android"). - WithVariables(map[string]interface{}{ - "device": "${ENV(SerialNumber)}", - "ups": "${ENV(LIVEUPLIST)}", - }). - SetAndroid( - option.WithSerialNumber("$device"), - option.WithAdbLogOn(true)), - TestSteps: []hrp.IStep{ - hrp.NewStep("启动抖音"). - Android(). - AppTerminate("com.ss.android.ugc.aweme"). - AppLaunch("com.ss.android.ugc.aweme"). - Home(). - SwipeToTapApp( - "抖音", - option.WithMaxRetryTimes(5), - option.WithTapOffset(0, -50), - ). - Sleep(20). - Validate(). - AssertOCRExists("推荐", "进入抖音失败"), - hrp.NewStep("点击放大镜"). - Android(). - TapXY(0.9, 0.08). - Sleep(5), - hrp.NewStep("输入账号名称"). - Android(). - Input("$ups"). - Sleep(5), - hrp.NewStep("点击搜索"). - Android(). - TapByOCR("搜索"). - Sleep(5), - hrp.NewStep("端到端采集").Loop(5). - Android(). - TapByOCR( - "直播中", - option.WithIgnoreNotFoundError(true), - option.WithIndex(-1), - ). - EndToEndDelay(option.WithInterval(5), option.WithTimeout(120)). - TapByUITypes(option.WithScreenShotUITypes("close")), - }, - } - - if err := testCase.Dump2JSON("android_e2e_delay_test.json"); err != nil { - t.Fatal(err) - } -} diff --git a/examples/uitest/android_e2e_delay_test.json b/examples/uitest/android_e2e_delay_test.json deleted file mode 100644 index d289d61f..00000000 --- a/examples/uitest/android_e2e_delay_test.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "config": { - "name": "直播_抖音_端到端时延_android", - "variables": { - "device": "${ENV(SerialNumber)}", - "ups": "${ENV(LIVEUPLIST)}" - }, - "android": [ - { - "serial": "$device", - "log_on": true, - "adb_server_host": "localhost", - "adb_server_port": 5037, - "uia2_ip": "localhost", - "uia2_port": 6790, - "uia2_server_package_name": "io.appium.uiautomator2.server", - "uia2_server_test_package_name": "io.appium.uiautomator2.server.test" - } - ] - }, - "teststeps": [ - { - "name": "启动抖音", - "android": { - "os_type": "android", - "actions": [ - { - "method": "app_terminate", - "params": "com.ss.android.ugc.aweme" - }, - { - "method": "app_launch", - "params": "com.ss.android.ugc.aweme" - }, - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "抖音", - "options": { - "max_retry_times": 5, - "tap_offset": [ - 0, - -50 - ] - } - }, - { - "method": "sleep", - "params": 20 - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "点击放大镜", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.9, - 0.08 - ], - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "输入账号名称", - "android": { - "os_type": "android", - "actions": [ - { - "method": "input", - "params": "$ups", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "点击搜索", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_ocr", - "params": "搜索", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "端到端采集", - "loops": 5, - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_ocr", - "params": "直播中", - "options": { - "index": -1, - "ignore_NotFoundError": true - } - }, - { - "method": "live_e2e", - "options": { - "interval": 5, - "timeout": 120 - } - }, - { - "method": "tap_cv", - "options": { - "screenshot_with_ui_types": [ - "close" - ] - } - } - ] - } - } - ] -} diff --git a/examples/uitest/android_expert_test.json b/examples/uitest/android_expert_test.json deleted file mode 100644 index 1b11d98d..00000000 --- a/examples/uitest/android_expert_test.json +++ /dev/null @@ -1,409 +0,0 @@ -{ - "config": { - "name": "安卓专家用例", - "variables": { - "app_name": "抖音", - "bundle_id": "com.ss.android.ugc.aweme", - "device": "${ENV(SerialNumber)}", - "query": "${ENV(query)}" - }, - "android": [ - { - "serial": "$device", - "log_on": true, - "adb_server_host": "localhost", - "adb_server_port": 5037, - "uia2": true, - "uia2_ip": "localhost", - "uia2_port": 6790, - "uia2_server_package_name": "io.appium.uiautomator2.server", - "uia2_server_test_package_name": "io.appium.uiautomator2.server.test" - } - ] - }, - "teststeps": [ - { - "name": "app_launch 以及 ui_foreground_app equal 断言", - "android": { - "os_type": "android", - "actions": [ - { - "method": "app_launch", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 2 - } - ] - }, - "validate": [ - { - "check": "ui_foreground_app", - "assert": "equal", - "expect": "$bundle_id", - "msg": "app [$bundle_id] should be in foreground" - } - ] - }, - { - "name": "home 以及 swipe_to_tap_app 默认配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "$app_name", - "options": {} - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言", - "android": { - "os_type": "android", - "actions": [ - { - "method": "close_popups", - "options": {} - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "【直播】feed头像或卡片进房 swipe_to_tap_texts 自定义配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "swipe_to_tap_texts", - "params": [ - "直播", - "直播中", - "点击进入直播间" - ], - "options": { - "identifier": "click_live", - "max_retry_times": 50, - "interval": 1.5, - "direction": [ - 0.5, - 0.7, - 0.5, - 0.3 - ], - "scope": [ - 0.2, - 0.2, - 1, - 0.8 - ] - } - } - ] - } - }, - { - "name": "sleep 10s", - "android": { - "os_type": "android", - "actions": [ - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "【直播】swipe 自定义配置 以及 back", - "android": { - "os_type": "android", - "actions": [ - { - "method": "swipe_coordinate", - "params": [ - 0.5, - 0.7, - 0.5, - 0.3 - ], - "options": { - "identifier": "slide_in_live" - } - }, - { - "method": "sleep", - "params": 5 - }, - { - "method": "back" - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】点击放大镜 tap_xy 自定义配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.9, - 0.08 - ], - "options": { - "identifier": "click_search_in_middle_page" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】输入query词 input", - "android": { - "os_type": "android", - "actions": [ - { - "method": "input", - "params": "$query", - "options": { - "identifier": "input_query" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】点击搜索按钮 tap_ocr 自定义配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_ocr", - "params": "搜索", - "options": { - "identifier": "click_search_after_input_query" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "选择直播页签 tap_ocr 默认配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_ocr", - "params": "直播", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【生活服务】进入直播间 tap_xy", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.5, - 0.5 - ], - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【生活服务】点击货架商品 tap_ocr 自定义配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "tap_cv", - "options": { - "identifier": "click_sales_rack", - "screenshot_with_ui_types": [ - "dyhouse", - "shoppingbag" - ] - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "app_terminate 以及 ui_foreground_app not_equal 断言", - "android": { - "os_type": "android", - "actions": [ - { - "method": "app_terminate", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 2 - } - ] - }, - "validate": [ - { - "check": "ui_foreground_app", - "assert": "not_equal", - "expect": "$bundle_id", - "msg": "app [$bundle_id] should not be in foreground" - } - ] - }, - { - "name": "home 以及 swipe_to_tap_app 自定义配置", - "android": { - "os_type": "android", - "actions": [ - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "$app_name", - "options": { - "max_retry_times": 5, - "interval": 1, - "tap_offset": [ - 0, - -50 - ] - } - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言", - "android": { - "os_type": "android", - "actions": [ - { - "method": "close_popups", - "options": { - "max_retry_times": 3, - "interval": 2 - } - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "返回主界面，并打开本地时间戳", - "android": { - "os_type": "android", - "actions": [ - { - "method": "home" - }, - { - "method": "app_terminate", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 3 - }, - { - "method": "swipe_to_tap_app", - "params": "local", - "options": { - "max_retry_times": 5 - } - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "screeshot 以及 sleep_random", - "loops": 3, - "android": { - "os_type": "android", - "actions": [ - { - "method": "screenshot", - "options": {} - }, - { - "method": "sleep_random", - "params": [ - 1, - 3 - ] - } - ] - } - } - ] -} diff --git a/examples/uitest/android_swipe_tap_loadmore.json b/examples/uitest/android_swipe_tap_loadmore.json new file mode 100644 index 00000000..7b935158 --- /dev/null +++ b/examples/uitest/android_swipe_tap_loadmore.json @@ -0,0 +1,171 @@ +{ + "config": { + "name": "起点_安卓_无限流加载耗时", + "variables": { + "device": "${ENV(SerialNumber)}" + }, + "android": [ + { + "serial": "$device", + "log_on": true, + "ignore_popup": true + } + ] + }, + "teststeps": [ + { + "name": "杀掉之前清除缓存的进程", + "android": { + "actions": [ + { + "method": "app_terminate", + "params": "com.qidian.QDReader" + }, + { + "method": "sleep", + "params": 30 + } + + ] + } + }, + { + "name": "冷启动起点读书app", + "android": { + "actions": [ + { + "method": "app_launch", + "params": "com.qidian.QDReader" + }, + { + "method": "sleep", + "params": 30 + } + ] + } + }, + { + "name": "进入精选-男生频道", + "android":{ + "actions":[ + { + "method": "tap_ocr", + "params": "精选", + "offset": [ + 0, + -50 + ] + }, + { + "method": "sleep", + "params": 7 + }, + { + "method": "tap_ocr", + "params": "男生" + }, + { + "method": "sleep", + "params": 7 + } + ] + } + }, + { + "name": "向下滑动，触发加载", + "android": { + "actions": [ + { + "method": "swipe", + "params": [ + 0.5, + 0.8, + 0.5, + 0.2 + ], + "steps": 1, + "identifier": "xiaoshuo_swip_tab_loadmore" + }, + { + "method": "sleep", + "params": 3 + }, + { + "method": "swipe", + "params": [ + 0.5, + 0.8, + 0.5, + 0.2 + ], + "steps": 1 + }, + { + "method": "sleep", + "params": 3 + }, + { + "method": "swipe", + "params": [ + 0.5, + 0.8, + 0.5, + 0.2 + ], + "steps": 1 + }, + { + "method": "sleep", + "params": 3 + }, + { + "method": "swipe", + "params": [ + 0.5, + 0.8, + 0.5, + 0.2 + ], + "steps": 1 + }, + { + "method": "sleep", + "params": 3 + }, + { + "method": "swipe", + "params": [ + 0.5, + 0.8, + 0.5, + 0.2 + ], + "steps": 1 + } + ] + } + }, + { + "name": "返回", + "android": { + "actions": [ + { + "method": "home" + }, + { + "method": "swipe_to_tap_app", + "params": "local", + "offset": [ + 0, + -50 + ] + }, + { + "method": "sleep", + "params": 7 + } + ] + } + } + ] +} diff --git a/examples/uitest/harmony_e2e_delay_test.go b/examples/uitest/harmony_e2e_delay_test.go deleted file mode 100644 index 31312a7a..00000000 --- a/examples/uitest/harmony_e2e_delay_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package uitest - -import ( - "testing" - - hrp "github.com/httprunner/httprunner/v5" - "github.com/httprunner/httprunner/v5/uixt/option" -) - -func TestHarmonyDouyinE2E(t *testing.T) { - testCase := &hrp.TestCase{ - Config: hrp.NewConfig("直播_抖音_端到端时延_harmony"). - WithVariables(map[string]interface{}{ - "device": "${ENV(SerialNumber)}", - "ups": "${ENV(LIVEUPLIST)}", - }). - SetHarmony( - option.WithConnectKey("$device"), - option.WithLogOn(true)), - TestSteps: []hrp.IStep{ - hrp.NewStep("启动抖音"). - Harmony(). - AppTerminate("com.ss.hm.ugc.aweme"). - SwipeToTapApp("com.ss.hm.ugc.aweme"). - Home(). - SwipeToTapApp( - "抖音", - option.WithMaxRetryTimes(5), - option.WithTapOffset(0, -50), - ). - Sleep(20). - Validate(). - AssertOCRExists("推荐", "进入抖音失败"), - hrp.NewStep("点击放大镜"). - Harmony(). - TapXY(0.9, 0.08). - Sleep(5), - hrp.NewStep("输入账号名称"). - Harmony(). - Input("$ups"). - Sleep(5), - hrp.NewStep("点击搜索"). - Harmony(). - TapByOCR("搜索"). - Sleep(5), - hrp.NewStep("端到端采集").Loop(5). - Harmony(). - TapByOCR( - "直播中", - option.WithIgnoreNotFoundError(true), - option.WithIndex(-1), - ). - EndToEndDelay(option.WithInterval(5), option.WithTimeout(120)). - TapByUITypes(option.WithScreenShotUITypes("close")), - }, - } - - if err := testCase.Dump2JSON("harmony_e2e_delay_test.json"); err != nil { - t.Fatal(err) - } - - err := hrp.Run(t, testCase) - if err != nil { - t.Fatal(err) - } -} diff --git a/examples/uitest/harmony_e2e_delay_test.json b/examples/uitest/harmony_e2e_delay_test.json deleted file mode 100644 index dc57f7ed..00000000 --- a/examples/uitest/harmony_e2e_delay_test.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "config": { - "name": "直播_抖音_端到端时延_harmony", - "variables": { - "device": "${ENV(SerialNumber)}", - "ups": "${ENV(LIVEUPLIST)}" - }, - "harmony": [ - { - "connect_key": "$device", - "log_on": true - } - ] - }, - "teststeps": [ - { - "name": "启动抖音", - "harmony": { - "os_type": "harmony", - "actions": [ - { - "method": "app_terminate", - "params": "com.ss.hm.ugc.aweme" - }, - { - "method": "swipe_to_tap_app", - "params": "com.ss.hm.ugc.aweme", - "options": {} - }, - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "抖音", - "options": { - "max_retry_times": 5, - "tap_offset": [ - 0, - -50 - ] - } - }, - { - "method": "sleep", - "params": 20 - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "点击放大镜", - "harmony": { - "os_type": "harmony", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.9, - 0.08 - ], - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "输入账号名称", - "harmony": { - "os_type": "harmony", - "actions": [ - { - "method": "input", - "params": "$ups", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "点击搜索", - "harmony": { - "os_type": "harmony", - "actions": [ - { - "method": "tap_ocr", - "params": "搜索", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "端到端采集", - "loops": 5, - "harmony": { - "os_type": "harmony", - "actions": [ - { - "method": "tap_ocr", - "params": "直播中", - "options": { - "index": -1, - "ignore_NotFoundError": true - } - }, - { - "method": "live_e2e", - "options": { - "interval": 5, - "timeout": 120 - } - }, - { - "method": "tap_cv", - "options": { - "screenshot_with_ui_types": [ - "close" - ] - } - } - ] - } - } - ] -} diff --git a/examples/uitest/ios_expert_test.json b/examples/uitest/ios_expert_test.json deleted file mode 100644 index 3693d169..00000000 --- a/examples/uitest/ios_expert_test.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "config": { - "name": "iOS 专家用例", - "variables": { - "app_name": "抖音", - "bundle_id": "com.ss.iphone.ugc.Aweme", - "device": "${ENV(UDID)}", - "query": "${ENV(query)}" - }, - "ios": [ - { - "udid": "$device", - "port": 8700, - "mjpeg_port": 8800, - "log_on": true - } - ] - }, - "teststeps": [ - { - "name": "启动应用程序 app_launch", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "app_launch", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 2 - } - ] - } - }, - { - "name": "home 以及 swipe_to_tap_app 默认配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "$app_name", - "options": {} - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "处理弹窗 close_popups 默认配置 以及 ui_ocr exists 断言", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "close_popups", - "options": {} - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "【直播】feed头像或卡片进房 swipe_to_tap_texts 自定义配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "swipe_to_tap_texts", - "params": [ - "直播", - "直播中", - "点击进入直播间" - ], - "options": { - "identifier": "click_live", - "max_retry_times": 50, - "interval": 1.5, - "direction": [ - 0.5, - 0.7, - 0.5, - 0.3 - ], - "scope": [ - 0.2, - 0.2, - 1, - 0.8 - ] - } - } - ] - } - }, - { - "name": "sleep 10s", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "【直播】swipe 自定义配置 以及 back", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "swipe_coordinate", - "params": [ - 0.5, - 0.7, - 0.5, - 0.3 - ], - "options": { - "identifier": "slide_in_live" - } - }, - { - "method": "sleep", - "params": 5 - }, - { - "method": "back" - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】点击放大镜 tap_xy 自定义配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.9, - 0.075 - ], - "options": { - "identifier": "click_search_in_middle_page" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】输入query词 input", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "input", - "params": "$query", - "options": { - "identifier": "input_query" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【搜索】点击搜索按钮 tap_ocr 自定义配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "tap_ocr", - "params": "搜索", - "options": { - "identifier": "click_search_after_input_query" - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "选择直播页签 tap_ocr 默认配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "tap_ocr", - "params": "直播", - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【生活服务】进入直播间 tap_xy", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "tap_xy", - "params": [ - 0.5, - 0.5 - ], - "options": {} - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "【生活服务】点击货架商品 tap_ocr 自定义配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "tap_cv", - "options": { - "identifier": "click_sales_rack", - "screenshot_with_ui_types": [ - "dyhouse", - "shoppingbag" - ] - } - }, - { - "method": "sleep", - "params": 5 - } - ] - } - }, - { - "name": "终止应用程序 app_terminate", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "app_terminate", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 2 - } - ] - } - }, - { - "name": "home 以及 swipe_to_tap_app 自定义配置", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "home" - }, - { - "method": "swipe_to_tap_app", - "params": "$app_name", - "options": { - "max_retry_times": 5, - "interval": 1, - "tap_offset": [ - 0, - -50 - ] - } - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "处理弹窗 close_popups 自定义配置 以及 ui_ocr exists 断言", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "close_popups", - "options": { - "max_retry_times": 3, - "interval": 2 - } - } - ] - }, - "validate": [ - { - "check": "ui_ocr", - "assert": "exists", - "expect": "推荐", - "msg": "进入抖音失败" - } - ] - }, - { - "name": "返回主界面，并打开本地时间戳", - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "home" - }, - { - "method": "app_terminate", - "params": "$bundle_id" - }, - { - "method": "sleep", - "params": 3 - }, - { - "method": "swipe_to_tap_app", - "params": "local", - "options": { - "max_retry_times": 5 - } - }, - { - "method": "sleep", - "params": 10 - } - ] - } - }, - { - "name": "screeshot 以及 sleep_random", - "loops": 3, - "ios": { - "os_type": "ios", - "actions": [ - { - "method": "screenshot", - "options": {} - }, - { - "method": "sleep_random", - "params": [ - 1, - 3 - ] - } - ] - } - } - ] -} diff --git a/examples/uitest/sph_search.json b/examples/uitest/sph_search.json new file mode 100644 index 00000000..e68a2ee5 --- /dev/null +++ b/examples/uitest/sph_search.json @@ -0,0 +1,206 @@ +{ + "config": { + "name": "视频号搜索", + "ai_options": { + "llm_service": "doubao-1.5-ui-tars-250328" + } + }, + "teststeps": [ + { + "name": "启动视频号 app", + "android": { + "os_type": "android", + "actions": [ + { + "method": "app_launch", + "params": "com.tencent.mm" + }, + { + "method": "sleep", + "params": 5 + } + ] + }, + "validate": [ + { + "check": "ui_foreground_app", + "assert": "equal", + "expect": "com.tencent.mm", + "msg": "app [com.tencent.mm] should be in foreground" + } + ] + }, + { + "name": "进入视频号搜索页面", + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "进入「发现」页，点击进入「视频号」页面，点击搜索框", + "options": {} + } + ] + }, + "validate": [ + { + "check": "ui_ai", + "assert": "ai_assert", + "expect": "当前页面包含搜索框和搜索按钮", + "msg": "assert ai prompt [当前页面包含搜索框和搜索按钮] failed" + } + ] + }, + { + "name": "搜索短剧「$dramaName」", + "parameters": { + "dramaName": [ + "督军，你家小福包有祖传乌鸦嘴", + "换亲后我顺便换了江山，很合理吧", + "穿过荆棘拥抱你", + "认亲后，误入帮派成团宠", + "欲念疯长", + "花轿临门她拒嫁，只盼故人归", + "太监武帝，功法自动大圆满", + "容先生，你的爱意藏不住了", + "回家给娘亲改命，心声咋还泄露了", + "相亲遇甜妹，偷娶她闺蜜" + ] + }, + "android": { + "os_type": "android", + "actions": [ + { + "method": "start_to_goal", + "params": "输入「$dramaName」，点击搜索", + "options": {} + }, + { + "method": "sleep", + "params": 1 + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + }, + { + "method": "swipe_direction", + "params": "up", + "options": {} + }, + { + "method": "sleep_random", + "params": [ + 1, + 2 + ] + } + ] + } + } + ] +} diff --git a/examples/uitest/sph_search_test.go b/examples/uitest/sph_search_test.go new file mode 100644 index 00000000..8f209276 --- /dev/null +++ b/examples/uitest/sph_search_test.go @@ -0,0 +1,64 @@ +package uitest + +import ( + "testing" + + "github.com/stretchr/testify/require" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt/option" +) + +func TestSPHSearchPage(t *testing.T) { + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("视频号搜索"). + SetLLMService(option.DOUBAO_1_5_UI_TARS_250328), // Configure LLM service for AI operations + TestSteps: []hrp.IStep{ + hrp.NewStep("启动视频号 app"). + Android(). + AppLaunch("com.tencent.mm"). + Sleep(5). + Validate(). + AssertAppInForeground("com.tencent.mm"), + hrp.NewStep("进入视频号搜索页面"). + Android(). + StartToGoal("进入「发现」页，点击进入「视频号」页面，点击搜索框"). + Validate(). + AssertAI("当前页面包含搜索框和搜索按钮"), + hrp.NewStep("搜索短剧「$dramaName」"). + WithParameters(map[string]interface{}{ + "dramaName": []string{ + "督军，你家小福包有祖传乌鸦嘴", + "换亲后我顺便换了江山，很合理吧", + "穿过荆棘拥抱你", + "认亲后，误入帮派成团宠", + "欲念疯长", + "花轿临门她拒嫁，只盼故人归", + "太监武帝，功法自动大圆满", + "容先生，你的爱意藏不住了", + "回家给娘亲改命，心声咋还泄露了", + "相亲遇甜妹，偷娶她闺蜜", + }, + }). + Android(). + StartToGoal("输入「$dramaName」，点击搜索"). + Sleep(1). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2). + SwipeUp().SleepRandom(1, 2), + }, + } + + err := testCase.Dump2JSON("sph_search.json") + require.Nil(t, err) + + // err := hrp.NewRunner(t).Run(testCase) + // assert.Nil(t, err) +} From 256ec4baf32012577fc5d63474f62d499454e49a Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 10:46:20 +0800 Subject: [PATCH 05/27] fix: get action log from file --- internal/config/config.go | 2 +- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index aba43f25..79afdcf0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,7 +25,7 @@ const ( CaseFileName = "case.json" // $PWD/results/20060102150405/case.json // mobile device path - DeviceActionLogFilePath = "/sdcard/Android/data/io.appium.uiautomator2.server/files/hodor" + DeviceActionLogFilePath = "/storage/emulated/0/Download/" ) type Config struct { diff --git a/internal/version/VERSION b/internal/version/VERSION index db846138..77429189 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250728 +v5.0.0-250806 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 0f54b9c3..813d5673 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/httprunner/funplugin/myexec" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/utf7" @@ -706,17 +707,17 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { log.Error().Err(err).Msg("failed to close adb log writer") } pointRes := ConvertPoints(ad.Device.Logcat.logs) - - // 没有解析到打点日志，走兜底逻辑 + //将pointRes置为空数组 // 没有解析到打点日志，走兜底逻辑 if len(pointRes) == 0 { log.Info().Msg("action log is null, use action file >>>") actionLogDirPath := config.GetConfig().ActionLogDirPath() - logFilePathPrefix := fmt.Sprintf("%v/data", actionLogDirPath) files := []string{} - ad.Device.RunShellCommand("pull", config.DeviceActionLogFilePath, actionLogDirPath) + actionLogRegStr := `.*data_\d+\.txt` + // ad.Device.RunShellCommand("pull", config.DeviceActionLogFilePath, actionLogDirPath) + myexec.RunCommand("adb", "-s", ad.Device.Serial(), "pull", config.DeviceActionLogFilePath, actionLogDirPath) err = filepath.Walk(actionLogDirPath, func(path string, info fs.FileInfo, err error) error { // 只是需要日志文件 - if ok := strings.Contains(path, logFilePathPrefix); ok { + if ok, _ := regexp.MatchString(actionLogRegStr, path); ok { files = append(files, path) } return nil From ae837ac8854a9fc96df9d73fc39e45a08da41c84 Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 10:49:22 +0800 Subject: [PATCH 06/27] fix: remove useless code --- uixt/android_driver_adb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 813d5673..6aa961b4 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -713,7 +713,6 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { actionLogDirPath := config.GetConfig().ActionLogDirPath() files := []string{} actionLogRegStr := `.*data_\d+\.txt` - // ad.Device.RunShellCommand("pull", config.DeviceActionLogFilePath, actionLogDirPath) myexec.RunCommand("adb", "-s", ad.Device.Serial(), "pull", config.DeviceActionLogFilePath, actionLogDirPath) err = filepath.Walk(actionLogDirPath, func(path string, info fs.FileInfo, err error) error { // 只是需要日志文件 From 72e0fe795dad53875a4c72eafca3cf6095a2fae0 Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 11:03:30 +0800 Subject: [PATCH 07/27] fix: remove useless note --- uixt/android_driver_adb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 6aa961b4..290e68a9 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -707,7 +707,7 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { log.Error().Err(err).Msg("failed to close adb log writer") } pointRes := ConvertPoints(ad.Device.Logcat.logs) - //将pointRes置为空数组 // 没有解析到打点日志，走兜底逻辑 + // 没有解析到打点日志，走兜底逻辑 if len(pointRes) == 0 { log.Info().Msg("action log is null, use action file >>>") actionLogDirPath := config.GetConfig().ActionLogDirPath() From 08849850f92fb643d90c2e46b108d06bd5c45ba4 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 6 Aug 2025 11:42:22 +0800 Subject: [PATCH 08/27] feat: add gadb pull folder/file --- .github/workflows/smoketest.yml | 8 ++++++ .github/workflows/unittest.yml | 8 +++++- .gitignore | 4 +++ internal/version/VERSION | 2 +- pkg/gadb/device.go | 49 +++++++++++++++++++++++++++++++++ pkg/gadb/device_test.go | 11 ++++++++ 6 files changed, 80 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml index 83df9a2b..c0a79a20 100644 --- a/.github/workflows/smoketest.yml +++ b/.github/workflows/smoketest.yml @@ -24,8 +24,16 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' - name: Checkout code uses: actions/checkout@v2 + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install funppy httprunner - name: Build hrp binary run: make build - name: Run smoketest - run with parameters diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 0a854858..55ce8434 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -23,8 +23,14 @@ jobs: uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' - name: Install Python plugin dependencies - run: python3 -m pip install funppy + run: | + python3 -m pip install --upgrade pip + python3 -m pip install funppy httprunner - name: Checkout code uses: actions/checkout@v2 - name: Run coverage diff --git a/.gitignore b/.gitignore index b2daa1e9..8a25c14f 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ dist *.egg-info .python-version .pytest_cache + +# generated go module files in templates +internal/scaffold/templates/plugin/go.mod +internal/scaffold/templates/plugin/go.sum diff --git a/internal/version/VERSION b/internal/version/VERSION index 0abffd80..77429189 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250803 +v5.0.0-250806 diff --git a/pkg/gadb/device.go b/pkg/gadb/device.go index e6cd5aed..c6857fee 100644 --- a/pkg/gadb/device.go +++ b/pkg/gadb/device.go @@ -610,6 +610,55 @@ func (d *Device) Pull(remotePath string, dest io.Writer) (err error) { return } +func (d *Device) PullFolder(remotePath string, localPath string) (err error) { + // Check if remote path exists and is a directory + fileInfos, err := d.List(remotePath) + if err != nil { + return fmt.Errorf("failed to list remote directory: %w", err) + } + + // Create local directory if it doesn't exist + if err = os.MkdirAll(localPath, 0o755); err != nil { + return fmt.Errorf("failed to create local directory: %w", err) + } + + // Pull each file/directory recursively + for _, fileInfo := range fileInfos { + remoteItemPath := remotePath + "/" + fileInfo.Name + localItemPath := localPath + "/" + fileInfo.Name + + if fileInfo.IsDir() { + // Recursively pull subdirectory + if err = d.PullFolder(remoteItemPath, localItemPath); err != nil { + return fmt.Errorf("failed to pull subdirectory %s: %w", remoteItemPath, err) + } + } else { + // Pull file + if err = d.PullFile(remoteItemPath, localItemPath); err != nil { + return fmt.Errorf("failed to pull file %s: %w", remoteItemPath, err) + } + } + } + + return nil +} + +func (d *Device) PullFile(remotePath string, localPath string) (err error) { + // Create local file + localFile, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("failed to create local file: %w", err) + } + defer localFile.Close() + + // Use existing Pull method to pull file content + if err = d.Pull(remotePath, localFile); err != nil { + return fmt.Errorf("failed to pull file content: %w", err) + } + + return nil +} + func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byte, err error) { var ( tp transport diff --git a/pkg/gadb/device_test.go b/pkg/gadb/device_test.go index 0a55e56b..8d8077a2 100644 --- a/pkg/gadb/device_test.go +++ b/pkg/gadb/device_test.go @@ -296,6 +296,17 @@ func TestDevice_Pull(t *testing.T) { } } +func TestDevice_PullFolder(t *testing.T) { + setupDevices(t) + + for _, dev := range devices { + err := dev.PullFolder("/storage/emulated/0/Download/", "/tmp/test/") + if err != nil { + t.Fatal(err) + } + } +} + func TestDevice_ScreenRecord(t *testing.T) { setupDevices(t) From b64e1b62ecbd43aa0276066218c96c57749b875b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 6 Aug 2025 14:35:22 +0800 Subject: [PATCH 09/27] revert: wings merge --- internal/version/VERSION | 2 +- pkg/gadb/device.go | 13 +-- uixt/ai/wings_service.go | 187 +++++++++++-------------------------- uixt/android_device.go | 4 +- uixt/android_test.go | 5 + uixt/driver_ext_ai_test.go | 23 ++++- 6 files changed, 84 insertions(+), 150 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index a36de551..77429189 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250802 +v5.0.0-250806 diff --git a/pkg/gadb/device.go b/pkg/gadb/device.go index e6cd5aed..d616a654 100644 --- a/pkg/gadb/device.go +++ b/pkg/gadb/device.go @@ -615,22 +615,14 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt tp transport filesize int64 ) - timeout := 8 - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute) - defer cancel() - filesize, err = apk.Seek(0, io.SeekEnd) if err != nil { return nil, err } - if tp, err = d.createDeviceTransport(4 * time.Minute); err != nil { + if tp, err = d.createDeviceTransport(5 * time.Minute); err != nil { return nil, err } defer func() { _ = tp.Close() }() - go func() { - <-ctx.Done() - _ = tp.Close() - }() cmd := "abb_exec:package\x00install\x00-t" for _, arg := range args { cmd += "\x00" + arg @@ -649,9 +641,6 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt return nil, err } raw, err = tp.ReadBytesAll() - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - return nil, fmt.Errorf("installation timed out after %d minutes", timeout) - } return } diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index ab25e576..35fa77db 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -26,7 +26,6 @@ type WingsService struct { bizId string accessKey string secretKey string - history []History // Conversation history for Wings API } // NewWingsService creates a new Wings service instance @@ -50,7 +49,6 @@ func NewWingsService() (ILLMService, error) { bizId: bizID, accessKey: accessKey, secretKey: secretKey, - history: []History{}, }, nil } @@ -61,11 +59,6 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni return nil, errors.Wrap(err, "validate planning parameters failed") } - // Reset history if requested - if opts.ResetHistory { - w.resetHistory() - } - // Extract screenshot from message screenshot, err := w.extractScreenshotFromMessage(opts.Message) if err != nil { @@ -77,11 +70,15 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: w.history, - DeviceInfo: deviceInfo, - StepText: fmt.Sprintf("%s", opts.UserInstruction), - BizId: w.bizId, - TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s\n停止操作。\n注意事项：\n", opts.UserInstruction), + Historys: []interface{}{}, // empty as specified + DeviceInfos: []WingsDeviceInfo{ + deviceInfo, + }, + StepText: opts.UserInstruction, + BizId: w.bizId, + TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", + StepType: "automation", + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -101,7 +98,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni } // Check API response status - if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { + if response.BaseResp.StatusCode != 0 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &PlanningResult{ Thought: response.ThoughtChain.Thought, @@ -110,50 +107,26 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni }, err } - // Update history with response data - newHistoryEntry := History{ - Observation: response.ThoughtChain.Observation, - Thought: response.ThoughtChain.Thought, - Summary: response.ThoughtChain.Summary, - StepText: response.StepText, - StepTextTrans: response.StepTextTrans, - OriStepIndex: response.OriStepIndex, - DeviceID: deviceInfo[0].DeviceID, - AgentType: response.AgentType, - ActionResult: "", // Always empty as requested - DeviceInfos: &deviceInfo, - ActionParams: response.ActionParams, + // Convert Wings API response to tool calls + toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams) + if err != nil { + return &PlanningResult{ + Thought: response.ThoughtChain.Thought, + Error: err.Error(), + ModelName: "wings-api", + }, errors.Wrap(err, "convert Wings response to tool calls failed") } - w.history = append(w.history, newHistoryEntry) - var toolCalls []schema.ToolCall - if response.StepType != "FINISH" { - // Convert Wings API response to tool calls - toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams) - if err != nil { - return &PlanningResult{ - Thought: response.ThoughtChain.Thought, - Error: err.Error(), - ModelName: "wings-api", - }, errors.Wrap(err, "convert Wings response to tool calls failed") - } - } - - // No need to update ActionResult as per user request - // ActionResult should always be empty log.Info(). Str("thought", response.ThoughtChain.Thought). - Str("action", response.AgentType). - Str("action_params", response.ActionParams). - Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)). Int("tool_calls_count", len(toolCalls)). Int64("elapsed_ms", elapsed). Msg("Wings API planning completed") return &PlanningResult{ ToolCalls: toolCalls, - Thought: response.StepTextTrans, - Content: response.StepTextTrans, + Thought: response.ThoughtChain.Thought, + Content: response.ThoughtChain.Summary, ModelName: "wings-api", }, nil } @@ -173,15 +146,20 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []History{}, - DeviceInfo: deviceInfo, - StepText: fmt.Sprintf("断言:%s", opts.Assertion), - BizId: w.bizId, - TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), + Historys: []interface{}{}, // empty as specified + DeviceInfos: []WingsDeviceInfo{ + deviceInfo, + }, + StepText: opts.Assertion, + BizId: w.bizId, + TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", + StepType: "assert", // Different from automation + DeviceID: deviceInfo.DeviceID, Base: WingsBase{ LogID: generateWingsUUID(), }, } + log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request") // Call Wings API startTime := time.Now() @@ -197,7 +175,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert } // Check API response status - if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { + if response.BaseResp.StatusCode != 0 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &AssertionResult{ Pass: false, @@ -206,22 +184,6 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, err } - // Update history with response data - newHistoryEntry := History{ - Observation: response.ThoughtChain.Observation, - Thought: response.ThoughtChain.Thought, - Summary: response.ThoughtChain.Summary, - StepText: response.StepText, - StepTextTrans: response.StepTextTrans, - OriStepIndex: response.OriStepIndex, - DeviceID: deviceInfo[0].DeviceID, - AgentType: response.AgentType, - ActionResult: "", // Always empty as requested - DeviceInfos: &deviceInfo, - ActionParams: response.ActionParams, - } - w.history = append(w.history, newHistoryEntry) - // Parse assertion result from action_params passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain) if err != nil { @@ -232,9 +194,6 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, errors.Wrap(err, "parse assertion result failed") } - // No need to update ActionResult as per user request - // ActionResult should always be empty - log.Info(). Bool("passed", passed). Str("thought", assertionThought). @@ -269,12 +228,14 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []History `json:"historys"` - DeviceInfo []WingsDeviceInfo `json:"device_infos"` - StepText string `json:"step_text"` - BizId string `json:"biz_id"` - TextCase string `json:"text_case"` - Base WingsBase `json:"Base"` + Historys []interface{} `json:"historys"` + DeviceInfos []WingsDeviceInfo `json:"device_infos"` + StepText string `json:"step_text"` + BizId string `json:"biz_id"` + TextCase string `json:"text_case"` + StepType string `json:"step_type"` + DeviceID string `json:"device_id"` + Base WingsBase `json:"Base"` } type WingsDeviceInfo struct { @@ -292,14 +253,10 @@ type WingsBase struct { } type WingsActionResponse struct { - AgentType string `json:"agent_type" thrift:"agent_type,1,required"` - StepText string `json:"step_text" thrift:"step_text,2,required"` - StepTextTrans string `json:"step_text_trans" thrift:"step_text_trans,3,required"` - OriStepIndex int `json:"ori_step_index" thrift:"ori_step_index,4,required"` - StepType string `json:"step_type" thrift:"step_type,5,required"` - ActionParams string `json:"action_params" thrift:"action_params,6,required"` - ThoughtChain WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"` - BaseResp WingsBaseResp `json:"BaseResp" thrift:"BaseResp,255,optional"` + StepType string `json:"step_type"` + ActionParams string `json:"action_params"` + ThoughtChain WingsThoughtChain `json:"thought_chain"` + BaseResp WingsBaseResp `json:"BaseResp"` } type WingsThoughtChain struct { @@ -319,21 +276,6 @@ type WingsExtra struct { LogID string `json:"_log_id"` } -// History structure for request and response -type History struct { - Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 - Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 - Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 - StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 - DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id - AgentType string `json:"agent_type" thrift:"agent_type,7"` // 最终决策的agent类型 - ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 - DeviceInfos *[]WingsDeviceInfo `json:"device_infos,omitempty" thrift:"device_infos,9"` // 所有设备的信息 - ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) - StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) - OriStepIndex int `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） -} - // Action parameter structures type WingsActionParams struct { Type string `json:"Type"` @@ -373,11 +315,6 @@ type WingsTextParams struct { // Helper methods -// resetHistory resets the conversation history -func (w *WingsService) resetHistory() { - w.history = []History{} -} - // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { return uuid.New().String() @@ -408,29 +345,19 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st } // getDeviceInfoFromContext gets device info from context with fallback -func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo { - // TODO: Extract device info from context if available - - // Use last history's NowImage as PreImage if history exists - preImage := screenshot - if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 { - preImage = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImage - } - - // use default device info with optimized PreImage - return []WingsDeviceInfo{ - { - DeviceID: "default-device", - NowImage: screenshot, - PreImage: preImage, - NowLayoutJSON: "", - OperationSystem: "android", - }, +func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo { + // use default device info + return WingsDeviceInfo{ + DeviceID: "default-device", + NowImage: screenshot, + PreImage: screenshot, + NowLayoutJSON: "", + OperationSystem: "android", } } // getDeviceInfoFromScreenshot gets device info from screenshot (for Assert) -func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo { +func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo { return w.getDeviceInfoFromContext(ctx, screenshot) } @@ -463,8 +390,6 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Set headers httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Accept", "application/json") - httpReq.Header.Add("x-use-ppe", "1") - httpReq.Header.Add("x-tt-env", "ppe_refactor_merge") // Add authentication headers if using external API if w.accessKey != "" && w.secretKey != "" { @@ -478,7 +403,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Execute HTTP request client := &http.Client{ - Timeout: 120 * time.Second, + Timeout: 60 * time.Second, } resp, err := client.Do(httpReq) @@ -486,9 +411,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ return nil, errors.Wrap(err, "HTTP request failed") } defer resp.Body.Close() - // resp X-Tt-Logid - logID := resp.Header.Get("X-Tt-Logid") - log.Info().Str("step_text", request.StepText).Str("log_id", logID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") + // Read response body responseBody, err := io.ReadAll(resp.Body) if err != nil { @@ -511,7 +434,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) { - if actionParamsStr == "" || actionParamsStr == "FINISH" { + if actionParamsStr == "" { return []schema.ToolCall{}, nil } diff --git a/uixt/android_device.go b/uixt/android_device.go index 0a47eb95..efb243e8 100644 --- a/uixt/android_device.go +++ b/uixt/android_device.go @@ -240,12 +240,12 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er return err } // 等待安装完成或超时 - timeout := 8 * time.Minute + timeout := 3 * time.Minute select { case err := <-done: return err case <-time.After(timeout): - return fmt.Errorf("install via installer timed out after %v", timeout) + return fmt.Errorf("installation timed out after %v", timeout) } } diff --git a/uixt/android_test.go b/uixt/android_test.go index c7cb883d..b8a085be 100644 --- a/uixt/android_test.go +++ b/uixt/android_test.go @@ -21,6 +21,11 @@ func setupADBDriverExt(t *testing.T) *XTDriver { Serial: "", // Let it auto-detect the device serial AIOptions: []option.AIServiceOption{ option.WithCVService(option.CVServiceTypeVEDEM), + option.WithLLMConfig( + option.NewLLMServiceConfig(option.DOUBAO_1_5_UI_TARS_250328). + WithPlannerModel(option.WINGS_SERVICE). + WithAsserterModel(option.WINGS_SERVICE), + ), }, } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 89fd65c8..6ca5d2e3 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,14 +292,31 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "[目标导向]向上滑动屏幕2次" + prompt := "点击搜索按钮" // Test both methods with the same prompt - aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) + aiResult, aiErr := driver.AIAction(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) - t.Logf("AIAction result: %v", aiResult) + + // If both succeed, compare results + if aiResult != nil { + assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'") + + // Both should have timing information + assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time") + + // Both should have screenshot information + assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path") + + // Compare model names + if aiResult.PlanningResult != nil { + t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName) + + assert.Equal(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should use wings-api") + } + } } // TestDriverExt_AIAction_ErrorHandling tests AIAction error handling From 721ed38c4c6f5b20edc93adb66899906ca40c7df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 6 Aug 2025 15:15:28 +0800 Subject: [PATCH 10/27] fix --- .../uitest/android_touch_simulator_test.go | 38 +++ examples/uitest/ios_touch_simulator_test.go | 204 ++++++++++++++ internal/simulation/device_config.go | 10 + internal/version/VERSION | 2 +- uixt/driver.go | 1 + uixt/ios_driver_wda.go | 266 ++++++++++++++++++ uixt/touch_simulator_test.go | 112 ++------ 7 files changed, 549 insertions(+), 84 deletions(-) create mode 100644 examples/uitest/ios_touch_simulator_test.go diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index cec638bb..e2afa7d8 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -602,3 +602,41 @@ func TestStepMultipleSIMActions(t *testing.T) { t.Logf("Successfully executed multiple SIM actions test") } + +func TestStepMultipleSIMAIOSctions(t *testing.T) { + // 创建包含多个SIM操作的测试用例 + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("多个SIM操作组合测试").SetIOS(option.WithUDID("")), + TestSteps: []hrp.IStep{ + hrp.NewStep("组合SIM操作测试"). + Android(). + SIMClickAtPoint(0.3, 0.3). // 点击屏幕中心 + Sleep(1). // 等待1秒 + SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0). // 向上滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0). // 在区域内向下滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeFromPointToPoint(0.1, 0.2, 0.9, 0.5). // 从左到右滑动 + Sleep(0.5). // 等待0.5秒 + SIMInput("测试组合操作 Test Combination 123"), // 仿真输入 + }, + } + + // 运行测试用例 + err := testCase.Dump2JSON("TestStepMultipleSIMActions.json") + if err != nil { + t.Fatalf("Failed to dump test case: %v", err) + } + defer func() { + // 清理生成的文件 + _ = os.Remove("TestStepMultipleSIMActions.json") + }() + + // 执行测试用例 + err = hrp.NewRunner(t).Run(testCase) + if err != nil { + t.Errorf("Test case failed: %v", err) + } + + t.Logf("Successfully executed multiple SIM actions test") +} diff --git a/examples/uitest/ios_touch_simulator_test.go b/examples/uitest/ios_touch_simulator_test.go new file mode 100644 index 00000000..3f280609 --- /dev/null +++ b/examples/uitest/ios_touch_simulator_test.go @@ -0,0 +1,204 @@ +//go:build localtest + +package uitest + +import ( + "os" + "testing" + + hrp "github.com/httprunner/httprunner/v5" + "github.com/httprunner/httprunner/v5/uixt" + "github.com/httprunner/httprunner/v5/uixt/option" +) + +// TestIOSStepMultipleSIMActions tests multiple SIM actions in a step-like manner for iOS +func TestIOSStepMultipleSIMActions(t *testing.T) { + // 创建包含多个 iOS SIM 操作的测试用例 + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("iOS多个SIM操作组合测试").SetIOS(option.WithUDID("")), + TestSteps: []hrp.IStep{ + hrp.NewStep("iOS组合SIM操作测试"). + IOS(). + SIMClickAtPoint(0.5, 0.5). // 点击屏幕中心 + Sleep(1). // 等待1秒 + SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0). // 向上滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0). // 在区域内向上滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeFromPointToPoint(0.1, 0.5, 0.9, 0.5). // 从左到右滑动 + Sleep(0.5). // 等待0.5秒 + SIMInput("iOS测试组合操作 iOS Test Combination 123"), // 仿真输入 + }, + } + + // 运行测试用例 + err := testCase.Dump2JSON("TestIOSStepMultipleSIMActions.json") + if err != nil { + t.Fatalf("Failed to dump test case: %v", err) + } + defer func() { + // 清理生成的文件 + _ = os.Remove("TestIOSStepMultipleSIMActions.json") + }() + + // 执行测试用例 + err = hrp.NewRunner(t).Run(testCase) + if err != nil { + t.Logf("Expected error (no iOS device): %v", err) + // 这是预期的错误，因为没有连接 iOS 设备 + if !containsString(err.Error(), "no attached ios devices") && + !containsString(err.Error(), "device general connection error") { + t.Errorf("Unexpected error: %v", err) + } + } + + t.Logf("Successfully executed multiple iOS SIM actions test (step level)") +} + +// TestIOSDriverDirectSIMFunctions tests iOS SIM functions directly via driver +func TestIOSDriverDirectSIMFunctions(t *testing.T) { + device, err := uixt.NewIOSDevice( + option.WithUDID(""), + ) + if err != nil { + t.Logf("Expected error (no iOS device): %v", err) + // 这是预期的错误，因为没有连接 iOS 设备 + if !containsString(err.Error(), "no attached ios devices") && + !containsString(err.Error(), "device general connection error") { + t.Errorf("Unexpected error: %v", err) + } + return + } + + driver, err := uixt.NewWDADriver(device) + if err != nil { + t.Logf("Expected error (cannot create driver): %v", err) + return + } + defer driver.TearDown() + + // 验证 WDADriver 实现了 SIMSupport 接口 + var iDriver uixt.IDriver = driver + simSupport, ok := iDriver.(uixt.SIMSupport) + if !ok { + t.Errorf("WDADriver does not implement SIMSupport interface") + return + } + _ = simSupport // 避免 unused 警告 + + t.Run("SIMClickAtPoint", func(t *testing.T) { + err := driver.SIMClickAtPoint(0.5, 0.5) + if err != nil { + t.Logf("SIMClickAtPoint error (expected if no device): %v", err) + } else { + t.Logf("Successfully executed SIMClickAtPoint at (0.5, 0.5)") + } + }) + + t.Run("SIMSwipeWithDirection", func(t *testing.T) { + err := driver.SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0) + if err != nil { + t.Logf("SIMSwipeWithDirection error (expected if no device): %v", err) + } else { + t.Logf("Successfully executed SIMSwipeWithDirection") + } + }) + + t.Run("SIMSwipeInArea", func(t *testing.T) { + err := driver.SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0) + if err != nil { + t.Logf("SIMSwipeInArea error (expected if no device): %v", err) + } else { + t.Logf("Successfully executed SIMSwipeInArea") + } + }) + + t.Run("SIMSwipeFromPointToPoint", func(t *testing.T) { + err := driver.SIMSwipeFromPointToPoint(0.1, 0.5, 0.9, 0.5) + if err != nil { + t.Logf("SIMSwipeFromPointToPoint error (expected if no device): %v", err) + } else { + t.Logf("Successfully executed SIMSwipeFromPointToPoint") + } + }) + + t.Run("SIMInput", func(t *testing.T) { + err := driver.SIMInput("iOS测试文本 Test iOS Input 123") + if err != nil { + t.Logf("SIMInput error (expected if no device): %v", err) + } else { + t.Logf("Successfully executed SIMInput") + } + }) +} + +// TestIOSMCPToolsIntegration tests iOS SIM functions via MCP tools (integration test) +func TestIOSMCPToolsIntegration(t *testing.T) { + // 这个测试验证 MCP 工具层是否正确支持 iOS SIM 功能 + device, err := uixt.NewIOSDevice( + option.WithUDID(""), + ) + if err != nil { + t.Logf("Expected error (no iOS device): %v", err) + // 验证错误类型 + if !containsString(err.Error(), "no attached ios devices") && + !containsString(err.Error(), "device general connection error") { + t.Errorf("Unexpected error: %v", err) + } + return + } + + // 需要先创建 WDADriver，然后创建 XTDriver + wdaDriver, err := uixt.NewWDADriver(device) + if err != nil { + t.Logf("Cannot create WDADriver: %v", err) + return + } + defer wdaDriver.TearDown() + + xtDriver, err := uixt.NewXTDriver(wdaDriver) + if err != nil { + t.Logf("Cannot create XTDriver: %v", err) + return + } + + // 验证 XTDriver 的底层驱动实现了 SIMSupport 接口 + if _, ok := xtDriver.IDriver.(uixt.SIMSupport); !ok { + t.Errorf("XTDriver's underlying driver does not implement SIMSupport interface") + return + } + + t.Logf("XTDriver's underlying driver correctly implements SIMSupport interface") + + // 简化测试 - 仅验证接口实现，因为 MCP 服务器的内部结构复杂 + simTools := []option.ActionName{ + option.ACTION_SIMClickAtPoint, + option.ACTION_SIMSwipeDirection, + option.ACTION_SIMSwipeInArea, + option.ACTION_SIMSwipeFromPointToPoint, + option.ACTION_SIMInput, + } + + // 验证这些工具确实存在于系统中 + t.Logf("Verified SIM tools: %v", simTools) + + t.Logf("iOS MCP tools integration test completed - all tools are registered") +} + +// Helper function to check if a string contains a substring +func containsString(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || + (len(s) > len(substr) && + (s[:len(substr)] == substr || + s[len(s)-len(substr):] == substr || + findSubstring(s, substr)))) +} + +func findSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/internal/simulation/device_config.go b/internal/simulation/device_config.go index 4197fe88..eb5a0812 100644 --- a/internal/simulation/device_config.go +++ b/internal/simulation/device_config.go @@ -139,6 +139,16 @@ func getDeviceConfig(deviceModel string) DeviceConfig { SizeMax: 225.0, } + // "Google" + case "iphone": + return DeviceConfig{ + DeviceID: 2, + PressureMin: 1, + PressureMax: 1, + SizeMin: 0.03, + SizeMax: 0.04, + } + // Default configuration for unknown devices default: return DeviceConfig{ diff --git a/internal/version/VERSION b/internal/version/VERSION index a36de551..77429189 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250802 +v5.0.0-250806 diff --git a/uixt/driver.go b/uixt/driver.go index dc21be00..ccc7ec7a 100644 --- a/uixt/driver.go +++ b/uixt/driver.go @@ -18,6 +18,7 @@ var ( // Ensure drivers implement SIMSupport interface _ SIMSupport = (*UIA2Driver)(nil) + _ SIMSupport = (*WDADriver)(nil) ) // current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index 283c5a92..d62a069c 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -24,6 +24,7 @@ import ( "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/json" + "github.com/httprunner/httprunner/v5/internal/simulation" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/types" ) @@ -678,6 +679,13 @@ func (wd *WDADriver) TouchByEvents(events []types.TouchEvent, opts ...option.Act x, y = toX, toY } + if x, err = wd.toScale(x); err != nil { + return err + } + if y, err = wd.toScale(y); err != nil { + return err + } + var actionMap map[string]interface{} switch event.Action { @@ -743,6 +751,201 @@ func (wd *WDADriver) TouchByEvents(events []types.TouchEvent, opts ...option.Act return err } +// SIMSwipeWithDirection 向指定方向滑动任意距离 +// direction: 滑动方向 ("up", "down", "left", "right") +// fromX, fromY: 起始坐标 +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (wd *WDADriver) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { + absStartX, absStartY, err := convertToAbsolutePoint(wd, fromX, fromY) + if err != nil { + return err + } + // 获取设备型号和配置参数 + deviceModel := "iphone" + deviceParams := simulation.GetRandomDeviceParams(deviceModel) + + log.Info().Str("direction", direction). + Float64("startX", absStartX).Float64("startY", absStartY). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). + Str("deviceModel", deviceModel). + Int("deviceID", deviceParams.DeviceID). + Float64("pressure", deviceParams.Pressure). + Float64("size", deviceParams.Size). + Msg("WDADriver.SIMSwipeWithDirection") + + // 导入滑动仿真库 + simulator := simulation.NewSlideSimulatorAPI(nil) + + // 转换方向字符串为Direction类型 + var slideDirection simulation.Direction + switch direction { + case "up": + slideDirection = simulation.Up + case "down": + slideDirection = simulation.Down + case "left": + slideDirection = simulation.Left + case "right": + slideDirection = simulation.Right + default: + return fmt.Errorf("invalid direction: %s, must be one of: up, down, left, right", direction) + } + + // 使用滑动仿真算法生成触摸事件序列 + events, err := simulator.GenerateSlideWithRandomDistance( + absStartX, absStartY, slideDirection, simMinDistance, simMaxDistance, + deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) + if err != nil { + return fmt.Errorf("generate slide events failed: %v", err) + } + + // 执行触摸事件序列 + return wd.TouchByEvents(events, opts...) +} + +// SIMSwipeInArea 在指定区域内向指定方向滑动任意距离 +// direction: 滑动方向 ("up", "down", "left", "right") +// simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY: 区域范围(相对坐标) +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (wd *WDADriver) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { + // 转换区域坐标为绝对坐标 + absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(wd, simAreaStartX, simAreaStartY) + if err != nil { + return err + } + absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(wd, simAreaEndX, simAreaEndY) + if err != nil { + return err + } + + // 确保区域坐标正确(start应该小于等于end) + if absAreaStartX > absAreaEndX { + absAreaStartX, absAreaEndX = absAreaEndX, absAreaStartX + } + if absAreaStartY > absAreaEndY { + absAreaStartY, absAreaEndY = absAreaEndY, absAreaStartY + } + + // 获取设备型号和配置参数 + deviceModel := "iphone" + deviceParams := simulation.GetRandomDeviceParams(deviceModel) + + log.Info().Str("direction", direction). + Float64("areaStartX", absAreaStartX).Float64("areaStartY", absAreaStartY). + Float64("areaEndX", absAreaEndX).Float64("areaEndY", absAreaEndY). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). + Str("deviceModel", deviceModel). + Int("deviceID", deviceParams.DeviceID). + Float64("pressure", deviceParams.Pressure). + Float64("size", deviceParams.Size). + Msg("WDADriver.SIMSwipeInArea") + + // 导入滑动仿真库 + simulator := simulation.NewSlideSimulatorAPI(nil) + + // 转换方向字符串为Direction类型 + var slideDirection simulation.Direction + switch direction { + case "up": + slideDirection = simulation.Up + case "down": + slideDirection = simulation.Down + case "left": + slideDirection = simulation.Left + case "right": + slideDirection = simulation.Right + default: + return fmt.Errorf("invalid direction: %s, must be one of: up, down, left, right", direction) + } + + // 使用滑动仿真算法生成区域内滑动的触摸事件序列 + events, err := simulator.GenerateSlideInArea( + absAreaStartX, absAreaStartY, absAreaEndX, absAreaEndY, + slideDirection, simMinDistance, simMaxDistance, + deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) + if err != nil { + return fmt.Errorf("generate slide in area events failed: %v", err) + } + + // 执行触摸事件序列 + return wd.TouchByEvents(events, opts...) +} + +// SIMSwipeFromPointToPoint 指定起始点和结束点进行滑动 +// fromX, fromY: 起始坐标(相对坐标) +// toX, toY: 结束坐标(相对坐标) +func (wd *WDADriver) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { + // 转换起始点和结束点为绝对坐标 + absStartX, absStartY, err := convertToAbsolutePoint(wd, fromX, fromY) + if err != nil { + return err + } + absEndX, absEndY, err := convertToAbsolutePoint(wd, toX, toY) + if err != nil { + return err + } + + // 获取设备型号和配置参数 + deviceModel := "iphone" + deviceParams := simulation.GetRandomDeviceParams(deviceModel) + + log.Info().Float64("startX", absStartX).Float64("startY", absStartY). + Float64("endX", absEndX).Float64("endY", absEndY). + Str("deviceModel", deviceModel). + Int("deviceID", deviceParams.DeviceID). + Float64("pressure", deviceParams.Pressure). + Float64("size", deviceParams.Size). + Msg("WDADriver.SIMSwipeFromPointToPoint") + + // 导入滑动仿真库 + simulator := simulation.NewSlideSimulatorAPI(nil) + + // 使用滑动仿真算法生成点对点滑动的触摸事件序列 + events, err := simulator.GeneratePointToPointSlideEvents( + absStartX, absStartY, absEndX, absEndY, + deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) + if err != nil { + return fmt.Errorf("generate point to point slide events failed: %v", err) + } + + // 执行触摸事件序列 + return wd.TouchByEvents(events, opts...) +} + +// SIMClickAtPoint 点击相对坐标 +// x, y: 点击坐标(相对坐标) +func (wd *WDADriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { + // 转换为绝对坐标 + absX, absY, err := convertToAbsolutePoint(wd, x, y) + if err != nil { + return err + } + + // 获取设备型号和配置参数 + deviceModel := "iphone" + deviceParams := simulation.GetRandomDeviceParams(deviceModel) + + log.Info().Float64("x", absX).Float64("y", absY). + Str("deviceModel", deviceModel). + Int("deviceID", deviceParams.DeviceID). + Float64("pressure", deviceParams.Pressure). + Float64("size", deviceParams.Size). + Msg("WDADriver.SIMClickAtPoint") + + // 导入点击仿真库 + clickSimulator := simulation.NewClickSimulatorAPI(nil) + + // 使用点击仿真算法生成触摸事件序列 + events, err := clickSimulator.GenerateClickEvents( + absX, absY, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) + if err != nil { + return fmt.Errorf("generate click events failed: %v", err) + } + + // 执行触摸事件序列 + return wd.TouchByEvents(events, opts...) +} + func (wd *WDADriver) SetPasteboard(contentType types.PasteboardType, content string) (err error) { // [[FBRoute POST:@"/wda/setPasteboard"] respondWithTarget:self action:@selector(handleSetPasteboard:)] data := map[string]interface{}{ @@ -784,6 +987,69 @@ func (wd *WDADriver) Input(text string, opts ...option.ActionOption) (err error) return } +// SIMInput 仿真输入函数，模拟人类分批输入行为 +// 将文本智能分割，英文单词和数字保持完整，中文按1-2个字符分割 +func (wd *WDADriver) SIMInput(text string, opts ...option.ActionOption) error { + log.Info().Str("text", text).Msg("WDADriver.SIMInput") + + if text == "" { + return nil + } + + // 创建输入仿真器（使用默认配置） + inputSimulator := simulation.NewInputSimulatorAPI(nil) + + // 生成输入片段（使用智能分割算法，所有参数使用默认值） + inputReq := simulation.InputRequest{ + Text: text, + // MinSegmentLen, MaxSegmentLen, MinDelayMs, MaxDelayMs 使用默认值 + } + + response := inputSimulator.GenerateInputSegments(inputReq) + if !response.Success { + return fmt.Errorf("failed to generate input segments: %s", response.Message) + } + + log.Info().Int("segments", response.Metrics.TotalSegments). + Int("totalDelayMs", response.Metrics.TotalDelayMs). + Int("estimatedTimeMs", response.Metrics.EstimatedTimeMs). + Msg("Input segments generated") + + // 逐个输入每个片段 + var segmentErrCnt int + for _, segment := range response.Segments { + // 使用Input进行输入（内部已包含Session.POST请求） + segmentErr := wd.Input(segment.Text, opts...) + if segmentErr != nil { + segmentErrCnt++ + log.Info().Err(segmentErr).Int("segmentErrCnt", segmentErrCnt). + Msg("segments err") + } + + log.Debug().Str("segment", segment.Text).Int("index", segment.Index). + Int("charLen", segment.CharLen).Msg("Successfully input segment") + + // 如果有延迟时间，则等待 + if segment.DelayMs > 0 { + time.Sleep(time.Duration(segment.DelayMs) * time.Millisecond) + + log.Debug().Int("delayMs", segment.DelayMs). + Msg("Delay between input segments") + } + } + if segmentErrCnt > 0 { + data := map[string]interface{}{"value": strings.Split(text, "")} + option.MergeOptions(data, opts...) + _, err := wd.Session.POST(data, "/wings/interaction/keys") + return err + } + log.Info().Int("totalSegments", response.Metrics.TotalSegments). + Int("actualDelayMs", response.Metrics.TotalDelayMs). + Msg("SIMInput completed successfully") + + return nil +} + func (wd *WDADriver) Backspace(count int, opts ...option.ActionOption) (err error) { log.Info().Int("count", count).Msg("WDADriver.Backspace") if count == 0 { diff --git a/uixt/touch_simulator_test.go b/uixt/touch_simulator_test.go index 4cdcb453..9cf2c1d9 100644 --- a/uixt/touch_simulator_test.go +++ b/uixt/touch_simulator_test.go @@ -66,7 +66,6 @@ func ParseTouchEvents(data string) ([]types.TouchEvent, error) { if event.Action, err = strconv.Atoi(parts[12]); err != nil { return nil, fmt.Errorf("invalid action: %v", err) } - events = append(events, event) } @@ -74,14 +73,14 @@ func ParseTouchEvents(data string) ([]types.TouchEvent, error) { } func TestAndroidTouchByEvents(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -142,28 +141,10 @@ func TestIOSTouchByEvents(t *testing.T) { driver := setupWDADriverExt(t) // Example touch event data as provided - touchEventData := `1752649131556,401.20703,1191.3164,2,1.0,0.03529412,457.20703,1359.3164,111586196,111586196,1,0,0 -1752649131595,402.913,1185.0792,2,1.0,0.039215688,458.913,1353.0792,111586196,111586236,1,0,2 -1752649131612,410.60825,1164.3806,2,1.0,0.03529412,466.60825,1332.3806,111586196,111586250,1,0,2 -1752649131629,437.7335,1093.1417,2,1.0,0.039215688,493.7335,1261.1417,111586196,111586270,1,0,2 -1752649131646,463.5786,1018.01746,2,1.0,0.039215688,519.5786,1186.0175,111586196,111586287,1,0,2 -1752649131662,487.56482,948.9773,2,1.0,0.03529412,543.5648,1116.9773,111586196,111586304,1,0,2 -1752649131679,511.81476,881.6183,2,1.0,0.039215688,567.81476,1049.6183,111586196,111586320,1,0,2 -1752649131696,543.4369,811.4982,2,1.0,0.03529412,599.4369,979.4982,111586196,111586337,1,0,2 -1752649131713,577.1632,747.4512,2,1.0,0.039215688,633.1632,915.4512,111586196,111586354,1,0,2 -1752649131729,610.1538,691.72034,2,1.0,0.03529412,666.1538,859.72034,111586196,111586370,1,0,2 -1752649131746,639.1683,642.6914,2,1.0,0.03529412,695.1683,810.6914,111586196,111586387,1,0,2 -1752649131763,658.9832,605.90857,2,1.0,0.03529412,714.9832,773.90857,111586196,111586404,1,0,2 -1752649131779,672.21954,581.1634,2,1.0,0.03529412,728.21954,749.1634,111586196,111586420,1,0,2 -1752649131796,680.7687,566.1778,2,1.0,0.03529412,736.7687,734.1778,111586196,111586434,1,0,2 -1752649131814,688.0894,554.2295,2,1.0,0.03529412,744.0894,722.2295,111586196,111586450,1,0,2 -1752649131830,694.542,544.7783,2,1.0,0.03529412,750.542,712.7783,111586196,111586466,1,0,2 -1752649131847,700.60645,537.2637,2,1.0,0.039215688,756.60645,705.2637,111586196,111586483,1,0,2 -1752649131863,705.08887,531.1406,2,1.0,0.039215688,761.08887,699.1406,111586196,111586500,1,0,2 -1752649131880,708.1211,527.8008,2,1.0,0.039215688,764.1211,695.8008,111586196,111586517,1,0,2 -1752649131897,709.43945,524.46094,2,1.0,0.039215688,765.43945,692.46094,111586196,111586533,1,0,2 -1752649131902,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586537,1,33554432,2 -1752649131907,709.1758,523.34766,2,1.0,0.03529412,765.1758,691.34766,111586196,111586546,1,0,1` + touchEventData := `1752649131556,401.20703,1191.3164,2,1.0,0.03529412,400.20703,400.3164,111586196,111586196,1,0,0 +1752649131595,402.913,1185.0792,2,1.0,0.039215688,300.913,300.0792,111586196,111586236,1,0,2 +1752649131612,410.60825,1164.3806,2,1.0,0.03529412,250.60825,250.3806,111586196,111586250,1,0,2 +1752649131907,709.1758,523.34766,2,1.0,0.03529412,200.1758,200.34766,111586196,111586546,1,0,1` // Parse touch events events, err := ParseTouchEvents(touchEventData) @@ -281,14 +262,14 @@ func TestTouchEventSequenceValidation(t *testing.T) { } func TestSwipeWithDirection(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -308,7 +289,7 @@ func TestSwipeWithDirection(t *testing.T) { direction: "up", startX: 0.5, startY: 0.5, - minDistance: 100.0, + minDistance: 500.0, maxDistance: 500.0, }, } @@ -332,50 +313,15 @@ func TestSwipeWithDirection(t *testing.T) { } } -func TestSwipeWithDirectionInvalidInputs(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), - ) - if err != nil { - t.Fatal(err) - } - - driver, err := NewUIA2Driver(device) - if err != nil { - t.Fatal(err) - } - defer driver.TearDown() - - // Test invalid direction - err = driver.SIMSwipeWithDirection("invalid", 500.0, 500.0, 100.0, 200.0) - if err == nil { - t.Error("Expected error for invalid direction, but got none") - } - - // Test invalid distance range (max < min) - err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 200.0, 100.0) - if err == nil { - t.Error("Expected error for invalid distance range, but got none") - } - - // Test zero distance - err = driver.SIMSwipeWithDirection("up", 500.0, 500.0, 0.0, 0.0) - if err == nil { - t.Error("Expected error for zero distance, but got none") - } - - t.Log("Invalid input validation tests passed") -} - func TestSwipeInArea(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -428,14 +374,14 @@ func TestSwipeInArea(t *testing.T) { } func TestSwipeFromPointToPoint(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -477,14 +423,14 @@ func TestSwipeFromPointToPoint(t *testing.T) { } func TestSwipeFromPointToPointInvalidInputs(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -506,14 +452,14 @@ func TestSwipeFromPointToPointInvalidInputs(t *testing.T) { } func TestClickAtPoint(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -546,14 +492,14 @@ func TestClickAtPoint(t *testing.T) { } func TestClickAtPointInvalidInputs(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } @@ -580,14 +526,14 @@ func TestClickAtPointInvalidInputs(t *testing.T) { } func TestSIMInput(t *testing.T) { - device, err := NewAndroidDevice( - option.WithSerialNumber(""), + device, err := NewIOSDevice( + option.WithUDID(""), ) if err != nil { t.Fatal(err) } - driver, err := NewUIA2Driver(device) + driver, err := NewWDADriver(device) if err != nil { t.Fatal(err) } From 62dd8e51c48cda2f0c08e7fa62cbb616ff0f2f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 6 Aug 2025 15:21:34 +0800 Subject: [PATCH 11/27] fix test --- .../uitest/android_touch_simulator_test.go | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index e2afa7d8..cec638bb 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -602,41 +602,3 @@ func TestStepMultipleSIMActions(t *testing.T) { t.Logf("Successfully executed multiple SIM actions test") } - -func TestStepMultipleSIMAIOSctions(t *testing.T) { - // 创建包含多个SIM操作的测试用例 - testCase := &hrp.TestCase{ - Config: hrp.NewConfig("多个SIM操作组合测试").SetIOS(option.WithUDID("")), - TestSteps: []hrp.IStep{ - hrp.NewStep("组合SIM操作测试"). - Android(). - SIMClickAtPoint(0.3, 0.3). // 点击屏幕中心 - Sleep(1). // 等待1秒 - SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0). // 向上滑动 - Sleep(0.5). // 等待0.5秒 - SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0). // 在区域内向下滑动 - Sleep(0.5). // 等待0.5秒 - SIMSwipeFromPointToPoint(0.1, 0.2, 0.9, 0.5). // 从左到右滑动 - Sleep(0.5). // 等待0.5秒 - SIMInput("测试组合操作 Test Combination 123"), // 仿真输入 - }, - } - - // 运行测试用例 - err := testCase.Dump2JSON("TestStepMultipleSIMActions.json") - if err != nil { - t.Fatalf("Failed to dump test case: %v", err) - } - defer func() { - // 清理生成的文件 - _ = os.Remove("TestStepMultipleSIMActions.json") - }() - - // 执行测试用例 - err = hrp.NewRunner(t).Run(testCase) - if err != nil { - t.Errorf("Test case failed: %v", err) - } - - t.Logf("Successfully executed multiple SIM actions test") -} From 206d3bc487341f662ca730e981f3b7935a9ba106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 6 Aug 2025 15:27:52 +0800 Subject: [PATCH 12/27] fix android name --- uixt/touch_simulator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uixt/touch_simulator_test.go b/uixt/touch_simulator_test.go index 9cf2c1d9..8e76c003 100644 --- a/uixt/touch_simulator_test.go +++ b/uixt/touch_simulator_test.go @@ -72,7 +72,7 @@ func ParseTouchEvents(data string) ([]types.TouchEvent, error) { return events, nil } -func TestAndroidTouchByEvents(t *testing.T) { +func TestIOSTouchByEvents(t *testing.T) { device, err := NewIOSDevice( option.WithUDID(""), ) From 42c4ffba8a5fccdc9ad3b081d1659414de33650a Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 15:57:06 +0800 Subject: [PATCH 13/27] fix: adb shell pull --- uixt/android_driver_adb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 290e68a9..d29ab91c 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/funplugin/myexec" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/utf7" @@ -708,12 +707,13 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { } pointRes := ConvertPoints(ad.Device.Logcat.logs) // 没有解析到打点日志，走兜底逻辑 + pointRes = []ExportPoint{} if len(pointRes) == 0 { log.Info().Msg("action log is null, use action file >>>") actionLogDirPath := config.GetConfig().ActionLogDirPath() files := []string{} actionLogRegStr := `.*data_\d+\.txt` - myexec.RunCommand("adb", "-s", ad.Device.Serial(), "pull", config.DeviceActionLogFilePath, actionLogDirPath) + ad.Device.PullFolder(config.DeviceActionLogFilePath, actionLogDirPath) err = filepath.Walk(actionLogDirPath, func(path string, info fs.FileInfo, err error) error { // 只是需要日志文件 if ok, _ := regexp.MatchString(actionLogRegStr, path); ok { From 2e2f1d8b26ca28e1cf86000c4a2c730aba5eec65 Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 19:17:47 +0800 Subject: [PATCH 14/27] fix: pull folder --- uixt/android_driver_adb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index d29ab91c..290e68a9 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -24,6 +24,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/httprunner/funplugin/myexec" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/utf7" @@ -707,13 +708,12 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { } pointRes := ConvertPoints(ad.Device.Logcat.logs) // 没有解析到打点日志，走兜底逻辑 - pointRes = []ExportPoint{} if len(pointRes) == 0 { log.Info().Msg("action log is null, use action file >>>") actionLogDirPath := config.GetConfig().ActionLogDirPath() files := []string{} actionLogRegStr := `.*data_\d+\.txt` - ad.Device.PullFolder(config.DeviceActionLogFilePath, actionLogDirPath) + myexec.RunCommand("adb", "-s", ad.Device.Serial(), "pull", config.DeviceActionLogFilePath, actionLogDirPath) err = filepath.Walk(actionLogDirPath, func(path string, info fs.FileInfo, err error) error { // 只是需要日志文件 if ok, _ := regexp.MatchString(actionLogRegStr, path); ok { From 2060305808bc883118df3601f89dab9948db4384 Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Wed, 6 Aug 2025 19:39:05 +0800 Subject: [PATCH 15/27] fix: gadb pull --- uixt/android_driver_adb.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 290e68a9..f0670644 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/funplugin/myexec" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/internal/utf7" @@ -713,7 +712,7 @@ func (ad *ADBDriver) StopCaptureLog() (result interface{}, err error) { actionLogDirPath := config.GetConfig().ActionLogDirPath() files := []string{} actionLogRegStr := `.*data_\d+\.txt` - myexec.RunCommand("adb", "-s", ad.Device.Serial(), "pull", config.DeviceActionLogFilePath, actionLogDirPath) + ad.Device.PullFolder(config.DeviceActionLogFilePath, actionLogDirPath) err = filepath.Walk(actionLogDirPath, func(path string, info fs.FileInfo, err error) error { // 只是需要日志文件 if ok, _ := regexp.MatchString(actionLogRegStr, path); ok { From a921e7b7c2b623776d8bc4085c414fd9b70e1738 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Fri, 8 Aug 2025 17:23:49 +0800 Subject: [PATCH 16/27] fix: unittest --- internal/version/VERSION | 2 +- uixt/touch_simulator_test.go | 36 ------------------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 77429189..3c159c22 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250806 +v5.0.0-250808 diff --git a/uixt/touch_simulator_test.go b/uixt/touch_simulator_test.go index 8e76c003..33109710 100644 --- a/uixt/touch_simulator_test.go +++ b/uixt/touch_simulator_test.go @@ -137,42 +137,6 @@ func TestIOSTouchByEvents(t *testing.T) { t.Logf("Successfully executed touch events: %d events processed", len(events)) } -func TestIOSTouchByEvents(t *testing.T) { - driver := setupWDADriverExt(t) - - // Example touch event data as provided - touchEventData := `1752649131556,401.20703,1191.3164,2,1.0,0.03529412,400.20703,400.3164,111586196,111586196,1,0,0 -1752649131595,402.913,1185.0792,2,1.0,0.039215688,300.913,300.0792,111586196,111586236,1,0,2 -1752649131612,410.60825,1164.3806,2,1.0,0.03529412,250.60825,250.3806,111586196,111586250,1,0,2 -1752649131907,709.1758,523.34766,2,1.0,0.03529412,200.1758,200.34766,111586196,111586546,1,0,1` - - // Parse touch events - events, err := ParseTouchEvents(touchEventData) - if err != nil { - t.Fatalf("ParseTouchEvents failed: %v", err) - } - - // Check first event - firstEvent := events[0] - if firstEvent.Action != 0 { // ACTION_DOWN - t.Errorf("Expected first event action to be 0 (ACTION_DOWN), got %d", firstEvent.Action) - } - - // Check last event - lastEvent := events[len(events)-1] - if lastEvent.Action != 1 { // ACTION_UP - t.Errorf("Expected last event action to be 1 (ACTION_UP), got %d", lastEvent.Action) - } - - // Use TouchByEvents with parsed events - err = driver.IDriver.(*WDADriver).TouchByEvents(events) - if err != nil { - t.Fatalf("TouchByEvents failed: %v", err) - } - - t.Logf("Successfully executed touch events: %d events processed", len(events)) -} - func TestTouchEventParsing(t *testing.T) { // Test single touch event parsing singleEventData := "1752646457403,456.78418,1574.0195,7,1.0,0.016666668,504.78418,1721.0195,924451292,924451292,1,0,0" From 56ba52ed31693a9c60ee76caf41e4b8d10163119 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Sat, 9 Aug 2025 09:44:05 +0800 Subject: [PATCH 17/27] feat: enhance sleep functionality with start time support --- internal/version/VERSION | 2 +- uixt/driver_utils.go | 5 +- uixt/mcp_tools_utility.go | 87 +++++++++--- uixt/mcp_tools_utility_test.go | 240 +++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 25 deletions(-) create mode 100644 uixt/mcp_tools_utility_test.go diff --git a/internal/version/VERSION b/internal/version/VERSION index 3c159c22..799592cc 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250808 +v5.0.0-250809 diff --git a/uixt/driver_utils.go b/uixt/driver_utils.go index 7538297e..b643d582 100644 --- a/uixt/driver_utils.go +++ b/uixt/driver_utils.go @@ -284,8 +284,9 @@ func getSimulationDuration(params []float64) (milliseconds int64) { return 0 } -// sleepStrict sleeps strict duration with given params -// startTime is used to correct sleep duration caused by process time +// sleepStrict sleeps for strict duration with optional start time correction +// If startTime is zero, acts as normal context-aware sleep +// If startTime is provided, corrects sleep duration by subtracting elapsed time // ctx allows for cancellation during sleep func sleepStrict(ctx context.Context, startTime time.Time, strictMilliseconds int64) { var elapsed int64 diff --git a/uixt/mcp_tools_utility.go b/uixt/mcp_tools_utility.go index 4515edf4..79f49bac 100644 --- a/uixt/mcp_tools_utility.go +++ b/uixt/mcp_tools_utility.go @@ -15,7 +15,29 @@ import ( "github.com/httprunner/httprunner/v5/uixt/option" ) -// ToolSleep implements the sleep tool call. +// extractStartTimeMs extracts start_time_ms from MCP request arguments +// Returns time.Time (zero if not provided) and any conversion error +func extractStartTimeMs(request mcp.CallToolRequest) (time.Time, error) { + startTimeMs, ok := request.GetArguments()["start_time_ms"] + if !ok || startTimeMs == nil { + return time.Time{}, nil // Return zero time for normal sleep + } + + var ms int64 + switch v := startTimeMs.(type) { + case float64: + ms = int64(v) + case int64: + ms = v + case int: + ms = int64(v) + default: + return time.Time{}, fmt.Errorf("invalid start_time_ms type: %T", v) + } + + return time.UnixMilli(ms), nil +} + type ToolSleep struct { // Return data fields - these define the structure of data returned by this tool Seconds float64 `json:"seconds" desc:"Duration in seconds that was slept"` @@ -33,6 +55,7 @@ func (t *ToolSleep) Description() string { func (t *ToolSleep) Options() []mcp.ToolOption { return []mcp.ToolOption{ mcp.WithNumber("seconds", mcp.Description("Number of seconds to sleep")), + mcp.WithNumber("start_time_ms", mcp.Description("Start time as Unix milliseconds for strict sleep calculation")), } } @@ -70,16 +93,15 @@ func (t *ToolSleep) Implement() server.ToolHandlerFunc { return nil, fmt.Errorf("unsupported sleep duration type: %T", v) } - // Use context-aware sleep instead of blocking time.Sleep - select { - case <-time.After(duration): - // Normal completion - case <-ctx.Done(): - // Interrupted by context cancellation (interrupt signal, timeout, time limit) - log.Info().Msg("sleep interrupted by context cancellation") - // Don't return error - let the upper layer handle timeout/time limit logic + // Extract start_time_ms and use sleepStrict for unified sleep logic + startTime, err := extractStartTimeMs(request) + if err != nil { + return nil, err } + milliseconds := int64(actualSeconds * 1000) + sleepStrict(ctx, startTime, milliseconds) + message := fmt.Sprintf("Successfully slept for %v seconds", actualSeconds) returnData := ToolSleep{ Seconds: actualSeconds, @@ -91,9 +113,24 @@ func (t *ToolSleep) Implement() server.ToolHandlerFunc { } func (t *ToolSleep) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { - arguments := map[string]any{ - "seconds": action.Params, + arguments := map[string]any{} + + var seconds float64 + if param, ok := action.Params.(json.Number); ok { + seconds, _ = param.Float64() + arguments["seconds"] = seconds + } else if param, ok := action.Params.(int64); ok { + seconds = float64(param) + arguments["seconds"] = seconds + } else if sleepConfig, ok := action.Params.(SleepConfig); ok { + // When startTime is provided, pass both seconds and startTime + seconds = sleepConfig.Seconds + arguments["seconds"] = seconds + arguments["start_time_ms"] = sleepConfig.StartTime.UnixMilli() + } else { + return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep params: %v", action.Params) } + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil } @@ -115,6 +152,7 @@ func (t *ToolSleepMS) Description() string { func (t *ToolSleepMS) Options() []mcp.ToolOption { return []mcp.ToolOption{ mcp.WithNumber("milliseconds", mcp.Description("Number of milliseconds to sleep")), + mcp.WithNumber("start_time_ms", mcp.Description("Start time as Unix milliseconds for strict sleep calculation")), } } @@ -152,16 +190,14 @@ func (t *ToolSleepMS) Implement() server.ToolHandlerFunc { return nil, fmt.Errorf("unsupported sleep duration type: %T", v) } - // Use context-aware sleep instead of blocking time.Sleep - select { - case <-time.After(duration): - // Normal completion - case <-ctx.Done(): - // Interrupted by context cancellation (interrupt signal, timeout, time limit) - log.Info().Msg("sleep interrupted by context cancellation") - // Don't return error - let the upper layer handle timeout/time limit logic + // Extract start_time_ms and use sleepStrict for unified sleep logic + startTime, err := extractStartTimeMs(request) + if err != nil { + return nil, err } + sleepStrict(ctx, startTime, actualMilliseconds) + message := fmt.Sprintf("Successfully slept for %d milliseconds", actualMilliseconds) returnData := ToolSleepMS{ Milliseconds: actualMilliseconds, @@ -173,17 +209,24 @@ func (t *ToolSleepMS) Implement() server.ToolHandlerFunc { } func (t *ToolSleepMS) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + arguments := map[string]any{} + var milliseconds int64 if param, ok := action.Params.(json.Number); ok { milliseconds, _ = param.Int64() + arguments["milliseconds"] = milliseconds } else if param, ok := action.Params.(int64); ok { milliseconds = param + arguments["milliseconds"] = milliseconds + } else if sleepConfig, ok := action.Params.(SleepConfig); ok { + // When startTime is provided, pass both milliseconds and startTime + milliseconds = sleepConfig.Milliseconds + arguments["milliseconds"] = milliseconds + arguments["start_time_ms"] = sleepConfig.StartTime.UnixMilli() } else { return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep ms params: %v", action.Params) } - arguments := map[string]any{ - "milliseconds": milliseconds, - } + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil } diff --git a/uixt/mcp_tools_utility_test.go b/uixt/mcp_tools_utility_test.go new file mode 100644 index 00000000..9e53ac2c --- /dev/null +++ b/uixt/mcp_tools_utility_test.go @@ -0,0 +1,240 @@ +package uixt + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/httprunner/httprunner/v5/uixt/option" +) + +func TestToolSleep_ConvertActionToCallToolRequest(t *testing.T) { + tool := &ToolSleep{} + + tests := []struct { + name string + action option.MobileAction + expectedArgs map[string]any + shouldError bool + }{ + { + name: "json.Number parameter", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: json.Number("3.5"), + }, + expectedArgs: map[string]any{"seconds": float64(3.5)}, + shouldError: false, + }, + { + name: "int64 parameter", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: int64(5), + }, + expectedArgs: map[string]any{"seconds": float64(5)}, + shouldError: false, + }, + { + name: "SleepConfig with startTime", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: SleepConfig{ + StartTime: time.UnixMilli(1691234567890), + Seconds: 2.5, + }, + }, + expectedArgs: map[string]any{ + "seconds": 2.5, + "start_time_ms": int64(1691234567890), + }, + shouldError: false, + }, + { + name: "invalid parameter type", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: "invalid", + }, + expectedArgs: nil, + shouldError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request, err := tool.ConvertActionToCallToolRequest(tt.action) + + if tt.shouldError { + assert.Error(t, err) + } else { + require.NoError(t, err) + args := request.GetArguments() + for key, expectedValue := range tt.expectedArgs { + assert.Equal(t, expectedValue, args[key], "Argument %s mismatch", key) + } + } + }) + } +} + +func TestToolSleepMS_ConvertActionToCallToolRequest(t *testing.T) { + tool := &ToolSleepMS{} + + tests := []struct { + name string + action option.MobileAction + expectedArgs map[string]any + shouldError bool + }{ + { + name: "json.Number parameter", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: json.Number("1500"), + }, + expectedArgs: map[string]any{"milliseconds": int64(1500)}, + shouldError: false, + }, + { + name: "int64 parameter", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: int64(2000), + }, + expectedArgs: map[string]any{"milliseconds": int64(2000)}, + shouldError: false, + }, + { + name: "SleepConfig with startTime", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: SleepConfig{ + StartTime: time.UnixMilli(1691234567890), + Milliseconds: 3000, + }, + }, + expectedArgs: map[string]any{ + "milliseconds": int64(3000), + "start_time_ms": int64(1691234567890), + }, + shouldError: false, + }, + { + name: "invalid parameter type", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: "invalid", + }, + expectedArgs: nil, + shouldError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request, err := tool.ConvertActionToCallToolRequest(tt.action) + + if tt.shouldError { + assert.Error(t, err) + } else { + require.NoError(t, err) + args := request.GetArguments() + for key, expectedValue := range tt.expectedArgs { + assert.Equal(t, expectedValue, args[key], "Argument %s mismatch", key) + } + } + }) + } +} + +func TestSleepStrictTiming(t *testing.T) { + // Test that strict sleep properly adjusts for elapsed time + startTime := time.Now() + + // Simulate some processing time + time.Sleep(50 * time.Millisecond) + + ctx := context.Background() + + // Test sleepStrict with the start time + testStart := time.Now() + sleepStrict(ctx, startTime, 200) // 200ms total duration + actualElapsed := time.Since(testStart) + + // Should sleep approximately 150ms (200ms - 50ms already elapsed) + // Allow some tolerance for timing variations + expectedSleep := 150 * time.Millisecond + assert.Greater(t, actualElapsed, expectedSleep/2, "Sleep too short") + assert.Less(t, actualElapsed, expectedSleep*2, "Sleep too long") +} + +func TestSleepCancellation(t *testing.T) { + // Test that sleep respects context cancellation + ctx, cancel := context.WithCancel(context.Background()) + + // Cancel after 50ms + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + start := time.Now() + sleepStrict(ctx, time.Time{}, 500) // Try to sleep 500ms + elapsed := time.Since(start) + + // Should be cancelled after ~50ms, not sleep full 500ms + assert.Less(t, elapsed, 200*time.Millisecond, "Sleep was not properly cancelled") +} + +func TestSleepStrictWithZeroTime(t *testing.T) { + // Test sleepStrict behaves like normal sleep when startTime is zero + ctx := context.Background() + + start := time.Now() + sleepStrict(ctx, time.Time{}, 100) // 100ms, no start time + elapsed := time.Since(start) + + // Should sleep full duration + expectedSleep := 100 * time.Millisecond + assert.Greater(t, elapsed, expectedSleep/2, "Sleep too short") + assert.Less(t, elapsed, expectedSleep*2, "Sleep too long") +} + +func TestSleepStrictWithPastStartTime(t *testing.T) { + // Test sleepStrict skips sleep when elapsed time exceeds duration + startTime := time.Now().Add(-300 * time.Millisecond) // 300ms ago + ctx := context.Background() + + start := time.Now() + sleepStrict(ctx, startTime, 200) // Want 200ms total, but 300ms already elapsed + elapsed := time.Since(start) + + // Should skip sleep entirely + assert.Less(t, elapsed, 50*time.Millisecond, "Should have skipped sleep") +} + +func TestJsonNumberHandling(t *testing.T) { + // Test that json.Number is correctly handled in different scenarios + + // Test float json.Number + floatNumber := json.Number("3.14") + floatVal, err := floatNumber.Float64() + assert.NoError(t, err) + assert.Equal(t, 3.14, floatVal) + + // Test int json.Number + intNumber := json.Number("1500") + intVal, err := intNumber.Int64() + assert.NoError(t, err) + assert.Equal(t, int64(1500), intVal) + + // Test invalid json.Number + invalidNumber := json.Number("invalid") + _, err = invalidNumber.Float64() + assert.Error(t, err) +} From 25f9510de12afa4fe2d549ad0d591f960f16a954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 11 Aug 2025 21:14:21 +0800 Subject: [PATCH 18/27] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=89=AA=E8=B4=B4=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- step.go | 1 + step_ui.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index a36de551..333d36aa 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250802 +v5.0.0-250811 diff --git a/step.go b/step.go index 636d497b..6e0f23cf 100644 --- a/step.go +++ b/step.go @@ -67,6 +67,7 @@ type ActionResult struct { Plannings []*uixt.PlanningExecutionResult `json:"plannings,omitempty"` // store planning results for start_to_goal actions, which contains multiple sub-actions AIResult *uixt.AIExecutionResult `json:"ai_result,omitempty"` // store unified AI execution result for ai_query/ai_action/ai_assert actions uixt.SessionData // store session data for other actions besides start_to_goal + ExtraData interface{} `json:"extra_data,omitempty"` } // one testcase contains one or multiple steps diff --git a/step_ui.go b/step_ui.go index ae4a6233..c40ae555 100644 --- a/step_ui.go +++ b/step_ui.go @@ -1043,6 +1043,22 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err continue } + if action.Method == option.ACTION_StartToGoal { + planningResults, err := uiDriver.StartToGoal(ctx, + action.Params.(string), action.GetOptions()...) + actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() + actionResult.Plannings = planningResults + stepResult.Actions = append(stepResult.Actions, actionResult) + if err != nil { + actionResult.Error = err.Error() + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.MobileUIDriverError, err.Error()) + } + return stepResult, err + } + continue + } + // handle AI operations (ai_action, ai_query, ai_assert) with unified result storage if action.Method == option.ACTION_AIAction || action.Method == option.ACTION_Query || action.Method == option.ACTION_AIAssert { var aiResult *uixt.AIExecutionResult @@ -1070,6 +1086,19 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err } continue } + if action.Method == option.ACTION_GetPasteboard { + content, err := uiDriver.GetPasteboard() + if err != nil { + actionResult.Error = err.Error() + if !code.IsErrorPredefined(err) { + err = errors.Wrap(code.MobileUIDriverError, err.Error()) + } + return stepResult, err + } + actionResult.ExtraData = content + stepResult.Actions = append(stepResult.Actions, actionResult) + continue + } // handle other non-AI actions sessionData, err := uiDriver.ExecuteAction(ctx, action) From 533f4d06b615161aa0e885993f84b947a09f6bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 11 Aug 2025 21:24:16 +0800 Subject: [PATCH 19/27] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=89=AA=E8=B4=B4=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- step_ui.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/step_ui.go b/step_ui.go index c40ae555..43e4c2e1 100644 --- a/step_ui.go +++ b/step_ui.go @@ -1043,22 +1043,6 @@ func runStepMobileUI(s *SessionRunner, step IStep) (stepResult *StepResult, err continue } - if action.Method == option.ACTION_StartToGoal { - planningResults, err := uiDriver.StartToGoal(ctx, - action.Params.(string), action.GetOptions()...) - actionResult.Elapsed = time.Since(actionStartTime).Milliseconds() - actionResult.Plannings = planningResults - stepResult.Actions = append(stepResult.Actions, actionResult) - if err != nil { - actionResult.Error = err.Error() - if !code.IsErrorPredefined(err) { - err = errors.Wrap(code.MobileUIDriverError, err.Error()) - } - return stepResult, err - } - continue - } - // handle AI operations (ai_action, ai_query, ai_assert) with unified result storage if action.Method == option.ACTION_AIAction || action.Method == option.ACTION_Query || action.Method == option.ACTION_AIAssert { var aiResult *uixt.AIExecutionResult From 2523be5880422ed95e6693e7375ef07d212916c8 Mon Sep 17 00:00:00 2001 From: "huangbin.beal" Date: Mon, 11 Aug 2025 21:37:46 +0800 Subject: [PATCH 20/27] fix: interval option for swipe to tap text --- internal/version/VERSION | 2 +- uixt/driver_ext_swipe.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 77429189..333d36aa 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250806 +v5.0.0-250811 diff --git a/uixt/driver_ext_swipe.go b/uixt/driver_ext_swipe.go index f23f2fe1..a8f167af 100644 --- a/uixt/driver_ext_swipe.go +++ b/uixt/driver_ext_swipe.go @@ -100,8 +100,7 @@ func (dExt *XTDriver) SwipeToTapTexts(texts []string, opts ...option.ActionOptio } log.Info().Strs("texts", texts).Msg("swipe to tap texts") - opts = append(opts, option.WithMatchOne(true), option.WithRegex(true), option.WithInterval(1)) - + opts = append([]option.ActionOption{option.WithMatchOne(true), option.WithRegex(true), option.WithInterval(1)}, opts...) // Remove identifier for swipe operations to avoid WDA/UIA2 logging actionOptions := option.NewActionOptions(opts...) actionOptions.Identifier = "" From 1fdf85a1c6cc771a467cd9e3ae4a644e2f26123d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Mon, 11 Aug 2025 22:24:29 +0800 Subject: [PATCH 21/27] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/ai/wings_service.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 35fa77db..7e4f29f5 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -412,6 +412,9 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ } defer resp.Body.Close() + logID := resp.Header.Get("X-Tt-Logid") + log.Info().Str("step_text", request.StepText).Str("log_id", logID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") + // Read response body responseBody, err := io.ReadAll(resp.Body) if err != nil { From 1253d5848d25564d50703b18a8e6902ad507e85b Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Tue, 12 Aug 2025 14:49:47 +0800 Subject: [PATCH 22/27] refactor: unify float64 conversion logic in ToolSleep and ToolSleepMS, enhance error logging --- internal/builtin/utils.go | 31 +++--------- internal/version/VERSION | 2 +- uixt/mcp_tools_utility.go | 87 +++++++++++----------------------- uixt/mcp_tools_utility_test.go | 45 ++++++++++++++++++ 4 files changed, 80 insertions(+), 85 deletions(-) diff --git a/internal/builtin/utils.go b/internal/builtin/utils.go index 5e378598..3beb0679 100644 --- a/internal/builtin/utils.go +++ b/internal/builtin/utils.go @@ -217,6 +217,8 @@ func Interface2Float64(i interface{}) (float64, error) { case string: // e.g. "1", "0.5" floatVar, err := strconv.ParseFloat(v, 64) if err != nil { + log.Error().Err(err).Str("value", v). + Msg("convert string to float64 failed") return 0, err } return floatVar, nil @@ -226,6 +228,10 @@ func Interface2Float64(i interface{}) (float64, error) { if ok { return value.Float64() } + + // Log error for unsupported types + log.Error().Interface("value", i).Type("type", i). + Msg("convert float64 failed") return 0, errors.New("failed to convert interface to float64") } @@ -334,29 +340,6 @@ func IsZeroFloat64(f float64) bool { return math.Abs(f) < threshold } -func ConvertToFloat64(val interface{}) (float64, error) { - switch v := val.(type) { - case float64: - return v, nil - case int: - return float64(v), nil - case int64: - return float64(v), nil - case string: - f, err := strconv.ParseFloat(v, 64) - if err != nil { - log.Error().Err(err).Str("value", v). - Msg("convert string to float64 failed") - return 0, err - } - return f, nil - default: - log.Error().Interface("value", val).Type("type", val). - Msg("convert float64 failed") - return 0, errors.New("convert float64 error") - } -} - func ConvertToFloat64Slice(val interface{}) ([]float64, error) { if paramsSlice, ok := val.([]float64); ok { return paramsSlice, nil @@ -369,7 +352,7 @@ func ConvertToFloat64Slice(val interface{}) ([]float64, error) { var err error float64Slice := make([]float64, len(paramsSlice)) for i, v := range paramsSlice { - float64Slice[i], err = ConvertToFloat64(v) + float64Slice[i], err = Interface2Float64(v) if err != nil { return nil, err } diff --git a/internal/version/VERSION b/internal/version/VERSION index 333d36aa..b7fffde9 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250811 +v5.0.0-250812 diff --git a/uixt/mcp_tools_utility.go b/uixt/mcp_tools_utility.go index 79f49bac..5b3656d9 100644 --- a/uixt/mcp_tools_utility.go +++ b/uixt/mcp_tools_utility.go @@ -2,9 +2,7 @@ package uixt import ( "context" - "encoding/json" "fmt" - "strconv" "time" "github.com/mark3labs/mcp-go/mcp" @@ -70,28 +68,12 @@ func (t *ToolSleep) Implement() server.ToolHandlerFunc { // Sleep action logic log.Info().Interface("seconds", seconds).Msg("sleeping") - var duration time.Duration - var actualSeconds float64 - switch v := seconds.(type) { - case float64: - actualSeconds = v - duration = time.Duration(v*1000) * time.Millisecond - case int: - actualSeconds = float64(v) - duration = time.Duration(v) * time.Second - case int64: - actualSeconds = float64(v) - duration = time.Duration(v) * time.Second - case string: - s, err := builtin.ConvertToFloat64(v) - if err != nil { - return nil, fmt.Errorf("invalid sleep duration: %v", v) - } - actualSeconds = s - duration = time.Duration(s*1000) * time.Millisecond - default: - return nil, fmt.Errorf("unsupported sleep duration type: %T", v) + // Use Interface2Float64 for unified type conversion + actualSeconds, err := builtin.Interface2Float64(seconds) + if err != nil { + return nil, fmt.Errorf("invalid sleep duration: %v", seconds) } + duration := time.Duration(actualSeconds) * time.Second // Extract start_time_ms and use sleepStrict for unified sleep logic startTime, err := extractStartTimeMs(request) @@ -116,19 +98,19 @@ func (t *ToolSleep) ConvertActionToCallToolRequest(action option.MobileAction) ( arguments := map[string]any{} var seconds float64 - if param, ok := action.Params.(json.Number); ok { - seconds, _ = param.Float64() - arguments["seconds"] = seconds - } else if param, ok := action.Params.(int64); ok { - seconds = float64(param) - arguments["seconds"] = seconds - } else if sleepConfig, ok := action.Params.(SleepConfig); ok { + if sleepConfig, ok := action.Params.(SleepConfig); ok { // When startTime is provided, pass both seconds and startTime seconds = sleepConfig.Seconds arguments["seconds"] = seconds arguments["start_time_ms"] = sleepConfig.StartTime.UnixMilli() } else { - return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep params: %v", action.Params) + // Use builtin.Interface2Float64 for unified parameter handling + var err error + seconds, err = builtin.Interface2Float64(action.Params) + if err != nil { + return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep params: %v", action.Params) + } + arguments["seconds"] = seconds } return BuildMCPCallToolRequest(t.Name(), arguments, action), nil @@ -167,28 +149,13 @@ func (t *ToolSleepMS) Implement() server.ToolHandlerFunc { // Sleep MS action logic log.Info().Interface("milliseconds", milliseconds).Msg("sleeping in milliseconds") - var duration time.Duration - var actualMilliseconds int64 - switch v := milliseconds.(type) { - case float64: - actualMilliseconds = int64(v) - duration = time.Duration(v) * time.Millisecond - case int: - actualMilliseconds = int64(v) - duration = time.Duration(v) * time.Millisecond - case int64: - actualMilliseconds = v - duration = time.Duration(v) * time.Millisecond - case string: - ms, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid sleep duration: %v", v) - } - actualMilliseconds = ms - duration = time.Duration(ms) * time.Millisecond - default: - return nil, fmt.Errorf("unsupported sleep duration type: %T", v) + // Use Interface2Float64 for unified type conversion, then convert to int64 + floatVal, err := builtin.Interface2Float64(milliseconds) + if err != nil { + return nil, fmt.Errorf("invalid sleep duration: %v", milliseconds) } + actualMilliseconds := int64(floatVal) + duration := time.Duration(actualMilliseconds) * time.Millisecond // Extract start_time_ms and use sleepStrict for unified sleep logic startTime, err := extractStartTimeMs(request) @@ -212,19 +179,19 @@ func (t *ToolSleepMS) ConvertActionToCallToolRequest(action option.MobileAction) arguments := map[string]any{} var milliseconds int64 - if param, ok := action.Params.(json.Number); ok { - milliseconds, _ = param.Int64() - arguments["milliseconds"] = milliseconds - } else if param, ok := action.Params.(int64); ok { - milliseconds = param - arguments["milliseconds"] = milliseconds - } else if sleepConfig, ok := action.Params.(SleepConfig); ok { + if sleepConfig, ok := action.Params.(SleepConfig); ok { // When startTime is provided, pass both milliseconds and startTime milliseconds = sleepConfig.Milliseconds arguments["milliseconds"] = milliseconds arguments["start_time_ms"] = sleepConfig.StartTime.UnixMilli() } else { - return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep ms params: %v", action.Params) + // Use builtin.Interface2Float64 for unified parameter handling, then convert to int64 + floatVal, err := builtin.Interface2Float64(action.Params) + if err != nil { + return mcp.CallToolRequest{}, fmt.Errorf("invalid sleep ms params: %v", action.Params) + } + milliseconds = int64(floatVal) + arguments["milliseconds"] = milliseconds } return BuildMCPCallToolRequest(t.Name(), arguments, action), nil diff --git a/uixt/mcp_tools_utility_test.go b/uixt/mcp_tools_utility_test.go index 9e53ac2c..733114c0 100644 --- a/uixt/mcp_tools_utility_test.go +++ b/uixt/mcp_tools_utility_test.go @@ -30,6 +30,15 @@ func TestToolSleep_ConvertActionToCallToolRequest(t *testing.T) { expectedArgs: map[string]any{"seconds": float64(3.5)}, shouldError: false, }, + { + name: "float64 parameter", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: float64(5.2), + }, + expectedArgs: map[string]any{"seconds": float64(5.2)}, + shouldError: false, + }, { name: "int64 parameter", action: option.MobileAction{ @@ -63,6 +72,24 @@ func TestToolSleep_ConvertActionToCallToolRequest(t *testing.T) { expectedArgs: nil, shouldError: true, }, + { + name: "json.Number with integer value", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: json.Number("10"), + }, + expectedArgs: map[string]any{"seconds": float64(10)}, + shouldError: false, + }, + { + name: "json.Number with decimal value", + action: option.MobileAction{ + Method: option.ACTION_Sleep, + Params: json.Number("1.25"), + }, + expectedArgs: map[string]any{"seconds": float64(1.25)}, + shouldError: false, + }, } for _, tt := range tests { @@ -109,6 +136,15 @@ func TestToolSleepMS_ConvertActionToCallToolRequest(t *testing.T) { expectedArgs: map[string]any{"milliseconds": int64(2000)}, shouldError: false, }, + { + name: "float64 parameter", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: float64(2500.7), + }, + expectedArgs: map[string]any{"milliseconds": int64(2500)}, + shouldError: false, + }, { name: "SleepConfig with startTime", action: option.MobileAction{ @@ -124,6 +160,15 @@ func TestToolSleepMS_ConvertActionToCallToolRequest(t *testing.T) { }, shouldError: false, }, + { + name: "json.Number with decimal value", + action: option.MobileAction{ + Method: option.ACTION_SleepMS, + Params: json.Number("1234.56"), + }, + expectedArgs: map[string]any{"milliseconds": int64(1234)}, + shouldError: false, + }, { name: "invalid parameter type", action: option.MobileAction{ From 9bf31f643d43b81e8bd677b623fef8b06e08e7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Tue, 12 Aug 2025 16:45:53 +0800 Subject: [PATCH 23/27] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=B8=85?= =?UTF-8?q?=E7=A9=BA/=E5=AF=BC=E5=85=A5=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/driver_session.go | 6 +- uixt/image_utils.go | 85 +++++++++++++++++ uixt/mcp_server.go | 4 + uixt/mcp_tools_device.go | 200 +++++++++++++++++++++++++++++++++++++++ uixt/option/action.go | 4 + 5 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 uixt/image_utils.go diff --git a/uixt/driver_session.go b/uixt/driver_session.go index 35f5cc57..5a837bb1 100644 --- a/uixt/driver_session.go +++ b/uixt/driver_session.go @@ -265,8 +265,10 @@ func (s *DriverSession) Request(method string, urlStr string, rawBody []byte, op logger = log.Debug().Bool("success", true) } - logger = logger.Str("logid", logid).Str("request_method", method).Str("request_url", rawURL). - Str("request_body", string(rawBody)) + logger = logger.Str("logid", logid).Str("request_method", method).Str("request_url", rawURL) + if len(rawBody) < 1024 { + logger = logger.Str("request_body", string(rawBody)) + } if !driverResult.RequestTime.IsZero() { logger = logger.Int64("request_time", driverResult.RequestTime.UnixMilli()) } diff --git a/uixt/image_utils.go b/uixt/image_utils.go new file mode 100644 index 00000000..338f895a --- /dev/null +++ b/uixt/image_utils.go @@ -0,0 +1,85 @@ +package uixt + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/rs/zerolog/log" +) + +// DetectAndRenameImageFile examines the file content to determine its image type +// and renames the file with the appropriate extension (.jpg, .png, etc.) +func DetectAndRenameImageFile(filePath string) (string, error) { + // Open the file + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("failed to open file for type detection: %v", err) + } + defer file.Close() + + // Read the first 512 bytes to detect content type + buffer := make([]byte, 512) + _, err = file.Read(buffer) + if err != nil && err != io.EOF { + return "", fmt.Errorf("failed to read file for type detection: %v", err) + } + + // Reset file pointer + _, err = file.Seek(0, 0) + if err != nil { + return "", fmt.Errorf("failed to reset file pointer: %v", err) + } + + // Detect content type + contentType := http.DetectContentType(buffer) + log.Info().Str("filePath", filePath).Str("contentType", contentType).Msg("Detected content type") + + // Determine file extension based on content type + var extension string + switch { + case strings.Contains(contentType, "image/jpeg"): + extension = ".jpg" + case strings.Contains(contentType, "image/png"): + extension = ".png" + case strings.Contains(contentType, "image/gif"): + extension = ".gif" + case strings.Contains(contentType, "image/webp"): + extension = ".webp" + case strings.Contains(contentType, "image/bmp"): + extension = ".bmp" + case strings.Contains(contentType, "image/tiff"): + extension = ".tiff" + case strings.Contains(contentType, "image/svg+xml"): + extension = ".svg" + default: + // Default to jpg if we can't determine the type but it's still an image + if strings.Contains(contentType, "image/") { + extension = ".jpg" + } else { + return filePath, fmt.Errorf("not a recognized image type: %s", contentType) + } + } + + // Create new file path with extension + dir := filepath.Dir(filePath) + base := filepath.Base(filePath) + newFilePath := filepath.Join(dir, base+extension) + + // If the file already has the correct extension, just return it + if filePath == newFilePath { + return filePath, nil + } + + // Rename the file + err = os.Rename(filePath, newFilePath) + if err != nil { + return "", fmt.Errorf("failed to rename file: %v", err) + } + + log.Info().Str("oldPath", filePath).Str("newPath", newFilePath).Msg("Renamed image file with proper extension") + return newFilePath, nil +} diff --git a/uixt/mcp_server.go b/uixt/mcp_server.go index 92f0b1be..bc4be2bc 100644 --- a/uixt/mcp_server.go +++ b/uixt/mcp_server.go @@ -133,6 +133,10 @@ func (s *MCPServer4XTDriver) registerTools() { s.registerTool(&ToolGetScreenSize{}) s.registerTool(&ToolGetSource{}) + // Image Tools + s.registerTool(&ToolPushImage{}) + s.registerTool(&ToolClearImage{}) + // Utility Tools s.registerTool(&ToolSleep{}) s.registerTool(&ToolSleepMS{}) diff --git a/uixt/mcp_tools_device.go b/uixt/mcp_tools_device.go index e09dd906..43fd5fe1 100644 --- a/uixt/mcp_tools_device.go +++ b/uixt/mcp_tools_device.go @@ -3,6 +3,7 @@ package uixt import ( "context" "fmt" + "os" "github.com/danielpaulus/go-ios/ios" "github.com/mark3labs/mcp-go/mcp" @@ -216,3 +217,202 @@ func (t *ToolScreenRecord) Implement() server.ToolHandlerFunc { func (t *ToolScreenRecord) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { return BuildMCPCallToolRequest(t.Name(), map[string]any{}, action), nil } + +// ToolPushImage implements the push_image tool call. +type ToolPushImage struct { + // Return data fields - these define the structure of data returned by this tool + ImagePath string `json:"imagePath" desc:"Path of the image that was pushed"` + ImageUrl string `json:"imageUrl,omitempty" desc:"URL of the image that was downloaded and pushed (if applicable)"` + Cleared bool `json:"cleared,omitempty" desc:"Whether images were cleared before pushing (if applicable)"` +} + +func (t *ToolPushImage) Name() option.ActionName { + return option.ACTION_PushImage +} + +func (t *ToolPushImage) Description() string { + return "Push an image to the device's gallery. For Android, the image will be pushed to the DCIM/Camera directory. For iOS, the image will be added to the device's photo album." +} + +func (t *ToolPushImage) Options() []mcp.ToolOption { + return []mcp.ToolOption{ + mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to push image to")), + mcp.WithString("serial", mcp.Description("The device serial number or UDID")), + mcp.WithString("imagePath", mcp.Description("Path to the local image file to push to the device")), + mcp.WithString("imageUrl", mcp.Description("URL of the image to download and push to the device")), + mcp.WithBoolean("cleanup", mcp.Description("Whether to delete the downloaded file after pushing it to the device")), + mcp.WithBoolean("clearBefore", mcp.Description("Whether to clear images before pushing (if applicable)")), + } +} + +func (t *ToolPushImage) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + driverExt, err := setupXTDriver(ctx, request.GetArguments()) + if err != nil { + return nil, err + } + + // Get image path or URL + imagePath, hasPath := request.GetArguments()["imagePath"].(string) + imageUrl, hasUrl := request.GetArguments()["imageUrl"].(string) + cleanup, _ := request.GetArguments()["cleanup"].(bool) + clearBefore, _ := request.GetArguments()["clearBefore"].(bool) + + // Check if we have either path or URL + if (!hasPath || imagePath == "") && (!hasUrl || imageUrl == "") { + return nil, fmt.Errorf("either imagePath or imageUrl is required") + } + + // If we have a URL, download it + downloadedFile := false + if hasUrl && imageUrl != "" { + log.Info().Str("imageUrl", imageUrl).Msg("Downloading image from URL") + downloadedPath, err := DownloadFileByUrl(imageUrl) + if err != nil { + return nil, fmt.Errorf("failed to download image from URL: %v", err) + } + + // Detect image type and rename with proper extension + renamedPath, err := DetectAndRenameImageFile(downloadedPath) + if err != nil { + log.Warn().Err(err).Str("path", downloadedPath).Msg("Failed to detect image type or rename file, using original file") + imagePath = downloadedPath + } else { + imagePath = renamedPath + } + downloadedFile = true + } + + // Clear images before pushing if requested + cleared := false + if clearBefore { + log.Info().Msg("Clearing images before pushing new image") + err := driverExt.IDriver.ClearImages() + if err != nil { + log.Warn().Err(err).Msg("Failed to clear images before pushing, continuing anyway") + } else { + cleared = true + } + } + + // Push the image to the device + err = driverExt.IDriver.PushImage(imagePath) + if err != nil { + // If we downloaded the file and failed to push it, clean up + if downloadedFile && cleanup { + _ = os.Remove(imagePath) + } + return nil, err + } + + // Clean up downloaded file if requested + if downloadedFile && cleanup { + log.Info().Str("imagePath", imagePath).Msg("Cleaning up downloaded image") + _ = os.Remove(imagePath) + } + + message := fmt.Sprintf("Successfully pushed image to device") + returnData := ToolPushImage{ + ImagePath: imagePath, + Cleared: cleared, + } + + // Include URL in response if it was used + if hasUrl && imageUrl != "" { + returnData.ImageUrl = imageUrl + message = fmt.Sprintf("Successfully downloaded and pushed image from %s to device", imageUrl) + } + + // Add cleared info to message if applicable + if cleared { + message = fmt.Sprintf("%s (images cleared before pushing)", message) + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolPushImage) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + arguments := map[string]any{} + + // Handle string param as imageUrl + if imageUrl, ok := action.Params.(string); ok && imageUrl != "" { + arguments["imageUrl"] = imageUrl + } + + // Handle map params with imageUrl or imagePath + if params, ok := action.Params.(map[string]interface{}); ok { + if imageUrl, ok := params["imageUrl"].(string); ok && imageUrl != "" { + arguments["imageUrl"] = imageUrl + } + if imagePath, ok := params["imagePath"].(string); ok && imagePath != "" { + arguments["imagePath"] = imagePath + } + if cleanup, ok := params["cleanup"].(bool); ok { + arguments["cleanup"] = cleanup + } + if clearBefore, ok := params["clearBefore"].(bool); ok { + arguments["clearBefore"] = clearBefore + } + } + + // Handle custom options + if imageUrl, ok := action.ActionOptions.Custom["imageUrl"].(string); ok && imageUrl != "" { + arguments["imageUrl"] = imageUrl + } + if imagePath, ok := action.ActionOptions.Custom["imagePath"].(string); ok && imagePath != "" { + arguments["imagePath"] = imagePath + } + if cleanup, ok := action.ActionOptions.Custom["cleanup"].(bool); ok { + arguments["cleanup"] = cleanup + } + if clearBefore, ok := action.ActionOptions.Custom["clearBefore"].(bool); ok { + arguments["clearBefore"] = clearBefore + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil +} + +// ToolClearImage implements the clear_image tool call. +type ToolClearImage struct { + // Return data fields - these define the structure of data returned by this tool + Success bool `json:"success" desc:"Whether the operation was successful"` +} + +func (t *ToolClearImage) Name() option.ActionName { + return option.ACTION_ClearImage +} + +func (t *ToolClearImage) Description() string { + return "Clear images from the device's gallery. For Android, this will remove all images from the DCIM/Camera directory. For iOS, this will clear the images added through the push_image tool." +} + +func (t *ToolClearImage) Options() []mcp.ToolOption { + return []mcp.ToolOption{ + mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to clear images from")), + mcp.WithString("serial", mcp.Description("The device serial number or UDID")), + } +} + +func (t *ToolClearImage) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + driverExt, err := setupXTDriver(ctx, request.GetArguments()) + if err != nil { + return nil, err + } + + err = driverExt.IDriver.ClearImages() + if err != nil { + return nil, err + } + + message := "Successfully cleared images from device" + returnData := ToolClearImage{Success: true} + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolClearImage) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + return BuildMCPCallToolRequest(t.Name(), map[string]any{}, action), nil +} diff --git a/uixt/option/action.go b/uixt/option/action.go index 7dfd872c..e13aa9f0 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -98,6 +98,10 @@ const ( ACTION_ListAvailableDevices ActionName = "list_available_devices" ACTION_SelectDevice ActionName = "select_device" + // image actions + ACTION_PushImage ActionName = "push_image" + ACTION_ClearImage ActionName = "clear_image" + // custom actions ACTION_SwipeToTapApp ActionName = "swipe_to_tap_app" // swipe left & right to find app and tap ACTION_SwipeToTapText ActionName = "swipe_to_tap_text" // swipe up & down to find text and tap From 16a1ea498973e31391e7c738324b4bf61dba1569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Tue, 12 Aug 2025 17:11:44 +0800 Subject: [PATCH 24/27] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E8=A7=86=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uixt/image_utils.go | 43 ++++++- uixt/mcp_server.go | 6 +- uixt/mcp_tools_device.go | 272 +++++++++++++++++++-------------------- uixt/option/action.go | 6 +- 4 files changed, 178 insertions(+), 149 deletions(-) diff --git a/uixt/image_utils.go b/uixt/image_utils.go index 338f895a..64208188 100644 --- a/uixt/image_utils.go +++ b/uixt/image_utils.go @@ -11,9 +11,9 @@ import ( "github.com/rs/zerolog/log" ) -// DetectAndRenameImageFile examines the file content to determine its image type -// and renames the file with the appropriate extension (.jpg, .png, etc.) -func DetectAndRenameImageFile(filePath string) (string, error) { +// DetectAndRenameImageFile examines the file content to determine its media type +// and renames the file with the appropriate extension (.jpg, .png, .mp4, etc.) +func DetectAndRenameMediaFile(filePath string) (string, error) { // Open the file file, err := os.Open(filePath) if err != nil { @@ -41,6 +41,7 @@ func DetectAndRenameImageFile(filePath string) (string, error) { // Determine file extension based on content type var extension string switch { + // Image types case strings.Contains(contentType, "image/jpeg"): extension = ".jpg" case strings.Contains(contentType, "image/png"): @@ -55,12 +56,42 @@ func DetectAndRenameImageFile(filePath string) (string, error) { extension = ".tiff" case strings.Contains(contentType, "image/svg+xml"): extension = ".svg" + + // Video types + case strings.Contains(contentType, "video/mp4"): + extension = ".mp4" + case strings.Contains(contentType, "video/quicktime"): + extension = ".mov" + case strings.Contains(contentType, "video/x-msvideo"): + extension = ".avi" + case strings.Contains(contentType, "video/x-ms-wmv"): + extension = ".wmv" + case strings.Contains(contentType, "video/x-flv"): + extension = ".flv" + case strings.Contains(contentType, "video/webm"): + extension = ".webm" + case strings.Contains(contentType, "video/x-matroska"): + extension = ".mkv" + default: - // Default to jpg if we can't determine the type but it's still an image + // Check for general image or video types if strings.Contains(contentType, "image/") { - extension = ".jpg" + extension = ".jpg" // Default for unknown image types + } else if strings.Contains(contentType, "video/") { + extension = ".mp4" // Default for unknown video types } else { - return filePath, fmt.Errorf("not a recognized image type: %s", contentType) + // Try to determine from original file extension + origExt := strings.ToLower(filepath.Ext(filePath)) + if origExt == ".mp4" || origExt == ".mov" || origExt == ".avi" || + origExt == ".wmv" || origExt == ".flv" || origExt == ".webm" || origExt == ".mkv" { + extension = origExt + } else if origExt == ".jpg" || origExt == ".jpeg" || origExt == ".png" || + origExt == ".gif" || origExt == ".webp" || origExt == ".bmp" || + origExt == ".tiff" || origExt == ".svg" { + extension = origExt + } else { + return filePath, fmt.Errorf("not a recognized media type: %s", contentType) + } } } diff --git a/uixt/mcp_server.go b/uixt/mcp_server.go index bc4be2bc..9ffae9ab 100644 --- a/uixt/mcp_server.go +++ b/uixt/mcp_server.go @@ -133,9 +133,9 @@ func (s *MCPServer4XTDriver) registerTools() { s.registerTool(&ToolGetScreenSize{}) s.registerTool(&ToolGetSource{}) - // Image Tools - s.registerTool(&ToolPushImage{}) - s.registerTool(&ToolClearImage{}) + // Media Album Tools + s.registerTool(&ToolPushAlbums{}) + s.registerTool(&ToolClearAlbums{}) // Utility Tools s.registerTool(&ToolSleep{}) diff --git a/uixt/mcp_tools_device.go b/uixt/mcp_tools_device.go index 43fd5fe1..5b933290 100644 --- a/uixt/mcp_tools_device.go +++ b/uixt/mcp_tools_device.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "os" + "path/filepath" + "strings" "github.com/danielpaulus/go-ios/ios" "github.com/mark3labs/mcp-go/mcp" @@ -218,135 +220,49 @@ func (t *ToolScreenRecord) ConvertActionToCallToolRequest(action option.MobileAc return BuildMCPCallToolRequest(t.Name(), map[string]any{}, action), nil } -// ToolPushImage implements the push_image tool call. -type ToolPushImage struct { +// ToolPushAlbums implements the push_albums tool call. +type ToolPushAlbums struct { // Return data fields - these define the structure of data returned by this tool - ImagePath string `json:"imagePath" desc:"Path of the image that was pushed"` - ImageUrl string `json:"imageUrl,omitempty" desc:"URL of the image that was downloaded and pushed (if applicable)"` - Cleared bool `json:"cleared,omitempty" desc:"Whether images were cleared before pushing (if applicable)"` + FilePath string `json:"filePath" desc:"Path of the file that was pushed"` + FileUrl string `json:"fileUrl,omitempty" desc:"URL of the file that was downloaded and pushed (if applicable)"` + FileType string `json:"fileType" desc:"Type of the file that was pushed (image or video)"` + Cleared bool `json:"cleared,omitempty" desc:"Whether albums were cleared before pushing (if applicable)"` } -func (t *ToolPushImage) Name() option.ActionName { - return option.ACTION_PushImage +func (t *ToolPushAlbums) Name() option.ActionName { + return option.ACTION_PushAlbums } -func (t *ToolPushImage) Description() string { - return "Push an image to the device's gallery. For Android, the image will be pushed to the DCIM/Camera directory. For iOS, the image will be added to the device's photo album." +func (t *ToolPushAlbums) Description() string { + return "Push a media file (image or video) to the device's gallery. For Android, this will push the file to the DCIM/Camera directory. For iOS, this will add the file to the photo album." } -func (t *ToolPushImage) Options() []mcp.ToolOption { +func (t *ToolPushAlbums) Options() []mcp.ToolOption { return []mcp.ToolOption{ - mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to push image to")), + mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to push media to")), mcp.WithString("serial", mcp.Description("The device serial number or UDID")), - mcp.WithString("imagePath", mcp.Description("Path to the local image file to push to the device")), - mcp.WithString("imageUrl", mcp.Description("URL of the image to download and push to the device")), + mcp.WithString("filePath", mcp.Description("Path to the local media file to push to the device")), + mcp.WithString("fileUrl", mcp.Description("URL of the media file to download and push to the device")), mcp.WithBoolean("cleanup", mcp.Description("Whether to delete the downloaded file after pushing it to the device")), - mcp.WithBoolean("clearBefore", mcp.Description("Whether to clear images before pushing (if applicable)")), + mcp.WithBoolean("clearBefore", mcp.Description("Whether to clear albums before pushing (if applicable)")), } } -func (t *ToolPushImage) Implement() server.ToolHandlerFunc { - return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { - driverExt, err := setupXTDriver(ctx, request.GetArguments()) - if err != nil { - return nil, err - } - - // Get image path or URL - imagePath, hasPath := request.GetArguments()["imagePath"].(string) - imageUrl, hasUrl := request.GetArguments()["imageUrl"].(string) - cleanup, _ := request.GetArguments()["cleanup"].(bool) - clearBefore, _ := request.GetArguments()["clearBefore"].(bool) - - // Check if we have either path or URL - if (!hasPath || imagePath == "") && (!hasUrl || imageUrl == "") { - return nil, fmt.Errorf("either imagePath or imageUrl is required") - } - - // If we have a URL, download it - downloadedFile := false - if hasUrl && imageUrl != "" { - log.Info().Str("imageUrl", imageUrl).Msg("Downloading image from URL") - downloadedPath, err := DownloadFileByUrl(imageUrl) - if err != nil { - return nil, fmt.Errorf("failed to download image from URL: %v", err) - } - - // Detect image type and rename with proper extension - renamedPath, err := DetectAndRenameImageFile(downloadedPath) - if err != nil { - log.Warn().Err(err).Str("path", downloadedPath).Msg("Failed to detect image type or rename file, using original file") - imagePath = downloadedPath - } else { - imagePath = renamedPath - } - downloadedFile = true - } - - // Clear images before pushing if requested - cleared := false - if clearBefore { - log.Info().Msg("Clearing images before pushing new image") - err := driverExt.IDriver.ClearImages() - if err != nil { - log.Warn().Err(err).Msg("Failed to clear images before pushing, continuing anyway") - } else { - cleared = true - } - } - - // Push the image to the device - err = driverExt.IDriver.PushImage(imagePath) - if err != nil { - // If we downloaded the file and failed to push it, clean up - if downloadedFile && cleanup { - _ = os.Remove(imagePath) - } - return nil, err - } - - // Clean up downloaded file if requested - if downloadedFile && cleanup { - log.Info().Str("imagePath", imagePath).Msg("Cleaning up downloaded image") - _ = os.Remove(imagePath) - } - - message := fmt.Sprintf("Successfully pushed image to device") - returnData := ToolPushImage{ - ImagePath: imagePath, - Cleared: cleared, - } - - // Include URL in response if it was used - if hasUrl && imageUrl != "" { - returnData.ImageUrl = imageUrl - message = fmt.Sprintf("Successfully downloaded and pushed image from %s to device", imageUrl) - } - - // Add cleared info to message if applicable - if cleared { - message = fmt.Sprintf("%s (images cleared before pushing)", message) - } - - return NewMCPSuccessResponse(message, &returnData), nil - } -} - -func (t *ToolPushImage) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { +func (t *ToolPushAlbums) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { arguments := map[string]any{} - // Handle string param as imageUrl - if imageUrl, ok := action.Params.(string); ok && imageUrl != "" { - arguments["imageUrl"] = imageUrl + // Handle string param as fileUrl + if fileUrl, ok := action.Params.(string); ok && fileUrl != "" { + arguments["fileUrl"] = fileUrl } - // Handle map params with imageUrl or imagePath + // Handle map params with fileUrl or filePath if params, ok := action.Params.(map[string]interface{}); ok { - if imageUrl, ok := params["imageUrl"].(string); ok && imageUrl != "" { - arguments["imageUrl"] = imageUrl + if fileUrl, ok := params["fileUrl"].(string); ok && fileUrl != "" { + arguments["fileUrl"] = fileUrl } - if imagePath, ok := params["imagePath"].(string); ok && imagePath != "" { - arguments["imagePath"] = imagePath + if filePath, ok := params["filePath"].(string); ok && filePath != "" { + arguments["filePath"] = filePath } if cleanup, ok := params["cleanup"].(bool); ok { arguments["cleanup"] = cleanup @@ -356,45 +272,127 @@ func (t *ToolPushImage) ConvertActionToCallToolRequest(action option.MobileActio } } - // Handle custom options - if imageUrl, ok := action.ActionOptions.Custom["imageUrl"].(string); ok && imageUrl != "" { - arguments["imageUrl"] = imageUrl - } - if imagePath, ok := action.ActionOptions.Custom["imagePath"].(string); ok && imagePath != "" { - arguments["imagePath"] = imagePath - } - if cleanup, ok := action.ActionOptions.Custom["cleanup"].(bool); ok { - arguments["cleanup"] = cleanup - } - if clearBefore, ok := action.ActionOptions.Custom["clearBefore"].(bool); ok { - arguments["clearBefore"] = clearBefore - } - return BuildMCPCallToolRequest(t.Name(), arguments, action), nil } -// ToolClearImage implements the clear_image tool call. -type ToolClearImage struct { +func (t *ToolPushAlbums) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + driverExt, err := setupXTDriver(ctx, request.GetArguments()) + if err != nil { + return nil, err + } + + // Get file path or URL + filePath, hasPath := request.GetArguments()["filePath"].(string) + fileUrl, hasUrl := request.GetArguments()["fileUrl"].(string) + cleanup, _ := request.GetArguments()["cleanup"].(bool) + clearBefore, _ := request.GetArguments()["clearBefore"].(bool) + + // Check if we have either path or URL + if (!hasPath || filePath == "") && (!hasUrl || fileUrl == "") { + return nil, fmt.Errorf("either filePath or fileUrl is required") + } + + // If we have a URL, download it + downloadedFile := false + fileType := "image" // Default file type + if hasUrl && fileUrl != "" { + log.Info().Str("fileUrl", fileUrl).Msg("Downloading media file from URL") + downloadedPath, err := DownloadFileByUrl(fileUrl) + if err != nil { + return nil, fmt.Errorf("failed to download media file from URL: %v", err) + } + + // Detect file type and rename with proper extension + renamedPath, err := DetectAndRenameMediaFile(downloadedPath) + if err != nil { + log.Warn().Err(err).Str("path", downloadedPath).Msg("Failed to detect file type or rename file, using original file") + filePath = downloadedPath + } else { + filePath = renamedPath + // Determine if it's a video based on extension + ext := strings.ToLower(filepath.Ext(renamedPath)) + if ext == ".mp4" || ext == ".mov" || ext == ".avi" || ext == ".wmv" || ext == ".flv" || ext == ".webm" || ext == ".mkv" { + fileType = "video" + } + } + downloadedFile = true + } + + // Clear albums before pushing if requested + cleared := false + if clearBefore { + log.Info().Msg("Clearing albums before pushing new media file") + err := driverExt.IDriver.ClearImages() + if err != nil { + log.Warn().Err(err).Msg("Failed to clear albums before pushing, continuing anyway") + } else { + cleared = true + } + } + + // Push the file to the device + err = driverExt.IDriver.PushImage(filePath) + if err != nil { + // If we downloaded the file and failed to push it, clean up + if downloadedFile && cleanup { + _ = os.Remove(filePath) + } + return nil, err + } + + // Clean up downloaded file if requested + if downloadedFile && cleanup { + log.Info().Str("filePath", filePath).Msg("Cleaning up downloaded media file") + _ = os.Remove(filePath) + } + + message := fmt.Sprintf("Successfully pushed %s to device", fileType) + returnData := ToolPushAlbums{ + FilePath: filePath, + FileType: fileType, + Cleared: cleared, + } + + // Include URL in response if it was used + if hasUrl && fileUrl != "" { + returnData.FileUrl = fileUrl + message = fmt.Sprintf("Successfully downloaded and pushed %s from %s to device", fileType, fileUrl) + } + + // Add cleared info to message if applicable + if cleared { + message = fmt.Sprintf("%s (albums cleared before pushing)", message) + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +// Old ToolPushImage implementation has been removed as part of the refactoring to ToolPushAlbums + +// ToolClearAlbums implements the clear_albums tool call. +type ToolClearAlbums struct { // Return data fields - these define the structure of data returned by this tool - Success bool `json:"success" desc:"Whether the operation was successful"` + Cleared bool `json:"cleared" desc:"Whether albums were cleared successfully"` } -func (t *ToolClearImage) Name() option.ActionName { - return option.ACTION_ClearImage +func (t *ToolClearAlbums) Name() option.ActionName { + return option.ACTION_ClearAlbums } -func (t *ToolClearImage) Description() string { - return "Clear images from the device's gallery. For Android, this will remove all images from the DCIM/Camera directory. For iOS, this will clear the images added through the push_image tool." +func (t *ToolClearAlbums) Description() string { + return "Clear media files (images and videos) from the device's gallery. For Android, this will clear media from the DCIM/Camera directory. For iOS, this will clear media from the device's photo album." } -func (t *ToolClearImage) Options() []mcp.ToolOption { +func (t *ToolClearAlbums) Options() []mcp.ToolOption { return []mcp.ToolOption{ - mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to clear images from")), + mcp.WithString("platform", mcp.Enum("android", "ios"), mcp.Description("The platform type of device to clear media from")), mcp.WithString("serial", mcp.Description("The device serial number or UDID")), } } -func (t *ToolClearImage) Implement() server.ToolHandlerFunc { +func (t *ToolClearAlbums) Implement() server.ToolHandlerFunc { return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { driverExt, err := setupXTDriver(ctx, request.GetArguments()) if err != nil { @@ -406,13 +404,13 @@ func (t *ToolClearImage) Implement() server.ToolHandlerFunc { return nil, err } - message := "Successfully cleared images from device" - returnData := ToolClearImage{Success: true} + message := "Successfully cleared media files from device" + returnData := ToolClearAlbums{Cleared: true} return NewMCPSuccessResponse(message, &returnData), nil } } -func (t *ToolClearImage) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { +func (t *ToolClearAlbums) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { return BuildMCPCallToolRequest(t.Name(), map[string]any{}, action), nil } diff --git a/uixt/option/action.go b/uixt/option/action.go index e13aa9f0..0a10e427 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -98,9 +98,9 @@ const ( ACTION_ListAvailableDevices ActionName = "list_available_devices" ACTION_SelectDevice ActionName = "select_device" - // image actions - ACTION_PushImage ActionName = "push_image" - ACTION_ClearImage ActionName = "clear_image" + // album actions (images and videos) + ACTION_PushAlbums ActionName = "push_albums" + ACTION_ClearAlbums ActionName = "clear_albums" // custom actions ACTION_SwipeToTapApp ActionName = "swipe_to_tap_app" // swipe left & right to find app and tap From 07395a2f9e23f7cec0057716c334077d2bb36d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Wed, 13 Aug 2025 15:21:42 +0800 Subject: [PATCH 25/27] =?UTF-8?q?feat:=20=E8=AE=BE=E7=BD=AE=E6=97=B6?= =?UTF-8?q?=E5=8C=BA=E4=B8=BA=E5=8C=97=E4=BA=AC=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/version/VERSION | 2 +- logger.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index b7fffde9..3c60993c 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250812 +v5.0.0-250813 diff --git a/logger.go b/logger.go index 3d3bcd11..ea7e01ce 100644 --- a/logger.go +++ b/logger.go @@ -18,7 +18,11 @@ func InitLogger(logLevel string, logJSON bool, logFile bool) { // Error Logging with Stacktrace zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack - // set log timestamp precise to milliseconds + // set log timestamp precise to milliseconds with Beijing timezone (UTC+8) + beijingLoc, _ := time.LoadLocation("Asia/Shanghai") + zerolog.TimestampFunc = func() time.Time { + return time.Now().In(beijingLoc) + } zerolog.TimeFieldFormat = "2006-01-02T15:04:05.999Z0700" // init log writers From 18de536dc061c728eaff21539ce9ae2cbd071873 Mon Sep 17 00:00:00 2001 From: "lilong.129" Date: Wed, 13 Aug 2025 22:15:01 +0800 Subject: [PATCH 26/27] fix: miss tap offset option --- CLAUDE.md | 6 ++- internal/version/VERSION | 2 +- uixt/mcp_server.go | 2 +- uixt/mcp_server_test.go | 93 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ec539501..2e996b2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,4 +125,8 @@ The framework supports both Go and Python plugins: ### Build Configuration - Static linking for deployment - Version info embedded via ldflags -- Cross-platform builds supported \ No newline at end of file +- Cross-platform builds supported + +### Code Standards +- All code comments must be written in English +- All documentation must be written in Chinese diff --git a/internal/version/VERSION b/internal/version/VERSION index b7fffde9..3c60993c 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250812 +v5.0.0-250813 diff --git a/uixt/mcp_server.go b/uixt/mcp_server.go index 9ffae9ab..b157d067 100644 --- a/uixt/mcp_server.go +++ b/uixt/mcp_server.go @@ -301,7 +301,7 @@ func extractActionOptionsToArguments(actionOptions []option.ActionOption, argume // Add tap/swipe offset options if len(tempOptions.TapOffset) == 2 { - arguments["tap_offset"] = tempOptions.TapOffset + arguments["offset"] = tempOptions.TapOffset } if len(tempOptions.SwipeOffset) == 4 { arguments["swipe_offset"] = tempOptions.SwipeOffset diff --git a/uixt/mcp_server_test.go b/uixt/mcp_server_test.go index 900adc12..45212116 100644 --- a/uixt/mcp_server_test.go +++ b/uixt/mcp_server_test.go @@ -169,27 +169,106 @@ func TestIgnoreNotFoundErrorOption(t *testing.T) { func TestExtractActionOptionsToArguments(t *testing.T) { // Test the extractActionOptionsToArguments helper function actionOptions := []option.ActionOption{ + // Boolean options option.WithIgnoreNotFoundError(true), - option.WithMaxRetryTimes(3), - option.WithIndex(2), option.WithRegex(true), option.WithTapRandomRect(false), // false should not be included - option.WithDuration(1.5), + option.WithAntiRisk(true), + option.WithPreMarkOperation(true), + option.WithResetHistory(true), + option.WithMatchOne(true), + + // Numeric options + option.WithMaxRetryTimes(3), + option.WithIndex(2), + option.WithInterval(1.5), + option.WithSteps(10), + option.WithTimeout(30), + option.WithFrequency(5), + option.WithDuration(2.0), + option.WithPressDuration(1.5), + + // Offset options (including the fixed offset field) + option.WithTapOffset(-300, 0), + option.WithSwipeOffset(1, 2, 3, 4), + option.WithOffsetRandomRange(-5, 5), + + // Scope options + option.WithScope(0.1, 0.2, 0.9, 0.8), + option.WithAbsScope(100, 200, 900, 800), + + // Screenshot options + option.WithScreenShotOCR(true), + option.WithScreenShotUpload(true), + option.WithScreenShotLiveType(true), + option.WithScreenShotLivePopularity(true), + option.WithScreenShotClosePopups(true), + option.WithScreenOCRCluster("test_cluster"), + option.WithScreenShotFileName("test.png"), + option.WithScreenShotUITypes("button", "input"), + + // Direction option + option.WithDirection("up"), + + // Identifier + option.WithIdentifier("test_id"), } arguments := make(map[string]any) extractActionOptionsToArguments(actionOptions, arguments) - // Verify extracted options + // Verify boolean options (only true values should be included) assert.Equal(t, true, arguments["ignore_NotFoundError"], "ignore_NotFoundError should be extracted") - assert.Equal(t, 3, arguments["max_retry_times"], "max_retry_times should be extracted") - assert.Equal(t, 2, arguments["index"], "index should be extracted") assert.Equal(t, true, arguments["regex"], "regex should be extracted") - assert.Equal(t, 1.5, arguments["duration"], "duration should be extracted") + assert.Equal(t, true, arguments["anti_risk"], "anti_risk should be extracted") + assert.Equal(t, true, arguments["pre_mark_operation"], "pre_mark_operation should be extracted") + assert.Equal(t, true, arguments["reset_history"], "reset_history should be extracted") + assert.Equal(t, true, arguments["match_one"], "match_one should be extracted") // tap_random_rect should not be included since it's false _, exists := arguments["tap_random_rect"] assert.False(t, exists, "tap_random_rect should not be included when false") + + // Verify numeric options + assert.Equal(t, 3, arguments["max_retry_times"], "max_retry_times should be extracted") + assert.Equal(t, 2, arguments["index"], "index should be extracted") + assert.Equal(t, 1.5, arguments["interval"], "interval should be extracted") + assert.Equal(t, 10, arguments["steps"], "steps should be extracted") + assert.Equal(t, 30, arguments["timeout"], "timeout should be extracted") + assert.Equal(t, 5, arguments["frequency"], "frequency should be extracted") + assert.Equal(t, 2.0, arguments["duration"], "duration should be extracted") + assert.Equal(t, 1.5, arguments["press_duration"], "press_duration should be extracted") + + // Verify offset options (including the critical 'offset' field that was fixed) + assert.Equal(t, []int{-300, 0}, arguments["offset"], "offset should be extracted (not tap_offset)") + assert.Equal(t, []int{1, 2, 3, 4}, arguments["swipe_offset"], "swipe_offset should be extracted") + assert.Equal(t, []int{-5, 5}, arguments["offset_random_range"], "offset_random_range should be extracted") + + // Verify scope options (these are custom types, not raw slices) + assert.Equal(t, option.Scope([]float64{0.1, 0.2, 0.9, 0.8}), arguments["scope"], "scope should be extracted") + assert.Equal(t, option.AbsScope([]int{100, 200, 900, 800}), arguments["abs_scope"], "abs_scope should be extracted") + + // Verify screenshot options + assert.Equal(t, true, arguments["screenshot_with_ocr"], "screenshot_with_ocr should be extracted") + assert.Equal(t, true, arguments["screenshot_with_upload"], "screenshot_with_upload should be extracted") + assert.Equal(t, true, arguments["screenshot_with_live_type"], "screenshot_with_live_type should be extracted") + assert.Equal(t, true, arguments["screenshot_with_live_popularity"], "screenshot_with_live_popularity should be extracted") + assert.Equal(t, true, arguments["screenshot_with_close_popups"], "screenshot_with_close_popups should be extracted") + assert.Equal(t, "test_cluster", arguments["screenshot_with_ocr_cluster"], "screenshot_with_ocr_cluster should be extracted") + assert.Equal(t, "test.png", arguments["screenshot_file_name"], "screenshot_file_name should be extracted") + assert.Equal(t, []string{"button", "input"}, arguments["screenshot_with_ui_types"], "screenshot_with_ui_types should be extracted") + + // Verify identifier and direction (only fields that exist) + assert.Equal(t, "test_id", arguments["identifier"], "identifier should be extracted") + assert.Equal(t, "up", arguments["direction"], "direction should be extracted") + + // Verify the critical fix: ensure "offset" is used instead of "tap_offset" + _, hasTapOffset := arguments["tap_offset"] + assert.False(t, hasTapOffset, "Should NOT contain 'tap_offset' field") + _, hasOffset := arguments["offset"] + assert.True(t, hasOffset, "Should contain 'offset' field") + + t.Logf("Extracted %d arguments from ActionOptions", len(arguments)) } // TestToolListAvailableDevices tests the ToolListAvailableDevices implementation From 6bf63cfcb327e4e655bfcc6c51b10258bd5143ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E6=B3=93=E9=93=AE?= Date: Fri, 15 Aug 2025 11:21:02 +0800 Subject: [PATCH 27/27] revert: --- internal/version/VERSION | 2 +- pkg/gadb/device.go | 13 ++- uixt/ai/wings_service.go | 183 ++++++++++++++++++++++++++----------- uixt/android_device.go | 4 +- uixt/android_test.go | 5 - uixt/driver_ext_ai_test.go | 23 +---- 6 files changed, 147 insertions(+), 83 deletions(-) diff --git a/internal/version/VERSION b/internal/version/VERSION index 3c60993c..a1341323 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250813 +v5.0.0-250815 diff --git a/pkg/gadb/device.go b/pkg/gadb/device.go index caf7e448..c6857fee 100644 --- a/pkg/gadb/device.go +++ b/pkg/gadb/device.go @@ -664,14 +664,22 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt tp transport filesize int64 ) + timeout := 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute) + defer cancel() + filesize, err = apk.Seek(0, io.SeekEnd) if err != nil { return nil, err } - if tp, err = d.createDeviceTransport(5 * time.Minute); err != nil { + if tp, err = d.createDeviceTransport(4 * time.Minute); err != nil { return nil, err } defer func() { _ = tp.Close() }() + go func() { + <-ctx.Done() + _ = tp.Close() + }() cmd := "abb_exec:package\x00install\x00-t" for _, arg := range args { cmd += "\x00" + arg @@ -690,6 +698,9 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt return nil, err } raw, err = tp.ReadBytesAll() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("installation timed out after %d minutes", timeout) + } return } diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 7e4f29f5..18f1a726 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -26,6 +26,7 @@ type WingsService struct { bizId string accessKey string secretKey string + history []History // Conversation history for Wings API } // NewWingsService creates a new Wings service instance @@ -49,6 +50,7 @@ func NewWingsService() (ILLMService, error) { bizId: bizID, accessKey: accessKey, secretKey: secretKey, + history: []History{}, }, nil } @@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni return nil, errors.Wrap(err, "validate planning parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Extract screenshot from message screenshot, err := w.extractScreenshotFromMessage(opts.Message) if err != nil { @@ -70,15 +77,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.UserInstruction, - BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", - StepType: "automation", - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfo: deviceInfo, + StepText: fmt.Sprintf("%s", opts.UserInstruction), + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s\n停止操作。\n注意事项：\n", opts.UserInstruction), Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -98,7 +101,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &PlanningResult{ Thought: response.ThoughtChain.Thought, @@ -107,26 +110,50 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni }, err } - // Convert Wings API response to tool calls - toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams) - if err != nil { - return &PlanningResult{ - Thought: response.ThoughtChain.Thought, - Error: err.Error(), - ModelName: "wings-api", - }, errors.Wrap(err, "convert Wings response to tool calls failed") + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: response.OriStepIndex, + DeviceID: deviceInfo[0].DeviceID, + AgentType: response.AgentType, + ActionResult: "", // Always empty as requested + DeviceInfos: &deviceInfo, + ActionParams: response.ActionParams, } + w.history = append(w.history, newHistoryEntry) + var toolCalls []schema.ToolCall + if response.StepType != "FINISH" { + // Convert Wings API response to tool calls + toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams) + if err != nil { + return &PlanningResult{ + Thought: response.ThoughtChain.Thought, + Error: err.Error(), + ModelName: "wings-api", + }, errors.Wrap(err, "convert Wings response to tool calls failed") + } + } + + // No need to update ActionResult as per user request + // ActionResult should always be empty log.Info(). Str("thought", response.ThoughtChain.Thought). + Str("action", response.AgentType). + Str("action_params", response.ActionParams). + Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)). Int("tool_calls_count", len(toolCalls)). Int64("elapsed_ms", elapsed). Msg("Wings API planning completed") return &PlanningResult{ ToolCalls: toolCalls, - Thought: response.ThoughtChain.Thought, - Content: response.ThoughtChain.Summary, + Thought: response.StepTextTrans, + Content: response.StepTextTrans, ModelName: "wings-api", }, nil } @@ -146,20 +173,15 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.Assertion, - BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", - StepType: "assert", // Different from automation - DeviceID: deviceInfo.DeviceID, + Historys: []History{}, + DeviceInfo: deviceInfo, + StepText: fmt.Sprintf("断言:%s", opts.Assertion), + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), Base: WingsBase{ LogID: generateWingsUUID(), }, } - log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request") // Call Wings API startTime := time.Now() @@ -175,7 +197,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &AssertionResult{ Pass: false, @@ -184,6 +206,22 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, err } + // Update history with response data + newHistoryEntry := History{ + Observation: response.ThoughtChain.Observation, + Thought: response.ThoughtChain.Thought, + Summary: response.ThoughtChain.Summary, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: response.OriStepIndex, + DeviceID: deviceInfo[0].DeviceID, + AgentType: response.AgentType, + ActionResult: "", // Always empty as requested + DeviceInfos: &deviceInfo, + ActionParams: response.ActionParams, + } + w.history = append(w.history, newHistoryEntry) + // Parse assertion result from action_params passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain) if err != nil { @@ -194,6 +232,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, errors.Wrap(err, "parse assertion result failed") } + // No need to update ActionResult as per user request + // ActionResult should always be empty + log.Info(). Bool("passed", passed). Str("thought", assertionThought). @@ -228,14 +269,12 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []interface{} `json:"historys"` - DeviceInfos []WingsDeviceInfo `json:"device_infos"` - StepText string `json:"step_text"` - BizId string `json:"biz_id"` - TextCase string `json:"text_case"` - StepType string `json:"step_type"` - DeviceID string `json:"device_id"` - Base WingsBase `json:"Base"` + Historys []History `json:"historys"` + DeviceInfo []WingsDeviceInfo `json:"device_infos"` + StepText string `json:"step_text"` + BizId string `json:"biz_id"` + TextCase string `json:"text_case"` + Base WingsBase `json:"Base"` } type WingsDeviceInfo struct { @@ -253,10 +292,14 @@ type WingsBase struct { } type WingsActionResponse struct { - StepType string `json:"step_type"` - ActionParams string `json:"action_params"` - ThoughtChain WingsThoughtChain `json:"thought_chain"` - BaseResp WingsBaseResp `json:"BaseResp"` + AgentType string `json:"agent_type" thrift:"agent_type,1,required"` + StepText string `json:"step_text" thrift:"step_text,2,required"` + StepTextTrans string `json:"step_text_trans" thrift:"step_text_trans,3,required"` + OriStepIndex int `json:"ori_step_index" thrift:"ori_step_index,4,required"` + StepType string `json:"step_type" thrift:"step_type,5,required"` + ActionParams string `json:"action_params" thrift:"action_params,6,required"` + ThoughtChain WingsThoughtChain `json:"thought_chain" thrift:"thought_chain,7,required"` + BaseResp WingsBaseResp `json:"BaseResp" thrift:"BaseResp,255,optional"` } type WingsThoughtChain struct { @@ -276,6 +319,21 @@ type WingsExtra struct { LogID string `json:"_log_id"` } +// History structure for request and response +type History struct { + Observation string `json:"observation" thrift:"observation,1,required"` // 思考结果 + Thought string `json:"thought" thrift:"thought,2,required"` // 思考结果 + Summary string `json:"summary" thrift:"summary,3,required"` // 思考结果 + StepText string `json:"step_text" thrift:"step_text,4"` // 操作的指令 + DeviceID string `json:"device_id" thrift:"device_id,5"` // 操作的设备id + AgentType string `json:"agent_type" thrift:"agent_type,7"` // 最终决策的agent类型 + ActionResult string `json:"action_result" thrift:"action_result,8"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 + DeviceInfos *[]WingsDeviceInfo `json:"device_infos,omitempty" thrift:"device_infos,9"` // 所有设备的信息 + ActionParams string `json:"action_params,omitempty" thrift:"action_params,10"` // 历史操作解析结果(断言，自动化，物料构造) + StepTextTrans string `json:"step_text_trans,omitempty" thrift:"step_text_trans,13"` // 归一化的步骤文本(为后续的实际执行解析文本) + OriStepIndex int `json:"ori_step_index,omitempty" thrift:"ori_step_index,14"` // 原本的执行序列（扩展前、目标导向原始文本步骤） +} + // Action parameter structures type WingsActionParams struct { Type string `json:"Type"` @@ -315,6 +373,11 @@ type WingsTextParams struct { // Helper methods +// resetHistory resets the conversation history +func (w *WingsService) resetHistory() { + w.history = []History{} +} + // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { return uuid.New().String() @@ -345,19 +408,29 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st } // getDeviceInfoFromContext gets device info from context with fallback -func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo { - // use default device info - return WingsDeviceInfo{ - DeviceID: "default-device", - NowImage: screenshot, - PreImage: screenshot, - NowLayoutJSON: "", - OperationSystem: "android", +func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo { + // TODO: Extract device info from context if available + + // Use last history's NowImage as PreImage if history exists + preImage := screenshot + if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 { + preImage = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImage + } + + // use default device info with optimized PreImage + return []WingsDeviceInfo{ + { + DeviceID: "default-device", + NowImage: screenshot, + PreImage: preImage, + NowLayoutJSON: "", + OperationSystem: "android", + }, } } // getDeviceInfoFromScreenshot gets device info from screenshot (for Assert) -func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo { +func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo { return w.getDeviceInfoFromContext(ctx, screenshot) } @@ -390,6 +463,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Set headers httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Accept", "application/json") + httpReq.Header.Add("x-use-ppe", "1") + httpReq.Header.Add("x-tt-env", "ppe_refactor_merge") // Add authentication headers if using external API if w.accessKey != "" && w.secretKey != "" { @@ -403,7 +478,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Execute HTTP request client := &http.Client{ - Timeout: 60 * time.Second, + Timeout: 120 * time.Second, } resp, err := client.Do(httpReq) @@ -437,7 +512,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) { - if actionParamsStr == "" { + if actionParamsStr == "" || actionParamsStr == "FINISH" { return []schema.ToolCall{}, nil } diff --git a/uixt/android_device.go b/uixt/android_device.go index efb243e8..0a47eb95 100644 --- a/uixt/android_device.go +++ b/uixt/android_device.go @@ -240,12 +240,12 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er return err } // 等待安装完成或超时 - timeout := 3 * time.Minute + timeout := 8 * time.Minute select { case err := <-done: return err case <-time.After(timeout): - return fmt.Errorf("installation timed out after %v", timeout) + return fmt.Errorf("install via installer timed out after %v", timeout) } } diff --git a/uixt/android_test.go b/uixt/android_test.go index b8a085be..c7cb883d 100644 --- a/uixt/android_test.go +++ b/uixt/android_test.go @@ -21,11 +21,6 @@ func setupADBDriverExt(t *testing.T) *XTDriver { Serial: "", // Let it auto-detect the device serial AIOptions: []option.AIServiceOption{ option.WithCVService(option.CVServiceTypeVEDEM), - option.WithLLMConfig( - option.NewLLMServiceConfig(option.DOUBAO_1_5_UI_TARS_250328). - WithPlannerModel(option.WINGS_SERVICE). - WithAsserterModel(option.WINGS_SERVICE), - ), }, } diff --git a/uixt/driver_ext_ai_test.go b/uixt/driver_ext_ai_test.go index 6ca5d2e3..89fd65c8 100644 --- a/uixt/driver_ext_ai_test.go +++ b/uixt/driver_ext_ai_test.go @@ -292,31 +292,14 @@ func TestDriverExt_AIAction(t *testing.T) { func TestDriverExt_AIAction_CompareWithAIAction(t *testing.T) { driver := setupDriverExt(t) - prompt := "点击搜索按钮" + prompt := "[目标导向]向上滑动屏幕2次" // Test both methods with the same prompt - aiResult, aiErr := driver.AIAction(context.Background(), prompt) + aiResult, aiErr := driver.StartToGoal(context.Background(), prompt) // Both should execute without critical errors (may have different implementations) t.Logf("AIAction error: %v", aiErr) - - // If both succeed, compare results - if aiResult != nil { - assert.Equal(t, "action", aiResult.Type, "AIAction result type should be 'action'") - - // Both should have timing information - assert.Greater(t, aiResult.ModelCallElapsed, int64(0), "AIAction should have model call elapsed time") - - // Both should have screenshot information - assert.NotEmpty(t, aiResult.ImagePath, "AIAction should have image path") - - // Compare model names - if aiResult.PlanningResult != nil { - t.Logf("AIAction model: %s", aiResult.PlanningResult.ModelName) - - assert.Equal(t, "wings-api", aiResult.PlanningResult.ModelName, "AIAction should use wings-api") - } - } + t.Logf("AIAction result: %v", aiResult) } // TestDriverExt_AIAction_ErrorHandling tests AIAction error handling