From f20e679855aa239d296e6ad5bce8436b9b13c237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Tue, 29 Jul 2025 19:45:30 +0800 Subject: [PATCH 1/5] add simulation --- .../uitest/android_touch_simulator_test.go | 114 ++--- internal/version/VERSION | 2 +- step_ui.go | 94 ++++ uixt/mcp_server.go | 23 +- uixt/mcp_tools_input.go | 81 ++++ uixt/mcp_tools_swipe.go | 409 ++++++++++++++++++ uixt/mcp_tools_touch.go | 93 ++++ uixt/option/action.go | 19 +- 8 files changed, 765 insertions(+), 70 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 1d46505c..3ef41180 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -256,30 +256,30 @@ func TestSwipeWithDirection(t *testing.T) { minDistance: 100.0, maxDistance: 500.0, }, - { - name: "随机距离下滑", - direction: "down", - startX: 0.5, - startY: 0.5, - minDistance: 150.0, - maxDistance: 350.0, // 范围内随机 - }, - { - name: "固定距离左滑", - direction: "left", - startX: 0.5, - startY: 0.5, - minDistance: 300.0, - maxDistance: 300.0, - }, - { - name: "随机距离右滑", - direction: "right", - startX: 0.6, - startY: 0.5, - minDistance: 100.0, - maxDistance: 250.0, - }, + //{ + // name: "随机距离下滑", + // direction: "down", + // startX: 0.5, + // startY: 0.5, + // minDistance: 150.0, + // maxDistance: 350.0, // 范围内随机 + //}, + //{ + // name: "固定距离左滑", + // direction: "left", + // startX: 0.5, + // startY: 0.5, + // minDistance: 300.0, + // maxDistance: 300.0, + //}, + //{ + // name: "随机距离右滑", + // direction: "right", + // startX: 0.6, + // startY: 0.5, + // minDistance: 100.0, + // maxDistance: 250.0, + //}, } for _, tc := range testCases { @@ -567,41 +567,41 @@ func TestSIMInput(t *testing.T) { name string text string }{ - { - name: "英文短文本", - text: "Hello", - }, - { - name: "英文长文本", - text: "Hello World! This is a test message.", - }, - { - name: "日文文本", - text: "英語の長い文字", - }, - { - name: "混合文本", - text: "Hello你好123", - }, - { - name: "特殊字符", - text: "!@#$%^&*()", - }, - { - name: "数字文本", - text: "1234567890", - }, - { - name: "空文本", - text: "", - }, - { - name: "单个字符", - text: "A", - }, + //{ + // name: "英文短文本", + // text: "Hello", + //}, + //{ + // name: "英文长文本", + // text: "Hello World! This is a test message.", + //}, + //{ + // name: "日文文本", + // text: "英語の長い文字", + //}, + //{ + // name: "混合文本", + // text: "Hello你好123", + //}, + //{ + // name: "特殊字符", + // text: "!@#$%^&*()", + //}, + //{ + // name: "数字文本", + // text: "1234567890", + //}, + //{ + // name: "空文本", + // text: "", + //}, + //{ + // name: "单个字符", + // text: "A", + //}, { name: "长文本", - text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文", + text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", }, } diff --git a/internal/version/VERSION b/internal/version/VERSION index db846138..e0737887 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250728 +v5.0.0-250729 diff --git a/step_ui.go b/step_ui.go index 9e2d7682..f0e0890a 100644 --- a/step_ui.go +++ b/step_ui.go @@ -290,6 +290,100 @@ func (s *StepMobile) SwipeRight(opts ...option.ActionOption) *StepMobile { return s } +// SIMSwipeWithDirection performs simulated swipe in specified direction with random distance +func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeWithDirection + params := map[string]interface{}{ + "direction": direction, + "start_x": startX, + "start_y": startY, + "min_distance": minDistance, + "max_distance": maxDistance, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeDirection, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMSwipeInArea performs simulated swipe in specified area with direction and random distance +func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeInArea + params := map[string]interface{}{ + "direction": direction, + "area_start_x": areaStartX, + "area_start_y": areaStartY, + "area_end_x": areaEndX, + "area_end_y": areaEndY, + "min_distance": minDistance, + "max_distance": maxDistance, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeInArea, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMSwipeFromPointToPoint performs simulated swipe from point to point +func (s *StepMobile) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMSwipeFromPointToPoint + params := map[string]interface{}{ + "start_x": startX, + "start_y": startY, + "end_x": endX, + "end_y": endY, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMSwipeFromPointToPoint, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMClickAtPoint performs simulated click at specified point +func (s *StepMobile) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) *StepMobile { + // Create params map for SIMClickAtPoint + params := map[string]interface{}{ + "x": x, + "y": y, + } + + action := option.MobileAction{ + Method: option.ACTION_SIMClickAtPoint, + Params: params, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + +// SIMInput performs simulated text input with intelligent segmentation +func (s *StepMobile) SIMInput(text string, opts ...option.ActionOption) *StepMobile { + action := option.MobileAction{ + Method: option.ACTION_SIMInput, + Params: text, + Options: option.NewActionOptions(opts...), + } + + s.obj().Actions = append(s.obj().Actions, action) + return s +} + func (s *StepMobile) SwipeToTapApp(appName string, opts ...option.ActionOption) *StepMobile { action := option.MobileAction{ Method: option.ACTION_SwipeToTapApp, diff --git a/uixt/mcp_server.go b/uixt/mcp_server.go index d8f2d4d4..92f0b1be 100644 --- a/uixt/mcp_server.go +++ b/uixt/mcp_server.go @@ -87,23 +87,28 @@ func (s *MCPServer4XTDriver) registerTools() { s.registerTool(&ToolSelectDevice{}) // SelectDevice // Touch Tools - s.registerTool(&ToolTapXY{}) // tap xy - s.registerTool(&ToolTapAbsXY{}) // tap abs xy - s.registerTool(&ToolTapByOCR{}) // tap by OCR - s.registerTool(&ToolTapByCV{}) // tap by CV - s.registerTool(&ToolDoubleTapXY{}) // double tap xy + s.registerTool(&ToolTapXY{}) // tap xy + s.registerTool(&ToolTapAbsXY{}) // tap abs xy + s.registerTool(&ToolTapByOCR{}) // tap by OCR + s.registerTool(&ToolTapByCV{}) // tap by CV + s.registerTool(&ToolDoubleTapXY{}) // double tap xy + s.registerTool(&ToolSIMClickAtPoint{}) // simulated click at point // Swipe Tools - s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate - s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right - s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY] + s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate + s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right + s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY] + s.registerTool(&ToolSIMSwipeDirection{}) // simulated swipe direction with random distance + s.registerTool(&ToolSIMSwipeInArea{}) // simulated swipe in area with direction and distance + s.registerTool(&ToolSIMSwipeFromPointToPoint{}) // simulated swipe from point to point s.registerTool(&ToolSwipeToTapApp{}) s.registerTool(&ToolSwipeToTapText{}) s.registerTool(&ToolSwipeToTapTexts{}) s.registerTool(&ToolDrag{}) // Input Tools - s.registerTool(&ToolInput{}) + s.registerTool(&ToolInput{}) // regular input + s.registerTool(&ToolSIMInput{}) // simulated input with intelligent segmentation s.registerTool(&ToolBackspace{}) s.registerTool(&ToolSetIme{}) diff --git a/uixt/mcp_tools_input.go b/uixt/mcp_tools_input.go index 8f4b306e..3341c047 100644 --- a/uixt/mcp_tools_input.go +++ b/uixt/mcp_tools_input.go @@ -6,6 +6,7 @@ import ( "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" + "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v5/uixt/option" ) @@ -192,3 +193,83 @@ func (t *ToolBackspace) ConvertActionToCallToolRequest(action option.MobileActio } return BuildMCPCallToolRequest(t.Name(), arguments, action), nil } + +// ToolSIMInput implements the sim_input tool call. +type ToolSIMInput struct { + // Return data fields - these define the structure of data returned by this tool + Text string `json:"text" desc:"Text that was input with simulation"` + Segments int `json:"segments" desc:"Number of segments the text was split into"` +} + +func (t *ToolSIMInput) Name() option.ActionName { + return option.ACTION_SIMInput +} + +func (t *ToolSIMInput) Description() string { + return "Input text with intelligent segmentation and human-like typing patterns" +} + +func (t *ToolSIMInput) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMInput) +} + +func (t *ToolSIMInput) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + if unifiedReq.Text == "" { + return nil, fmt.Errorf("text is required") + } + + text := unifiedReq.Text + + log.Info(). + Str("text", text). + Int("textLength", len(text)). + Msg("performing simulated input") + + opts := unifiedReq.Options() + + // Call the underlying SIMInput method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMInput(text, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMInput is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMInput") + } + + // Estimate segments count (this is approximate since the actual segmentation happens in the driver) + estimatedSegments := len([]rune(text))/2 + 1 + if estimatedSegments < 1 { + estimatedSegments = 1 + } + + message := fmt.Sprintf("Successfully performed simulated input: %s", text) + returnData := ToolSIMInput{ + Text: text, + Segments: estimatedSegments, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMInput) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + text := fmt.Sprintf("%v", action.Params) + arguments := map[string]any{ + "text": text, + } + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil +} diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index 3cbc774b..31df991f 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -547,3 +547,412 @@ func (t *ToolDrag) ConvertActionToCallToolRequest(action option.MobileAction) (m } return mcp.CallToolRequest{}, fmt.Errorf("invalid drag parameters: %v", action.Params) } + +// ToolSIMSwipeDirection implements the sim_swipe_direction tool call. +type ToolSIMSwipeDirection struct { + // Return data fields - these define the structure of data returned by this tool + Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"` + StartX float64 `json:"startX" desc:"Starting X coordinate of the simulated swipe"` + StartY float64 `json:"startY" desc:"Starting Y coordinate of the simulated swipe"` + MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"` + MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"` + ActualDistance float64 `json:"actualDistance" desc:"Actual distance of the simulated swipe"` +} + +func (t *ToolSIMSwipeDirection) Name() option.ActionName { + return option.ACTION_SIMSwipeDirection +} + +func (t *ToolSIMSwipeDirection) Description() string { + return "Perform simulated swipe in specified direction with random distance and human-like touch patterns" +} + +func (t *ToolSIMSwipeDirection) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeDirection) +} + +func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.Direction == nil { + return nil, fmt.Errorf("direction parameter is required") + } + direction, ok := unifiedReq.Direction.(string) + if !ok { + return nil, fmt.Errorf("direction must be a string") + } + + // Validate direction + validDirections := []string{"up", "down", "left", "right"} + if !slices.Contains(validDirections, direction) { + return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v", + direction, validDirections) + } + + // Default values if not provided + startX := unifiedReq.StartX + startY := unifiedReq.StartY + minDistance := unifiedReq.MinDistance + maxDistance := unifiedReq.MaxDistance + + if startX == 0 { + startX = 0.5 // default to center + } + if startY == 0 { + startY = 0.5 // default to center + } + if minDistance == 0 { + minDistance = 100 // default minimum distance + } + if maxDistance == 0 { + maxDistance = 300 // default maximum distance + } + + log.Info(). + Str("direction", direction). + Float64("startX", startX). + Float64("startY", startY). + Float64("minDistance", minDistance). + Float64("maxDistance", maxDistance). + Msg("performing simulated swipe with direction") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeWithDirection method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeWithDirection is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeWithDirection") + } + + // Calculate actual distance for response (approximate) + actualDistance := minDistance + if maxDistance > minDistance { + actualDistance = minDistance + (maxDistance-minDistance)*0.5 // approximate middle value + } + + message := fmt.Sprintf("Successfully performed simulated swipe %s from (%.2f, %.2f) with distance %.2f", + direction, startX, startY, actualDistance) + returnData := ToolSIMSwipeDirection{ + Direction: direction, + StartX: startX, + StartY: startY, + MinDistance: minDistance, + MaxDistance: maxDistance, + ActualDistance: actualDistance, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeDirection) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract direction + if direction, exists := paramsMap["direction"]; exists { + arguments["direction"] = direction + } + + // Extract coordinates and distances + if startX, exists := paramsMap["start_x"]; exists { + arguments["start_x"] = startX + } + if startY, exists := paramsMap["start_y"]; exists { + arguments["start_y"] = startY + } + if minDistance, exists := paramsMap["min_distance"]; exists { + arguments["min_distance"] = minDistance + } + if maxDistance, exists := paramsMap["max_distance"]; exists { + arguments["max_distance"] = maxDistance + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe direction params: %v", action.Params) +} + +// ToolSIMSwipeInArea implements the sim_swipe_in_area tool call. +type ToolSIMSwipeInArea struct { + // Return data fields - these define the structure of data returned by this tool + Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"` + AreaStartX float64 `json:"areaStartX" desc:"Area starting X coordinate"` + AreaStartY float64 `json:"areaStartY" desc:"Area starting Y coordinate"` + AreaEndX float64 `json:"areaEndX" desc:"Area ending X coordinate"` + AreaEndY float64 `json:"areaEndY" desc:"Area ending Y coordinate"` + MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"` + MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"` +} + +func (t *ToolSIMSwipeInArea) Name() option.ActionName { + return option.ACTION_SIMSwipeInArea +} + +func (t *ToolSIMSwipeInArea) Description() string { + return "Perform simulated swipe in specified area with direction and random distance" +} + +func (t *ToolSIMSwipeInArea) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeInArea) +} + +func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.Direction == nil { + return nil, fmt.Errorf("direction parameter is required") + } + direction, ok := unifiedReq.Direction.(string) + if !ok { + return nil, fmt.Errorf("direction must be a string") + } + + // Validate direction + validDirections := []string{"up", "down", "left", "right"} + if !slices.Contains(validDirections, direction) { + return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v", + direction, validDirections) + } + + // Get area coordinates + areaStartX := unifiedReq.AreaStartX + areaStartY := unifiedReq.AreaStartY + areaEndX := unifiedReq.AreaEndX + areaEndY := unifiedReq.AreaEndY + minDistance := unifiedReq.MinDistance + maxDistance := unifiedReq.MaxDistance + + // Default values + if minDistance == 0 { + minDistance = 100 + } + if maxDistance == 0 { + maxDistance = 300 + } + + log.Info(). + Str("direction", direction). + Float64("areaStartX", areaStartX). + Float64("areaStartY", areaStartY). + Float64("areaEndX", areaEndX). + Float64("areaEndY", areaEndY). + Float64("minDistance", minDistance). + Float64("maxDistance", maxDistance). + Msg("performing simulated swipe in area") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeInArea method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeInArea is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeInArea") + } + + message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", + direction, areaStartX, areaStartY, areaEndX, areaEndY) + returnData := ToolSIMSwipeInArea{ + Direction: direction, + AreaStartX: areaStartX, + AreaStartY: areaStartY, + AreaEndX: areaEndX, + AreaEndY: areaEndY, + MinDistance: minDistance, + MaxDistance: maxDistance, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeInArea) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract direction + if direction, exists := paramsMap["direction"]; exists { + arguments["direction"] = direction + } + + // Extract area coordinates and distances + if areaStartX, exists := paramsMap["area_start_x"]; exists { + arguments["area_start_x"] = areaStartX + } + if areaStartY, exists := paramsMap["area_start_y"]; exists { + arguments["area_start_y"] = areaStartY + } + if areaEndX, exists := paramsMap["area_end_x"]; exists { + arguments["area_end_x"] = areaEndX + } + if areaEndY, exists := paramsMap["area_end_y"]; exists { + arguments["area_end_y"] = areaEndY + } + if minDistance, exists := paramsMap["min_distance"]; exists { + arguments["min_distance"] = minDistance + } + if maxDistance, exists := paramsMap["max_distance"]; exists { + arguments["max_distance"] = maxDistance + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe in area params: %v", action.Params) +} + +// ToolSIMSwipeFromPointToPoint implements the sim_swipe_point_to_point tool call. +type ToolSIMSwipeFromPointToPoint struct { + // Return data fields - these define the structure of data returned by this tool + StartX float64 `json:"startX" desc:"Starting X coordinate"` + StartY float64 `json:"startY" desc:"Starting Y coordinate"` + EndX float64 `json:"endX" desc:"Ending X coordinate"` + EndY float64 `json:"endY" desc:"Ending Y coordinate"` +} + +func (t *ToolSIMSwipeFromPointToPoint) Name() option.ActionName { + return option.ACTION_SIMSwipeFromPointToPoint +} + +func (t *ToolSIMSwipeFromPointToPoint) Description() string { + return "Perform simulated swipe from point to point with human-like touch patterns" +} + +func (t *ToolSIMSwipeFromPointToPoint) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeFromPointToPoint) +} + +func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Get coordinates from arguments + startX := unifiedReq.StartX + startY := unifiedReq.StartY + endX := unifiedReq.ToX // Using existing ToX field + endY := unifiedReq.ToY // Using existing ToY field + + log.Info(). + Float64("startX", startX). + Float64("startY", startY). + Float64("endX", endX). + Float64("endY", endY). + Msg("performing simulated point to point swipe") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMSwipeFromPointToPoint method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMSwipeFromPointToPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeFromPointToPoint") + } + + message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", + startX, startY, endX, endY) + returnData := ToolSIMSwipeFromPointToPoint{ + StartX: startX, + StartY: startY, + EndX: endX, + EndY: endY, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMSwipeFromPointToPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract coordinates + if startX, exists := paramsMap["start_x"]; exists { + arguments["start_x"] = startX + } + if startY, exists := paramsMap["start_y"]; exists { + arguments["start_y"] = startY + } + if endX, exists := paramsMap["end_x"]; exists { + arguments["to_x"] = endX // Map to existing ToX field + } + if endY, exists := paramsMap["end_y"]; exists { + arguments["to_y"] = endY // Map to existing ToY field + } + + // Add duration and press duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 { + arguments["pressDuration"] = pressDuration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe point to point params: %v", action.Params) +} diff --git a/uixt/mcp_tools_touch.go b/uixt/mcp_tools_touch.go index f78d7ef1..751b7620 100644 --- a/uixt/mcp_tools_touch.go +++ b/uixt/mcp_tools_touch.go @@ -6,6 +6,7 @@ import ( "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" + "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/uixt/option" @@ -341,3 +342,95 @@ func (t *ToolDoubleTapXY) ConvertActionToCallToolRequest(action option.MobileAct } return mcp.CallToolRequest{}, fmt.Errorf("invalid double tap params: %v", action.Params) } + +// ToolSIMClickAtPoint implements the sim_click_at_point tool call. +type ToolSIMClickAtPoint struct { + // Return data fields - these define the structure of data returned by this tool + X float64 `json:"x" desc:"X coordinate where simulated click was performed"` + Y float64 `json:"y" desc:"Y coordinate where simulated click was performed"` +} + +func (t *ToolSIMClickAtPoint) Name() option.ActionName { + return option.ACTION_SIMClickAtPoint +} + +func (t *ToolSIMClickAtPoint) Description() string { + return "Perform simulated click at specified point with human-like touch patterns" +} + +func (t *ToolSIMClickAtPoint) Options() []mcp.ToolOption { + unifiedReq := &option.ActionOptions{} + return unifiedReq.GetMCPOptions(option.ACTION_SIMClickAtPoint) +} + +func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + arguments := request.GetArguments() + driverExt, err := setupXTDriver(ctx, arguments) + if err != nil { + return nil, fmt.Errorf("setup driver failed: %w", err) + } + + unifiedReq, err := parseActionOptions(arguments) + if err != nil { + return nil, err + } + + // Validate required parameters + if unifiedReq.X == 0 || unifiedReq.Y == 0 { + return nil, fmt.Errorf("x and y coordinates are required") + } + + x := unifiedReq.X + y := unifiedReq.Y + + log.Info(). + Float64("x", x). + Float64("y", y). + Msg("performing simulated click at point") + + // Build all options from request arguments + opts := unifiedReq.Options() + + // Call the underlying SIMClickAtPoint method (Android UIA2 specific) + if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { + err = uia2Driver.SIMClickAtPoint(x, y, opts...) + if err != nil { + return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err + } + } else { + return NewMCPErrorResponse("SIMClickAtPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMClickAtPoint") + } + + message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y) + returnData := ToolSIMClickAtPoint{ + X: x, + Y: y, + } + + return NewMCPSuccessResponse(message, &returnData), nil + } +} + +func (t *ToolSIMClickAtPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) { + // Handle params as map[string]interface{} + if paramsMap, ok := action.Params.(map[string]interface{}); ok { + arguments := map[string]any{} + + // Extract coordinates + if x, exists := paramsMap["x"]; exists { + arguments["x"] = x + } + if y, exists := paramsMap["y"]; exists { + arguments["y"] = y + } + + // Add duration from options + if duration := action.ActionOptions.Duration; duration > 0 { + arguments["duration"] = duration + } + + return BuildMCPCallToolRequest(t.Name(), arguments, action), nil + } + return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM click at point params: %v", action.Params) +} diff --git a/uixt/option/action.go b/uixt/option/action.go index a6f40736..f165cf36 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -66,9 +66,14 @@ const ( ACTION_TapByCV ActionName = "tap_cv" ACTION_DoubleTap ActionName = "double_tap" // generic double tap action ACTION_DoubleTapXY ActionName = "double_tap_xy" - ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates - ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right) - ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY) + ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates + ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right) + ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY) + ACTION_SIMSwipeDirection ActionName = "sim_swipe_direction" // simulated swipe by direction with random distance + ACTION_SIMSwipeInArea ActionName = "sim_swipe_in_area" // simulated swipe in area with direction and distance + ACTION_SIMSwipeFromPointToPoint ActionName = "sim_swipe_point_to_point" // simulated swipe from point to point + ACTION_SIMClickAtPoint ActionName = "sim_click_at_point" // simulated click at point + ACTION_SIMInput ActionName = "sim_input" // simulated text input with segments ACTION_Drag ActionName = "drag" ACTION_Input ActionName = "input" ACTION_PressButton ActionName = "press_button" @@ -201,6 +206,14 @@ type ActionOptions struct { PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"` Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` + StartX float64 `json:"start_x,omitempty" yaml:"start_x,omitempty" desc:"Starting X coordinate for simulated swipe"` + StartY float64 `json:"start_y,omitempty" yaml:"start_y,omitempty" desc:"Starting Y coordinate for simulated swipe"` + MinDistance float64 `json:"min_distance,omitempty" yaml:"min_distance,omitempty" desc:"Minimum distance for simulated swipe"` + MaxDistance float64 `json:"max_distance,omitempty" yaml:"max_distance,omitempty" desc:"Maximum distance for simulated swipe"` + AreaStartX float64 `json:"area_start_x,omitempty" yaml:"area_start_x,omitempty" desc:"Area starting X coordinate for simulated swipe"` + AreaStartY float64 `json:"area_start_y,omitempty" yaml:"area_start_y,omitempty" desc:"Area starting Y coordinate for simulated swipe"` + AreaEndX float64 `json:"area_end_x,omitempty" yaml:"area_end_x,omitempty" desc:"Area ending X coordinate for simulated swipe"` + AreaEndY float64 `json:"area_end_y,omitempty" yaml:"area_end_y,omitempty" desc:"Area ending Y coordinate for simulated swipe"` Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` From 9b529eb125a9b9bf2b8b3715d8eaa171e010c721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Tue, 29 Jul 2025 20:01:44 +0800 Subject: [PATCH 2/5] fix test --- .../uitest/android_touch_simulator_test.go | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 3ef41180..588265d0 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -256,30 +256,6 @@ func TestSwipeWithDirection(t *testing.T) { minDistance: 100.0, maxDistance: 500.0, }, - //{ - // name: "随机距离下滑", - // direction: "down", - // startX: 0.5, - // startY: 0.5, - // minDistance: 150.0, - // maxDistance: 350.0, // 范围内随机 - //}, - //{ - // name: "固定距离左滑", - // direction: "left", - // startX: 0.5, - // startY: 0.5, - // minDistance: 300.0, - // maxDistance: 300.0, - //}, - //{ - // name: "随机距离右滑", - // direction: "right", - // startX: 0.6, - // startY: 0.5, - // minDistance: 100.0, - // maxDistance: 250.0, - //}, } for _, tc := range testCases { @@ -567,38 +543,6 @@ func TestSIMInput(t *testing.T) { name string text string }{ - //{ - // name: "英文短文本", - // text: "Hello", - //}, - //{ - // name: "英文长文本", - // text: "Hello World! This is a test message.", - //}, - //{ - // name: "日文文本", - // text: "英語の長い文字", - //}, - //{ - // name: "混合文本", - // text: "Hello你好123", - //}, - //{ - // name: "特殊字符", - // text: "!@#$%^&*()", - //}, - //{ - // name: "数字文本", - // text: "1234567890", - //}, - //{ - // name: "空文本", - // text: "", - //}, - //{ - // name: "单个字符", - // text: "A", - //}, { name: "长文本", text: "This is a very long text to test the performance of SIMInput function. 这是一个很长的文本用来测试SIMInput函数的性能。1234567890!@#$%^&*()英語の長い文字", From ff13d907552787da09aac4c62ccf7b16a4486fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 11:18:26 +0800 Subject: [PATCH 3/5] add mcp_sim --- .../uitest/android_touch_simulator_test.go | 41 +++++++++++++++++++ internal/simulation/click_api.go | 2 +- internal/version/VERSION | 2 +- uixt/android_driver_adb.go | 23 +++++++++++ uixt/android_driver_uia2.go | 2 +- uixt/driver.go | 14 +++++++ uixt/mcp_tools_input.go | 8 ++-- uixt/mcp_tools_swipe.go | 24 +++++------ uixt/mcp_tools_touch.go | 8 ++-- 9 files changed, 101 insertions(+), 23 deletions(-) diff --git a/examples/uitest/android_touch_simulator_test.go b/examples/uitest/android_touch_simulator_test.go index 588265d0..cec638bb 100644 --- a/examples/uitest/android_touch_simulator_test.go +++ b/examples/uitest/android_touch_simulator_test.go @@ -2,10 +2,12 @@ package uitest import ( "fmt" + "os" "strconv" "strings" "testing" + hrp "github.com/httprunner/httprunner/v5" "github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt/option" "github.com/httprunner/httprunner/v5/uixt/types" @@ -561,3 +563,42 @@ func TestSIMInput(t *testing.T) { }) } } + +// TestStepMultipleSIMActions tests multiple SIM actions in one test case +func TestStepMultipleSIMActions(t *testing.T) { + // 创建包含多个SIM操作的测试用例 + testCase := &hrp.TestCase{ + Config: hrp.NewConfig("多个SIM操作组合测试").SetAndroid(option.WithUIA2(true), option.WithSerialNumber("")), + TestSteps: []hrp.IStep{ + hrp.NewStep("组合SIM操作测试"). + Android(). + SIMClickAtPoint(0.5, 0.5). // 点击屏幕中心 + Sleep(1). // 等待1秒 + SIMSwipeWithDirection("up", 0.5, 0.7, 200.0, 400.0). // 向上滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeInArea("up", 0.2, 0.2, 0.6, 0.6, 350.0, 500.0). // 在区域内向下滑动 + Sleep(0.5). // 等待0.5秒 + SIMSwipeFromPointToPoint(0.1, 0.5, 0.9, 0.5). // 从左到右滑动 + Sleep(0.5). // 等待0.5秒 + SIMInput("测试组合操作 Test Combination 123"), // 仿真输入 + }, + } + + // 运行测试用例 + err := testCase.Dump2JSON("TestStepMultipleSIMActions.json") + if err != nil { + t.Fatalf("Failed to dump test case: %v", err) + } + defer func() { + // 清理生成的文件 + _ = os.Remove("TestStepMultipleSIMActions.json") + }() + + // 执行测试用例 + err = hrp.NewRunner(t).Run(testCase) + if err != nil { + t.Errorf("Test case failed: %v", err) + } + + t.Logf("Successfully executed multiple SIM actions test") +} diff --git a/internal/simulation/click_api.go b/internal/simulation/click_api.go index edf813a3..dbc8c2ed 100644 --- a/internal/simulation/click_api.go +++ b/internal/simulation/click_api.go @@ -61,7 +61,7 @@ type ClickConfig struct { var DefaultClickConfig = ClickConfig{ MinDuration: 40, MaxDuration: 90, - MinPoints: 3, + MinPoints: 4, // 增加最小点数从3到4，确保至少有2个MOVE事件 MaxPoints: 6, MaxDeviation: 2.0, NoiseLevel: 0.5, diff --git a/internal/version/VERSION b/internal/version/VERSION index e0737887..dc1711bb 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250729 +v5.0.0-250730 diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 0f54b9c3..070fe47d 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -1273,3 +1273,26 @@ func (ad *ADBDriver) SecondaryClick(x, y float64) (err error) { func (ad *ADBDriver) SecondaryClickBySelector(selector string, options ...option.ActionOption) (err error) { return err } + +// SIMSupport interface implementation for ADBDriver +// These methods return "not supported" errors since SIM functionality is only available in UIA2Driver + +func (ad *ADBDriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMClickAtPoint is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeWithDirection is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeInArea is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { + return fmt.Errorf("SIMSwipeFromPointToPoint is not supported in ADBDriver, please use UIA2Driver instead") +} + +func (ad *ADBDriver) SIMInput(text string, opts ...option.ActionOption) error { + return fmt.Errorf("SIMInput is not supported in ADBDriver, please use UIA2Driver instead") +} diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 183e4cdb..3919c98e 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -533,7 +533,7 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping") continue } - + log.Warn().Any("actionMap", actionMap).Msg("ActionMap") actions = append(actions, actionMap) } diff --git a/uixt/driver.go b/uixt/driver.go index 58d31fc6..81a871e5 100644 --- a/uixt/driver.go +++ b/uixt/driver.go @@ -15,6 +15,10 @@ var ( _ IDriver = (*WDADriver)(nil) _ IDriver = (*HDCDriver)(nil) _ IDriver = (*BrowserDriver)(nil) + + // Ensure drivers implement SIMSupport interface + _ SIMSupport = (*UIA2Driver)(nil) + _ SIMSupport = (*ADBDriver)(nil) ) // current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver @@ -90,3 +94,13 @@ type IDriver interface { // clipboard operations GetPasteboard() (string, error) } + +// SIMSupport interface defines simulated interaction methods +// Any driver that supports simulated touch and input should implement this interface +type SIMSupport interface { + SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error + SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error + SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error + SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error + SIMInput(text string, opts ...option.ActionOption) error +} diff --git a/uixt/mcp_tools_input.go b/uixt/mcp_tools_input.go index 3341c047..64198125 100644 --- a/uixt/mcp_tools_input.go +++ b/uixt/mcp_tools_input.go @@ -240,14 +240,14 @@ func (t *ToolSIMInput) Implement() server.ToolHandlerFunc { opts := unifiedReq.Options() - // Call the underlying SIMInput method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMInput(text, opts...) + // Call the underlying SIMInput method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMInput(text, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMInput is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMInput") + return NewMCPErrorResponse("SIMInput is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } // Estimate segments count (this is approximate since the actual segmentation happens in the driver) diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index 31df991f..a889ae7f 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -631,14 +631,14 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeWithDirection method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + // Call the underlying SIMSwipeWithDirection method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeWithDirection is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeWithDirection") + return NewMCPErrorResponse("SIMSwipeWithDirection is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } // Calculate actual distance for response (approximate) @@ -782,14 +782,14 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeInArea method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + // Call the underlying SIMSwipeInArea method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeInArea is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeInArea") + return NewMCPErrorResponse("SIMSwipeInArea is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", @@ -902,14 +902,14 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMSwipeFromPointToPoint method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + // Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMSwipeFromPointToPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeFromPointToPoint") + return NewMCPErrorResponse("SIMSwipeFromPointToPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", diff --git a/uixt/mcp_tools_touch.go b/uixt/mcp_tools_touch.go index 751b7620..ac35b3c2 100644 --- a/uixt/mcp_tools_touch.go +++ b/uixt/mcp_tools_touch.go @@ -392,14 +392,14 @@ func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc { // Build all options from request arguments opts := unifiedReq.Options() - // Call the underlying SIMClickAtPoint method (Android UIA2 specific) - if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok { - err = uia2Driver.SIMClickAtPoint(x, y, opts...) + // Call the underlying SIMClickAtPoint method (check if driver supports SIM) + if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { + err = simDriver.SIMClickAtPoint(x, y, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err } } else { - return NewMCPErrorResponse("SIMClickAtPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMClickAtPoint") + return NewMCPErrorResponse("SIMClickAtPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface") } message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y) From 9ef98f828d574ad03fcc763fd1b8367c5fd3c226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 11:21:38 +0800 Subject: [PATCH 4/5] fix --- uixt/android_driver_uia2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 3919c98e..2997d030 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -533,7 +533,6 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping") continue } - log.Warn().Any("actionMap", actionMap).Msg("ActionMap") actions = append(actions, actionMap) } From f580ea4168e62e1629b51347c34abd2d2e140b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BC=80=E5=85=83?= Date: Wed, 30 Jul 2025 15:17:40 +0800 Subject: [PATCH 5/5] fix --- step_ui.go | 38 +++---- uixt/android_driver_adb.go | 23 ----- uixt/android_driver_uia2.go | 36 +++---- uixt/driver.go | 7 +- uixt/mcp_tools_swipe.go | 194 ++++++++++++++++++------------------ uixt/option/action.go | 30 ++++-- 6 files changed, 156 insertions(+), 172 deletions(-) diff --git a/step_ui.go b/step_ui.go index f0e0890a..ae4a6233 100644 --- a/step_ui.go +++ b/step_ui.go @@ -291,14 +291,14 @@ func (s *StepMobile) SwipeRight(opts ...option.ActionOption) *StepMobile { } // SIMSwipeWithDirection performs simulated swipe in specified direction with random distance -func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeWithDirection params := map[string]interface{}{ - "direction": direction, - "start_x": startX, - "start_y": startY, - "min_distance": minDistance, - "max_distance": maxDistance, + "direction": direction, + "from_x": fromX, + "from_y": fromY, + "sim_min_distance": simMinDistance, + "sim_max_distance": simMaxDistance, } action := option.MobileAction{ @@ -312,16 +312,16 @@ func (s *StepMobile) SIMSwipeWithDirection(direction string, startX, startY, min } // SIMSwipeInArea performs simulated swipe in specified area with direction and random distance -func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeInArea params := map[string]interface{}{ - "direction": direction, - "area_start_x": areaStartX, - "area_start_y": areaStartY, - "area_end_x": areaEndX, - "area_end_y": areaEndY, - "min_distance": minDistance, - "max_distance": maxDistance, + "direction": direction, + "sim_area_start_x": simAreaStartX, + "sim_area_start_y": simAreaStartY, + "sim_area_end_x": simAreaEndX, + "sim_area_end_y": simAreaEndY, + "sim_min_distance": simMinDistance, + "sim_max_distance": simMaxDistance, } action := option.MobileAction{ @@ -335,13 +335,13 @@ func (s *StepMobile) SIMSwipeInArea(direction string, areaStartX, areaStartY, ar } // SIMSwipeFromPointToPoint performs simulated swipe from point to point -func (s *StepMobile) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) *StepMobile { +func (s *StepMobile) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) *StepMobile { // Create params map for SIMSwipeFromPointToPoint params := map[string]interface{}{ - "start_x": startX, - "start_y": startY, - "end_x": endX, - "end_y": endY, + "from_x": fromX, + "from_y": fromY, + "to_x": toX, + "to_y": toY, } action := option.MobileAction{ diff --git a/uixt/android_driver_adb.go b/uixt/android_driver_adb.go index 070fe47d..0f54b9c3 100644 --- a/uixt/android_driver_adb.go +++ b/uixt/android_driver_adb.go @@ -1273,26 +1273,3 @@ func (ad *ADBDriver) SecondaryClick(x, y float64) (err error) { func (ad *ADBDriver) SecondaryClickBySelector(selector string, options ...option.ActionOption) (err error) { return err } - -// SIMSupport interface implementation for ADBDriver -// These methods return "not supported" errors since SIM functionality is only available in UIA2Driver - -func (ad *ADBDriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMClickAtPoint is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeWithDirection is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeInArea is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { - return fmt.Errorf("SIMSwipeFromPointToPoint is not supported in ADBDriver, please use UIA2Driver instead") -} - -func (ad *ADBDriver) SIMInput(text string, opts ...option.ActionOption) error { - return fmt.Errorf("SIMInput is not supported in ADBDriver, please use UIA2Driver instead") -} diff --git a/uixt/android_driver_uia2.go b/uixt/android_driver_uia2.go index 2997d030..0b3f8810 100644 --- a/uixt/android_driver_uia2.go +++ b/uixt/android_driver_uia2.go @@ -555,10 +555,10 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac // SwipeWithDirection 向指定方向滑动任意距离 // direction: 滑动方向 ("up", "down", "left", "right") -// startX, startY: 起始坐标 -// minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error { - absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) +// fromX, fromY: 起始坐标 +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { + absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY) if err != nil { return err } @@ -568,7 +568,7 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi log.Info().Str("direction", direction). Float64("startX", absStartX).Float64("startY", absStartY). - Float64("minDistance", minDistance).Float64("maxDistance", maxDistance). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). Str("deviceModel", deviceModel). Int("deviceID", deviceParams.DeviceID). Float64("pressure", deviceParams.Pressure). @@ -595,7 +595,7 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi // 使用滑动仿真算法生成触摸事件序列 events, err := simulator.GenerateSlideWithRandomDistance( - absStartX, absStartY, slideDirection, minDistance, maxDistance, + absStartX, absStartY, slideDirection, simMinDistance, simMaxDistance, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) if err != nil { return fmt.Errorf("generate slide events failed: %v", err) @@ -607,15 +607,15 @@ func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, startX, startY, mi // SwipeInArea 在指定区域内向指定方向滑动任意距离 // direction: 滑动方向 ("up", "down", "left", "right") -// areaStartX, areaStartY, areaEndX, areaEndY: 区域范围(相对坐标) -// minDistance, maxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 -func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error { +// simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY: 区域范围(相对坐标) +// simMinDistance, simMaxDistance: 距离范围，如果相等则为固定距离，否则为随机距离 +func (ud *UIA2Driver) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error { // 转换区域坐标为绝对坐标 - absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, areaStartX, areaStartY) + absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, simAreaStartX, simAreaStartY) if err != nil { return err } - absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(ud, areaEndX, areaEndY) + absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(ud, simAreaEndX, simAreaEndY) if err != nil { return err } @@ -635,7 +635,7 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a log.Info().Str("direction", direction). Float64("areaStartX", absAreaStartX).Float64("areaStartY", absAreaStartY). Float64("areaEndX", absAreaEndX).Float64("areaEndY", absAreaEndY). - Float64("minDistance", minDistance).Float64("maxDistance", maxDistance). + Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance). Str("deviceModel", deviceModel). Int("deviceID", deviceParams.DeviceID). Float64("pressure", deviceParams.Pressure). @@ -663,7 +663,7 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a // 使用滑动仿真算法生成区域内滑动的触摸事件序列 events, err := simulator.GenerateSlideInArea( absAreaStartX, absAreaStartY, absAreaEndX, absAreaEndY, - slideDirection, minDistance, maxDistance, + slideDirection, simMinDistance, simMaxDistance, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size) if err != nil { return fmt.Errorf("generate slide in area events failed: %v", err) @@ -674,15 +674,15 @@ func (ud *UIA2Driver) SIMSwipeInArea(direction string, areaStartX, areaStartY, a } // SwipeFromPointToPoint 指定起始点和结束点进行滑动 -// startX, startY: 起始坐标(相对坐标) -// endX, endY: 结束坐标(相对坐标) -func (ud *UIA2Driver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error { +// fromX, fromY: 起始坐标(相对坐标) +// toX, toY: 结束坐标(相对坐标) +func (ud *UIA2Driver) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error { // 转换起始点和结束点为绝对坐标 - absStartX, absStartY, err := convertToAbsolutePoint(ud, startX, startY) + absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY) if err != nil { return err } - absEndX, absEndY, err := convertToAbsolutePoint(ud, endX, endY) + absEndX, absEndY, err := convertToAbsolutePoint(ud, toX, toY) if err != nil { return err } diff --git a/uixt/driver.go b/uixt/driver.go index 81a871e5..dc21be00 100644 --- a/uixt/driver.go +++ b/uixt/driver.go @@ -18,7 +18,6 @@ var ( // Ensure drivers implement SIMSupport interface _ SIMSupport = (*UIA2Driver)(nil) - _ SIMSupport = (*ADBDriver)(nil) ) // current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver @@ -99,8 +98,8 @@ type IDriver interface { // Any driver that supports simulated touch and input should implement this interface type SIMSupport interface { SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error - SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error - SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error - SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error + SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error + SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error + SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error SIMInput(text string, opts ...option.ActionOption) error } diff --git a/uixt/mcp_tools_swipe.go b/uixt/mcp_tools_swipe.go index a889ae7f..23e6fd46 100644 --- a/uixt/mcp_tools_swipe.go +++ b/uixt/mcp_tools_swipe.go @@ -601,31 +601,31 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { direction, validDirections) } - // Default values if not provided - startX := unifiedReq.StartX - startY := unifiedReq.StartY - minDistance := unifiedReq.MinDistance - maxDistance := unifiedReq.MaxDistance + // Default values if not provided - use fromX/fromY instead of startX/startY + fromX := unifiedReq.FromX + fromY := unifiedReq.FromY + simMinDistance := unifiedReq.SIMMinDistance + simMaxDistance := unifiedReq.SIMMaxDistance - if startX == 0 { - startX = 0.5 // default to center + if fromX == 0 { + fromX = 0.5 // default to center } - if startY == 0 { - startY = 0.5 // default to center + if fromY == 0 { + fromY = 0.5 // default to center } - if minDistance == 0 { - minDistance = 100 // default minimum distance + if simMinDistance == 0 { + simMinDistance = 100 // default minimum distance } - if maxDistance == 0 { - maxDistance = 300 // default maximum distance + if simMaxDistance == 0 { + simMaxDistance = 300 // default maximum distance } log.Info(). Str("direction", direction). - Float64("startX", startX). - Float64("startY", startY). - Float64("minDistance", minDistance). - Float64("maxDistance", maxDistance). + Float64("startX", fromX). + Float64("startY", fromY). + Float64("minDistance", simMinDistance). + Float64("maxDistance", simMaxDistance). Msg("performing simulated swipe with direction") // Build all options from request arguments @@ -633,7 +633,7 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeWithDirection method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...) + err = simDriver.SIMSwipeWithDirection(direction, fromX, fromY, simMinDistance, simMaxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err } @@ -642,19 +642,19 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc { } // Calculate actual distance for response (approximate) - actualDistance := minDistance - if maxDistance > minDistance { - actualDistance = minDistance + (maxDistance-minDistance)*0.5 // approximate middle value + actualDistance := simMinDistance + if simMaxDistance > simMinDistance { + actualDistance = simMinDistance + (simMaxDistance-simMinDistance)*0.5 // approximate middle value } message := fmt.Sprintf("Successfully performed simulated swipe %s from (%.2f, %.2f) with distance %.2f", - direction, startX, startY, actualDistance) + direction, fromX, fromY, actualDistance) returnData := ToolSIMSwipeDirection{ Direction: direction, - StartX: startX, - StartY: startY, - MinDistance: minDistance, - MaxDistance: maxDistance, + StartX: fromX, + StartY: fromY, + MinDistance: simMinDistance, + MaxDistance: simMaxDistance, ActualDistance: actualDistance, } @@ -672,18 +672,18 @@ func (t *ToolSIMSwipeDirection) ConvertActionToCallToolRequest(action option.Mob arguments["direction"] = direction } - // Extract coordinates and distances - if startX, exists := paramsMap["start_x"]; exists { - arguments["start_x"] = startX + // Extract coordinates and distances - use new field names directly + if fromX, exists := paramsMap["from_x"]; exists { + arguments["from_x"] = fromX } - if startY, exists := paramsMap["start_y"]; exists { - arguments["start_y"] = startY + if fromY, exists := paramsMap["from_y"]; exists { + arguments["from_y"] = fromY } - if minDistance, exists := paramsMap["min_distance"]; exists { - arguments["min_distance"] = minDistance + if minDistance, exists := paramsMap["sim_min_distance"]; exists { + arguments["sim_min_distance"] = minDistance } - if maxDistance, exists := paramsMap["max_distance"]; exists { - arguments["max_distance"] = maxDistance + if maxDistance, exists := paramsMap["sim_max_distance"]; exists { + arguments["sim_max_distance"] = maxDistance } // Add duration and press duration from options @@ -753,30 +753,30 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { direction, validDirections) } - // Get area coordinates - areaStartX := unifiedReq.AreaStartX - areaStartY := unifiedReq.AreaStartY - areaEndX := unifiedReq.AreaEndX - areaEndY := unifiedReq.AreaEndY - minDistance := unifiedReq.MinDistance - maxDistance := unifiedReq.MaxDistance + // Get area coordinates - use SIM-prefixed fields + simAreaStartX := unifiedReq.SIMAreaStartX + simAreaStartY := unifiedReq.SIMAreaStartY + simAreaEndX := unifiedReq.SIMAreaEndX + simAreaEndY := unifiedReq.SIMAreaEndY + simMinDistance := unifiedReq.SIMMinDistance + simMaxDistance := unifiedReq.SIMMaxDistance // Default values - if minDistance == 0 { - minDistance = 100 + if simMinDistance == 0 { + simMinDistance = 100 } - if maxDistance == 0 { - maxDistance = 300 + if simMaxDistance == 0 { + simMaxDistance = 300 } log.Info(). Str("direction", direction). - Float64("areaStartX", areaStartX). - Float64("areaStartY", areaStartY). - Float64("areaEndX", areaEndX). - Float64("areaEndY", areaEndY). - Float64("minDistance", minDistance). - Float64("maxDistance", maxDistance). + Float64("areaStartX", simAreaStartX). + Float64("areaStartY", simAreaStartY). + Float64("areaEndX", simAreaEndX). + Float64("areaEndY", simAreaEndY). + Float64("minDistance", simMinDistance). + Float64("maxDistance", simMaxDistance). Msg("performing simulated swipe in area") // Build all options from request arguments @@ -784,7 +784,7 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeInArea method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...) + err = simDriver.SIMSwipeInArea(direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err } @@ -793,15 +793,15 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc { } message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)", - direction, areaStartX, areaStartY, areaEndX, areaEndY) + direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY) returnData := ToolSIMSwipeInArea{ Direction: direction, - AreaStartX: areaStartX, - AreaStartY: areaStartY, - AreaEndX: areaEndX, - AreaEndY: areaEndY, - MinDistance: minDistance, - MaxDistance: maxDistance, + AreaStartX: simAreaStartX, + AreaStartY: simAreaStartY, + AreaEndX: simAreaEndX, + AreaEndY: simAreaEndY, + MinDistance: simMinDistance, + MaxDistance: simMaxDistance, } return NewMCPSuccessResponse(message, &returnData), nil @@ -818,24 +818,24 @@ func (t *ToolSIMSwipeInArea) ConvertActionToCallToolRequest(action option.Mobile arguments["direction"] = direction } - // Extract area coordinates and distances - if areaStartX, exists := paramsMap["area_start_x"]; exists { - arguments["area_start_x"] = areaStartX + // Extract area coordinates and distances - use SIM-prefixed field names + if areaStartX, exists := paramsMap["sim_area_start_x"]; exists { + arguments["sim_area_start_x"] = areaStartX } - if areaStartY, exists := paramsMap["area_start_y"]; exists { - arguments["area_start_y"] = areaStartY + if areaStartY, exists := paramsMap["sim_area_start_y"]; exists { + arguments["sim_area_start_y"] = areaStartY } - if areaEndX, exists := paramsMap["area_end_x"]; exists { - arguments["area_end_x"] = areaEndX + if areaEndX, exists := paramsMap["sim_area_end_x"]; exists { + arguments["sim_area_end_x"] = areaEndX } - if areaEndY, exists := paramsMap["area_end_y"]; exists { - arguments["area_end_y"] = areaEndY + if areaEndY, exists := paramsMap["sim_area_end_y"]; exists { + arguments["sim_area_end_y"] = areaEndY } - if minDistance, exists := paramsMap["min_distance"]; exists { - arguments["min_distance"] = minDistance + if minDistance, exists := paramsMap["sim_min_distance"]; exists { + arguments["sim_min_distance"] = minDistance } - if maxDistance, exists := paramsMap["max_distance"]; exists { - arguments["max_distance"] = maxDistance + if maxDistance, exists := paramsMap["sim_max_distance"]; exists { + arguments["sim_max_distance"] = maxDistance } // Add duration and press duration from options @@ -886,17 +886,17 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { return nil, err } - // Get coordinates from arguments - startX := unifiedReq.StartX - startY := unifiedReq.StartY - endX := unifiedReq.ToX // Using existing ToX field - endY := unifiedReq.ToY // Using existing ToY field + // Get coordinates from arguments - use fromX/fromY instead of startX/startY + fromX := unifiedReq.FromX + fromY := unifiedReq.FromY + toX := unifiedReq.ToX + toY := unifiedReq.ToY log.Info(). - Float64("startX", startX). - Float64("startY", startY). - Float64("endX", endX). - Float64("endY", endY). + Float64("startX", fromX). + Float64("startY", fromY). + Float64("endX", toX). + Float64("endY", toY). Msg("performing simulated point to point swipe") // Build all options from request arguments @@ -904,7 +904,7 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { // Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM) if simDriver, ok := driverExt.IDriver.(SIMSupport); ok { - err = simDriver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...) + err = simDriver.SIMSwipeFromPointToPoint(fromX, fromY, toX, toY, opts...) if err != nil { return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err } @@ -913,12 +913,12 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc { } message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)", - startX, startY, endX, endY) + fromX, fromY, toX, toY) returnData := ToolSIMSwipeFromPointToPoint{ - StartX: startX, - StartY: startY, - EndX: endX, - EndY: endY, + StartX: fromX, + StartY: fromY, + EndX: toX, + EndY: toY, } return NewMCPSuccessResponse(message, &returnData), nil @@ -930,18 +930,18 @@ func (t *ToolSIMSwipeFromPointToPoint) ConvertActionToCallToolRequest(action opt if paramsMap, ok := action.Params.(map[string]interface{}); ok { arguments := map[string]any{} - // Extract coordinates - if startX, exists := paramsMap["start_x"]; exists { - arguments["start_x"] = startX + // Extract coordinates - use new field names directly + if fromX, exists := paramsMap["from_x"]; exists { + arguments["from_x"] = fromX } - if startY, exists := paramsMap["start_y"]; exists { - arguments["start_y"] = startY + if fromY, exists := paramsMap["from_y"]; exists { + arguments["from_y"] = fromY } - if endX, exists := paramsMap["end_x"]; exists { - arguments["to_x"] = endX // Map to existing ToX field + if toX, exists := paramsMap["to_x"]; exists { + arguments["to_x"] = toX } - if endY, exists := paramsMap["end_y"]; exists { - arguments["to_y"] = endY // Map to existing ToY field + if toY, exists := paramsMap["to_y"]; exists { + arguments["to_y"] = toY } // Add duration and press duration from options diff --git a/uixt/option/action.go b/uixt/option/action.go index f165cf36..7dfd872c 100644 --- a/uixt/option/action.go +++ b/uixt/option/action.go @@ -206,17 +206,18 @@ type ActionOptions struct { PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"` Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"` Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"` - StartX float64 `json:"start_x,omitempty" yaml:"start_x,omitempty" desc:"Starting X coordinate for simulated swipe"` - StartY float64 `json:"start_y,omitempty" yaml:"start_y,omitempty" desc:"Starting Y coordinate for simulated swipe"` - MinDistance float64 `json:"min_distance,omitempty" yaml:"min_distance,omitempty" desc:"Minimum distance for simulated swipe"` - MaxDistance float64 `json:"max_distance,omitempty" yaml:"max_distance,omitempty" desc:"Maximum distance for simulated swipe"` - AreaStartX float64 `json:"area_start_x,omitempty" yaml:"area_start_x,omitempty" desc:"Area starting X coordinate for simulated swipe"` - AreaStartY float64 `json:"area_start_y,omitempty" yaml:"area_start_y,omitempty" desc:"Area starting Y coordinate for simulated swipe"` - AreaEndX float64 `json:"area_end_x,omitempty" yaml:"area_end_x,omitempty" desc:"Area ending X coordinate for simulated swipe"` - AreaEndY float64 `json:"area_end_y,omitempty" yaml:"area_end_y,omitempty" desc:"Area ending Y coordinate for simulated swipe"` - Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` - TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` - Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` + + // SIM specific options with SIM prefix + SIMMinDistance float64 `json:"sim_min_distance,omitempty" yaml:"sim_min_distance,omitempty" desc:"Minimum distance for SIM simulated actions"` + SIMMaxDistance float64 `json:"sim_max_distance,omitempty" yaml:"sim_max_distance,omitempty" desc:"Maximum distance for SIM simulated actions"` + SIMAreaStartX float64 `json:"sim_area_start_x,omitempty" yaml:"sim_area_start_x,omitempty" desc:"Area starting X coordinate for SIM simulated swipe"` + SIMAreaStartY float64 `json:"sim_area_start_y,omitempty" yaml:"sim_area_start_y,omitempty" desc:"Area starting Y coordinate for SIM simulated swipe"` + SIMAreaEndX float64 `json:"sim_area_end_x,omitempty" yaml:"sim_area_end_x,omitempty" desc:"Area ending X coordinate for SIM simulated swipe"` + SIMAreaEndY float64 `json:"sim_area_end_y,omitempty" yaml:"sim_area_end_y,omitempty" desc:"Area ending Y coordinate for SIM simulated swipe"` + + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"` + TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"` + Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"` ScreenOptions @@ -662,6 +663,13 @@ func (o *ActionOptions) GetMCPOptions(actionType ActionName) []mcp.ToolOption { ACTION_Back: {"platform", "serial"}, ACTION_ListPackages: {"platform", "serial"}, ACTION_ClosePopups: {"platform", "serial"}, + + // SIM specific actions using fromX/fromY for startX/startY and SIM-prefixed fields + ACTION_SIMSwipeDirection: {"platform", "serial", "direction", "fromX", "fromY", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"}, + ACTION_SIMSwipeInArea: {"platform", "serial", "direction", "sim_area_start_x", "sim_area_start_y", "sim_area_end_x", "sim_area_end_y", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"}, + ACTION_SIMSwipeFromPointToPoint: {"platform", "serial", "fromX", "fromY", "toX", "toY", "duration", "pressDuration"}, + ACTION_SIMClickAtPoint: {"platform", "serial", "x", "y", "duration", "pressDuration"}, + ACTION_SIMInput: {"platform", "serial", "text", "frequency"}, } fields := fieldMappings[actionType]