add mcp_sim

This commit is contained in:
张开元
2025-07-30 11:18:26 +08:00
parent 9b529eb125
commit ff13d90755
9 changed files with 101 additions and 23 deletions

View File

@@ -1273,3 +1273,26 @@ func (ad *ADBDriver) SecondaryClick(x, y float64) (err error) {
func (ad *ADBDriver) SecondaryClickBySelector(selector string, options ...option.ActionOption) (err error) {
return err
}
// SIMSupport interface implementation for ADBDriver
// These methods return "not supported" errors since SIM functionality is only available in UIA2Driver
func (ad *ADBDriver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error {
return fmt.Errorf("SIMClickAtPoint is not supported in ADBDriver, please use UIA2Driver instead")
}
func (ad *ADBDriver) SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error {
return fmt.Errorf("SIMSwipeWithDirection is not supported in ADBDriver, please use UIA2Driver instead")
}
func (ad *ADBDriver) SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error {
return fmt.Errorf("SIMSwipeInArea is not supported in ADBDriver, please use UIA2Driver instead")
}
func (ad *ADBDriver) SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error {
return fmt.Errorf("SIMSwipeFromPointToPoint is not supported in ADBDriver, please use UIA2Driver instead")
}
func (ad *ADBDriver) SIMInput(text string, opts ...option.ActionOption) error {
return fmt.Errorf("SIMInput is not supported in ADBDriver, please use UIA2Driver instead")
}

View File

@@ -533,7 +533,7 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac
log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping")
continue
}
log.Warn().Any("actionMap", actionMap).Msg("ActionMap")
actions = append(actions, actionMap)
}

View File

@@ -15,6 +15,10 @@ var (
_ IDriver = (*WDADriver)(nil)
_ IDriver = (*HDCDriver)(nil)
_ IDriver = (*BrowserDriver)(nil)
// Ensure drivers implement SIMSupport interface
_ SIMSupport = (*UIA2Driver)(nil)
_ SIMSupport = (*ADBDriver)(nil)
)
// current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver
@@ -90,3 +94,13 @@ type IDriver interface {
// clipboard operations
GetPasteboard() (string, error)
}
// SIMSupport interface defines simulated interaction methods
// Any driver that supports simulated touch and input should implement this interface
type SIMSupport interface {
SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error
SIMSwipeWithDirection(direction string, startX, startY, minDistance, maxDistance float64, opts ...option.ActionOption) error
SIMSwipeInArea(direction string, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance float64, opts ...option.ActionOption) error
SIMSwipeFromPointToPoint(startX, startY, endX, endY float64, opts ...option.ActionOption) error
SIMInput(text string, opts ...option.ActionOption) error
}

View File

@@ -240,14 +240,14 @@ func (t *ToolSIMInput) Implement() server.ToolHandlerFunc {
opts := unifiedReq.Options()
// Call the underlying SIMInput method (Android UIA2 specific)
if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok {
err = uia2Driver.SIMInput(text, opts...)
// Call the underlying SIMInput method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMInput(text, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMInput is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMInput")
return NewMCPErrorResponse("SIMInput is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
// Estimate segments count (this is approximate since the actual segmentation happens in the driver)

View File

@@ -631,14 +631,14 @@ func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc {
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeWithDirection method (Android UIA2 specific)
if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok {
err = uia2Driver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...)
// Call the underlying SIMSwipeWithDirection method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeWithDirection(direction, startX, startY, minDistance, maxDistance, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeWithDirection is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeWithDirection")
return NewMCPErrorResponse("SIMSwipeWithDirection is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
// Calculate actual distance for response (approximate)
@@ -782,14 +782,14 @@ func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc {
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeInArea method (Android UIA2 specific)
if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok {
err = uia2Driver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...)
// Call the underlying SIMSwipeInArea method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeInArea(direction, areaStartX, areaStartY, areaEndX, areaEndY, minDistance, maxDistance, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeInArea is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeInArea")
return NewMCPErrorResponse("SIMSwipeInArea is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)",
@@ -902,14 +902,14 @@ func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc {
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeFromPointToPoint method (Android UIA2 specific)
if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok {
err = uia2Driver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...)
// Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeFromPointToPoint(startX, startY, endX, endY, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeFromPointToPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMSwipeFromPointToPoint")
return NewMCPErrorResponse("SIMSwipeFromPointToPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)",

View File

@@ -392,14 +392,14 @@ func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc {
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMClickAtPoint method (Android UIA2 specific)
if uia2Driver, ok := driverExt.IDriver.(*UIA2Driver); ok {
err = uia2Driver.SIMClickAtPoint(x, y, opts...)
// Call the underlying SIMClickAtPoint method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMClickAtPoint(x, y, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMClickAtPoint is only supported on Android UIA2 driver"), fmt.Errorf("unsupported driver type for SIMClickAtPoint")
return NewMCPErrorResponse("SIMClickAtPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y)