Merge branch 'master' into wings_interface_merge

This commit is contained in:
余泓铮
2025-07-30 21:33:55 +08:00
17 changed files with 3349 additions and 116 deletions

View File

@@ -12,6 +12,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/code"
"github.com/httprunner/httprunner/v5/internal/simulation"
"github.com/httprunner/httprunner/v5/internal/utf7"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/httprunner/httprunner/v5/uixt/types"
@@ -532,7 +533,6 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac
log.Warn().Int("action", event.Action).Msg("Unknown action type, skipping")
continue
}
actions = append(actions, actionMap)
}
@@ -553,6 +553,201 @@ func (ud *UIA2Driver) TouchByEvents(events []types.TouchEvent, opts ...option.Ac
return err
}
// SwipeWithDirection 向指定方向滑动任意距离
// direction: 滑动方向 ("up", "down", "left", "right")
// fromX, fromY: 起始坐标
// simMinDistance, simMaxDistance: 距离范围,如果相等则为固定距离,否则为随机距离
func (ud *UIA2Driver) SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error {
absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY)
if err != nil {
return err
}
// 获取设备型号和配置参数
deviceModel, _ := ud.Device.Model()
deviceParams := simulation.GetRandomDeviceParams(deviceModel)
log.Info().Str("direction", direction).
Float64("startX", absStartX).Float64("startY", absStartY).
Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance).
Str("deviceModel", deviceModel).
Int("deviceID", deviceParams.DeviceID).
Float64("pressure", deviceParams.Pressure).
Float64("size", deviceParams.Size).
Msg("UIA2Driver.SwipeWithDirection")
// 导入滑动仿真库
simulator := simulation.NewSlideSimulatorAPI(nil)
// 转换方向字符串为Direction类型
var slideDirection simulation.Direction
switch direction {
case "up":
slideDirection = simulation.Up
case "down":
slideDirection = simulation.Down
case "left":
slideDirection = simulation.Left
case "right":
slideDirection = simulation.Right
default:
return fmt.Errorf("invalid direction: %s, must be one of: up, down, left, right", direction)
}
// 使用滑动仿真算法生成触摸事件序列
events, err := simulator.GenerateSlideWithRandomDistance(
absStartX, absStartY, slideDirection, simMinDistance, simMaxDistance,
deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size)
if err != nil {
return fmt.Errorf("generate slide events failed: %v", err)
}
// 执行触摸事件序列
return ud.TouchByEvents(events, opts...)
}
// SwipeInArea 在指定区域内向指定方向滑动任意距离
// direction: 滑动方向 ("up", "down", "left", "right")
// simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY: 区域范围(相对坐标)
// simMinDistance, simMaxDistance: 距离范围,如果相等则为固定距离,否则为随机距离
func (ud *UIA2Driver) SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error {
// 转换区域坐标为绝对坐标
absAreaStartX, absAreaStartY, err := convertToAbsolutePoint(ud, simAreaStartX, simAreaStartY)
if err != nil {
return err
}
absAreaEndX, absAreaEndY, err := convertToAbsolutePoint(ud, simAreaEndX, simAreaEndY)
if err != nil {
return err
}
// 确保区域坐标正确(start应该小于等于end)
if absAreaStartX > absAreaEndX {
absAreaStartX, absAreaEndX = absAreaEndX, absAreaStartX
}
if absAreaStartY > absAreaEndY {
absAreaStartY, absAreaEndY = absAreaEndY, absAreaStartY
}
// 获取设备型号和配置参数
deviceModel, _ := ud.Device.Model()
deviceParams := simulation.GetRandomDeviceParams(deviceModel)
log.Info().Str("direction", direction).
Float64("areaStartX", absAreaStartX).Float64("areaStartY", absAreaStartY).
Float64("areaEndX", absAreaEndX).Float64("areaEndY", absAreaEndY).
Float64("minDistance", simMinDistance).Float64("maxDistance", simMaxDistance).
Str("deviceModel", deviceModel).
Int("deviceID", deviceParams.DeviceID).
Float64("pressure", deviceParams.Pressure).
Float64("size", deviceParams.Size).
Msg("UIA2Driver.SwipeInArea")
// 导入滑动仿真库
simulator := simulation.NewSlideSimulatorAPI(nil)
// 转换方向字符串为Direction类型
var slideDirection simulation.Direction
switch direction {
case "up":
slideDirection = simulation.Up
case "down":
slideDirection = simulation.Down
case "left":
slideDirection = simulation.Left
case "right":
slideDirection = simulation.Right
default:
return fmt.Errorf("invalid direction: %s, must be one of: up, down, left, right", direction)
}
// 使用滑动仿真算法生成区域内滑动的触摸事件序列
events, err := simulator.GenerateSlideInArea(
absAreaStartX, absAreaStartY, absAreaEndX, absAreaEndY,
slideDirection, simMinDistance, simMaxDistance,
deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size)
if err != nil {
return fmt.Errorf("generate slide in area events failed: %v", err)
}
// 执行触摸事件序列
return ud.TouchByEvents(events, opts...)
}
// SwipeFromPointToPoint 指定起始点和结束点进行滑动
// fromX, fromY: 起始坐标(相对坐标)
// toX, toY: 结束坐标(相对坐标)
func (ud *UIA2Driver) SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error {
// 转换起始点和结束点为绝对坐标
absStartX, absStartY, err := convertToAbsolutePoint(ud, fromX, fromY)
if err != nil {
return err
}
absEndX, absEndY, err := convertToAbsolutePoint(ud, toX, toY)
if err != nil {
return err
}
// 获取设备型号和配置参数
deviceModel, _ := ud.Device.Model()
deviceParams := simulation.GetRandomDeviceParams(deviceModel)
log.Info().Float64("startX", absStartX).Float64("startY", absStartY).
Float64("endX", absEndX).Float64("endY", absEndY).
Str("deviceModel", deviceModel).
Int("deviceID", deviceParams.DeviceID).
Float64("pressure", deviceParams.Pressure).
Float64("size", deviceParams.Size).
Msg("UIA2Driver.SwipeFromPointToPoint")
// 导入滑动仿真库
simulator := simulation.NewSlideSimulatorAPI(nil)
// 使用滑动仿真算法生成点对点滑动的触摸事件序列
events, err := simulator.GeneratePointToPointSlideEvents(
absStartX, absStartY, absEndX, absEndY,
deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size)
if err != nil {
return fmt.Errorf("generate point to point slide events failed: %v", err)
}
// 执行触摸事件序列
return ud.TouchByEvents(events, opts...)
}
// ClickAtPoint 点击相对坐标
// x, y: 点击坐标(相对坐标)
func (ud *UIA2Driver) SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error {
// 转换为绝对坐标
absX, absY, err := convertToAbsolutePoint(ud, x, y)
if err != nil {
return err
}
// 获取设备型号和配置参数
deviceModel, _ := ud.Device.Model()
deviceParams := simulation.GetRandomDeviceParams(deviceModel)
log.Info().Float64("x", absX).Float64("y", absY).
Str("deviceModel", deviceModel).
Int("deviceID", deviceParams.DeviceID).
Float64("pressure", deviceParams.Pressure).
Float64("size", deviceParams.Size).
Msg("UIA2Driver.ClickAtPoint")
// 导入点击仿真库
clickSimulator := simulation.NewClickSimulatorAPI(nil)
// 使用点击仿真算法生成触摸事件序列
events, err := clickSimulator.GenerateClickEvents(
absX, absY, deviceParams.DeviceID, deviceParams.Pressure, deviceParams.Size)
if err != nil {
return fmt.Errorf("generate click events failed: %v", err)
}
// 执行触摸事件序列
return ud.TouchByEvents(events, opts...)
}
func (ud *UIA2Driver) SetPasteboard(contentType types.PasteboardType, content string) (err error) {
log.Info().Str("contentType", string(contentType)).
Str("content", content).Msg("UIA2Driver.SetPasteboard")
@@ -593,6 +788,72 @@ func (ud *UIA2Driver) Input(text string, opts ...option.ActionOption) (err error
return
}
// SIMInput 仿真输入函数,模拟人类分批输入行为
// 将文本智能分割英文单词和数字保持完整中文按1-2个字符分割
func (ud *UIA2Driver) SIMInput(text string, opts ...option.ActionOption) error {
log.Info().Str("text", text).Msg("UIA2Driver.SIMInput")
if text == "" {
return nil
}
// 创建输入仿真器(使用默认配置)
inputSimulator := simulation.NewInputSimulatorAPI(nil)
// 生成输入片段(使用智能分割算法,所有参数使用默认值)
inputReq := simulation.InputRequest{
Text: text,
// MinSegmentLen, MaxSegmentLen, MinDelayMs, MaxDelayMs 使用默认值
}
response := inputSimulator.GenerateInputSegments(inputReq)
if !response.Success {
return fmt.Errorf("failed to generate input segments: %s", response.Message)
}
log.Info().Int("segments", response.Metrics.TotalSegments).
Int("totalDelayMs", response.Metrics.TotalDelayMs).
Int("estimatedTimeMs", response.Metrics.EstimatedTimeMs).
Msg("Input segments generated")
// 逐个输入每个片段
var segmentErrCnt int
for _, segment := range response.Segments {
// 使用SendUnicodeKeys进行输入内部已包含Session.POST请求
segmentErr := ud.SendUnicodeKeys(segment.Text, opts...)
if segmentErr != nil {
segmentErrCnt++
log.Info().Err(segmentErr).Int("segmentErrCnt", segmentErrCnt).
Msg("segments err")
}
log.Debug().Str("segment", segment.Text).Int("index", segment.Index).
Int("charLen", segment.CharLen).Msg("Successfully input segment")
// 如果有延迟时间,则等待
if segment.DelayMs > 0 {
time.Sleep(time.Duration(segment.DelayMs) * time.Millisecond)
log.Debug().Int("delayMs", segment.DelayMs).
Msg("Delay between input segments")
}
}
if segmentErrCnt > 0 {
data := map[string]interface{}{
"text": text,
}
option.MergeOptions(data, opts...)
urlStr := fmt.Sprintf("/session/%s/keys", ud.Session.ID)
_, err := ud.Session.POST(data, urlStr)
return err
}
log.Info().Int("totalSegments", response.Metrics.TotalSegments).
Int("actualDelayMs", response.Metrics.TotalDelayMs).
Msg("SIMInput completed successfully")
return nil
}
func (ud *UIA2Driver) SendUnicodeKeys(text string, opts ...option.ActionOption) (err error) {
log.Info().Str("text", text).Msg("UIA2Driver.SendUnicodeKeys")
// If the Unicode IME is not installed, fall back to the old interface.

View File

@@ -323,7 +323,6 @@ func createXTDriverWithConfig(config DriverCacheConfig) (*XTDriver, error) {
// Default AI options
aiOpts = []option.AIServiceOption{
option.WithCVService(option.CVServiceTypeVEDEM),
option.WithLLMConfig(option.RecommendedConfigurations()["ui_focused"]),
}
}

View File

@@ -15,6 +15,9 @@ var (
_ IDriver = (*WDADriver)(nil)
_ IDriver = (*HDCDriver)(nil)
_ IDriver = (*BrowserDriver)(nil)
// Ensure drivers implement SIMSupport interface
_ SIMSupport = (*UIA2Driver)(nil)
)
// current implemeted driver: ADBDriver, UIA2Driver, WDADriver, HDCDriver
@@ -90,3 +93,13 @@ type IDriver interface {
// clipboard operations
GetPasteboard() (string, error)
}
// SIMSupport interface defines simulated interaction methods
// Any driver that supports simulated touch and input should implement this interface
type SIMSupport interface {
SIMClickAtPoint(x, y float64, opts ...option.ActionOption) error
SIMSwipeWithDirection(direction string, fromX, fromY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error
SIMSwipeInArea(direction string, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance float64, opts ...option.ActionOption) error
SIMSwipeFromPointToPoint(fromX, fromY, toX, toY float64, opts ...option.ActionOption) error
SIMInput(text string, opts ...option.ActionOption) error
}

View File

@@ -7,7 +7,6 @@ import (
"image"
"image/color"
"image/draw"
"image/gif"
"image/jpeg"
"image/png"
"math"
@@ -244,8 +243,8 @@ func getScreenShotBuffer(driver IDriver) (compressedBufSource *bytes.Buffer, err
"take screenshot failed %v", err)
}
// compress screenshot
compressBufSource, err := compressImageBufferWithOptions(bufSource, false, 800)
// compress screenshot with quality 95
compressBufSource, err := compressImageBufferWithOptions(bufSource, 95)
if err != nil {
return nil, errors.Wrapf(code.DeviceScreenShotError,
"compress screenshot failed %v", err)
@@ -262,11 +261,7 @@ func saveScreenShot(raw *bytes.Buffer, screenshotPath string) error {
log.Error().Err(err).Msg("copy screenshot buffer failed")
}
img, format, err := image.Decode(copiedBuffer)
if err != nil {
return errors.Wrap(err, "decode screenshot image failed")
}
// create file
file, err := os.Create(screenshotPath)
if err != nil {
return errors.Wrap(err, "create screenshot image file failed")
@@ -275,26 +270,10 @@ func saveScreenShot(raw *bytes.Buffer, screenshotPath string) error {
_ = file.Close()
}()
// compress image and save to file
switch format {
case "jpeg":
jpegOptions := &jpeg.Options{Quality: 95}
err = jpeg.Encode(file, img, jpegOptions)
case "png":
encoder := png.Encoder{
CompressionLevel: png.BestCompression,
}
err = encoder.Encode(file, img)
case "gif":
gifOptions := &gif.Options{
NumColors: 256,
}
err = gif.Encode(file, img, gifOptions)
default:
return fmt.Errorf("unsupported image format %s", format)
}
// directly write compressed JPEG data to avoid quality loss
_, err = file.Write(copiedBuffer.Bytes())
if err != nil {
return errors.Wrap(err, "save image file failed")
return errors.Wrap(err, "write image file failed")
}
var fileSize int64
@@ -303,14 +282,14 @@ func saveScreenShot(raw *bytes.Buffer, screenshotPath string) error {
fileSize = fileInfo.Size()
}
log.Info().Str("path", screenshotPath).
Int("rawBytes", raw.Len()).Int64("saveBytes", fileSize).
Int64("fileSize", fileSize).
Msg("save screenshot file success")
return nil
}
// compressImageBufferWithOptions compresses image buffer with advanced options
func compressImageBufferWithOptions(raw *bytes.Buffer, enableResize bool, maxWidth int) (compressed *bytes.Buffer, err error) {
func compressImageBufferWithOptions(raw *bytes.Buffer, quality int) (compressed *bytes.Buffer, err error) {
rawSize := raw.Len()
// decode image from buffer
img, format, err := image.Decode(raw)
@@ -318,32 +297,12 @@ func compressImageBufferWithOptions(raw *bytes.Buffer, enableResize bool, maxWid
return nil, err
}
// Get original image dimensions
bounds := img.Bounds()
originalWidth := bounds.Dx()
originalHeight := bounds.Dy()
// Calculate new dimensions for compression if resize is enabled
var newWidth, newHeight int
var resizedImg image.Image = img
if enableResize && originalWidth > maxWidth {
ratio := float64(maxWidth) / float64(originalWidth)
newWidth = maxWidth
newHeight = int(float64(originalHeight) * ratio)
resizedImg = resizeImage(img, newWidth, newHeight)
} else {
newWidth = originalWidth
newHeight = originalHeight
}
jpegQuality := 95
var buf bytes.Buffer
switch format {
case "jpeg", "jpg", "png":
// compress with compression rate of 95
jpegOptions := &jpeg.Options{Quality: jpegQuality}
err = jpeg.Encode(&buf, resizedImg, jpegOptions)
// compress with compression rate
jpegOptions := &jpeg.Options{Quality: quality}
err = jpeg.Encode(&buf, img, jpegOptions)
if err != nil {
return nil, err
}
@@ -354,55 +313,18 @@ func compressImageBufferWithOptions(raw *bytes.Buffer, enableResize bool, maxWid
compressedSize := buf.Len()
log.Debug().
Int("rawSize", rawSize).
Int("originalWidth", originalWidth).
Int("originalHeight", originalHeight).
Int("newWidth", newWidth).
Int("newHeight", newHeight).
Int("jpegQuality", jpegQuality).
Int("quality", quality).
Int("compressedSize", compressedSize).
Bool("resized", enableResize && originalWidth > maxWidth).
Msg("compress image buffer")
// return compressed image buffer
return &buf, nil
}
// resizeImage resizes an image using simple nearest neighbor algorithm
func resizeImage(src image.Image, width, height int) image.Image {
srcBounds := src.Bounds()
srcWidth := srcBounds.Dx()
srcHeight := srcBounds.Dy()
// Create a new image with the target dimensions
dst := image.NewRGBA(image.Rect(0, 0, width, height))
// Simple nearest neighbor resizing
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
// Map destination coordinates to source coordinates
srcX := x * srcWidth / width
srcY := y * srcHeight / height
// Ensure we don't go out of bounds
if srcX >= srcWidth {
srcX = srcWidth - 1
}
if srcY >= srcHeight {
srcY = srcHeight - 1
}
// Copy pixel from source to destination
dst.Set(x, y, src.At(srcBounds.Min.X+srcX, srcBounds.Min.Y+srcY))
}
}
return dst
}
// CompressImageFile compresses an image file and returns the compressed data
func CompressImageFile(imagePath string, enableResize bool, maxWidth int) ([]byte, error) {
log.Debug().Str("imagePath", imagePath).Bool("enableResize", enableResize).
Int("maxWidth", maxWidth).Msg("compress image file")
func CompressImageFile(imagePath string, quality int) ([]byte, error) {
log.Debug().Str("imagePath", imagePath).
Int("quality", quality).Msg("compress image file")
// Read the original image file
file, err := os.Open(imagePath)
@@ -419,7 +341,7 @@ func CompressImageFile(imagePath string, enableResize bool, maxWidth int) ([]byt
}
// Compress using the buffer compression function
compressedBuf, err := compressImageBufferWithOptions(&buf, enableResize, maxWidth)
compressedBuf, err := compressImageBufferWithOptions(&buf, quality)
if err != nil {
return nil, fmt.Errorf("failed to compress image: %w", err)
}

View File

@@ -87,23 +87,28 @@ func (s *MCPServer4XTDriver) registerTools() {
s.registerTool(&ToolSelectDevice{}) // SelectDevice
// Touch Tools
s.registerTool(&ToolTapXY{}) // tap xy
s.registerTool(&ToolTapAbsXY{}) // tap abs xy
s.registerTool(&ToolTapByOCR{}) // tap by OCR
s.registerTool(&ToolTapByCV{}) // tap by CV
s.registerTool(&ToolDoubleTapXY{}) // double tap xy
s.registerTool(&ToolTapXY{}) // tap xy
s.registerTool(&ToolTapAbsXY{}) // tap abs xy
s.registerTool(&ToolTapByOCR{}) // tap by OCR
s.registerTool(&ToolTapByCV{}) // tap by CV
s.registerTool(&ToolDoubleTapXY{}) // double tap xy
s.registerTool(&ToolSIMClickAtPoint{}) // simulated click at point
// Swipe Tools
s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate
s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right
s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY]
s.registerTool(&ToolSwipe{}) // generic swipe, auto-detect direction or coordinate
s.registerTool(&ToolSwipeDirection{}) // swipe direction, up/down/left/right
s.registerTool(&ToolSwipeCoordinate{}) // swipe coordinate, [fromX, fromY, toX, toY]
s.registerTool(&ToolSIMSwipeDirection{}) // simulated swipe direction with random distance
s.registerTool(&ToolSIMSwipeInArea{}) // simulated swipe in area with direction and distance
s.registerTool(&ToolSIMSwipeFromPointToPoint{}) // simulated swipe from point to point
s.registerTool(&ToolSwipeToTapApp{})
s.registerTool(&ToolSwipeToTapText{})
s.registerTool(&ToolSwipeToTapTexts{})
s.registerTool(&ToolDrag{})
// Input Tools
s.registerTool(&ToolInput{})
s.registerTool(&ToolInput{}) // regular input
s.registerTool(&ToolSIMInput{}) // simulated input with intelligent segmentation
s.registerTool(&ToolBackspace{})
s.registerTool(&ToolSetIme{})

View File

@@ -6,6 +6,7 @@ import (
"github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/uixt/option"
)
@@ -192,3 +193,83 @@ func (t *ToolBackspace) ConvertActionToCallToolRequest(action option.MobileActio
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}
// ToolSIMInput implements the sim_input tool call.
type ToolSIMInput struct {
// Return data fields - these define the structure of data returned by this tool
Text string `json:"text" desc:"Text that was input with simulation"`
Segments int `json:"segments" desc:"Number of segments the text was split into"`
}
func (t *ToolSIMInput) Name() option.ActionName {
return option.ACTION_SIMInput
}
func (t *ToolSIMInput) Description() string {
return "Input text with intelligent segmentation and human-like typing patterns"
}
func (t *ToolSIMInput) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SIMInput)
}
func (t *ToolSIMInput) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
arguments := request.GetArguments()
driverExt, err := setupXTDriver(ctx, arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(arguments)
if err != nil {
return nil, err
}
if unifiedReq.Text == "" {
return nil, fmt.Errorf("text is required")
}
text := unifiedReq.Text
log.Info().
Str("text", text).
Int("textLength", len(text)).
Msg("performing simulated input")
opts := unifiedReq.Options()
// Call the underlying SIMInput method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMInput(text, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated input failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMInput is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
// Estimate segments count (this is approximate since the actual segmentation happens in the driver)
estimatedSegments := len([]rune(text))/2 + 1
if estimatedSegments < 1 {
estimatedSegments = 1
}
message := fmt.Sprintf("Successfully performed simulated input: %s", text)
returnData := ToolSIMInput{
Text: text,
Segments: estimatedSegments,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolSIMInput) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
text := fmt.Sprintf("%v", action.Params)
arguments := map[string]any{
"text": text,
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}

View File

@@ -547,3 +547,412 @@ func (t *ToolDrag) ConvertActionToCallToolRequest(action option.MobileAction) (m
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid drag parameters: %v", action.Params)
}
// ToolSIMSwipeDirection implements the sim_swipe_direction tool call.
type ToolSIMSwipeDirection struct {
// Return data fields - these define the structure of data returned by this tool
Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"`
StartX float64 `json:"startX" desc:"Starting X coordinate of the simulated swipe"`
StartY float64 `json:"startY" desc:"Starting Y coordinate of the simulated swipe"`
MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"`
MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"`
ActualDistance float64 `json:"actualDistance" desc:"Actual distance of the simulated swipe"`
}
func (t *ToolSIMSwipeDirection) Name() option.ActionName {
return option.ACTION_SIMSwipeDirection
}
func (t *ToolSIMSwipeDirection) Description() string {
return "Perform simulated swipe in specified direction with random distance and human-like touch patterns"
}
func (t *ToolSIMSwipeDirection) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeDirection)
}
func (t *ToolSIMSwipeDirection) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
arguments := request.GetArguments()
driverExt, err := setupXTDriver(ctx, arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(arguments)
if err != nil {
return nil, err
}
// Validate required parameters
if unifiedReq.Direction == nil {
return nil, fmt.Errorf("direction parameter is required")
}
direction, ok := unifiedReq.Direction.(string)
if !ok {
return nil, fmt.Errorf("direction must be a string")
}
// Validate direction
validDirections := []string{"up", "down", "left", "right"}
if !slices.Contains(validDirections, direction) {
return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v",
direction, validDirections)
}
// Default values if not provided - use fromX/fromY instead of startX/startY
fromX := unifiedReq.FromX
fromY := unifiedReq.FromY
simMinDistance := unifiedReq.SIMMinDistance
simMaxDistance := unifiedReq.SIMMaxDistance
if fromX == 0 {
fromX = 0.5 // default to center
}
if fromY == 0 {
fromY = 0.5 // default to center
}
if simMinDistance == 0 {
simMinDistance = 100 // default minimum distance
}
if simMaxDistance == 0 {
simMaxDistance = 300 // default maximum distance
}
log.Info().
Str("direction", direction).
Float64("startX", fromX).
Float64("startY", fromY).
Float64("minDistance", simMinDistance).
Float64("maxDistance", simMaxDistance).
Msg("performing simulated swipe with direction")
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeWithDirection method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeWithDirection(direction, fromX, fromY, simMinDistance, simMaxDistance, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeWithDirection is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
// Calculate actual distance for response (approximate)
actualDistance := simMinDistance
if simMaxDistance > simMinDistance {
actualDistance = simMinDistance + (simMaxDistance-simMinDistance)*0.5 // approximate middle value
}
message := fmt.Sprintf("Successfully performed simulated swipe %s from (%.2f, %.2f) with distance %.2f",
direction, fromX, fromY, actualDistance)
returnData := ToolSIMSwipeDirection{
Direction: direction,
StartX: fromX,
StartY: fromY,
MinDistance: simMinDistance,
MaxDistance: simMaxDistance,
ActualDistance: actualDistance,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolSIMSwipeDirection) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
// Handle params as map[string]interface{}
if paramsMap, ok := action.Params.(map[string]interface{}); ok {
arguments := map[string]any{}
// Extract direction
if direction, exists := paramsMap["direction"]; exists {
arguments["direction"] = direction
}
// Extract coordinates and distances - use new field names directly
if fromX, exists := paramsMap["from_x"]; exists {
arguments["from_x"] = fromX
}
if fromY, exists := paramsMap["from_y"]; exists {
arguments["from_y"] = fromY
}
if minDistance, exists := paramsMap["sim_min_distance"]; exists {
arguments["sim_min_distance"] = minDistance
}
if maxDistance, exists := paramsMap["sim_max_distance"]; exists {
arguments["sim_max_distance"] = maxDistance
}
// Add duration and press duration from options
if duration := action.ActionOptions.Duration; duration > 0 {
arguments["duration"] = duration
}
if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 {
arguments["pressDuration"] = pressDuration
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe direction params: %v", action.Params)
}
// ToolSIMSwipeInArea implements the sim_swipe_in_area tool call.
type ToolSIMSwipeInArea struct {
// Return data fields - these define the structure of data returned by this tool
Direction string `json:"direction" desc:"Direction that was swiped (up/down/left/right)"`
AreaStartX float64 `json:"areaStartX" desc:"Area starting X coordinate"`
AreaStartY float64 `json:"areaStartY" desc:"Area starting Y coordinate"`
AreaEndX float64 `json:"areaEndX" desc:"Area ending X coordinate"`
AreaEndY float64 `json:"areaEndY" desc:"Area ending Y coordinate"`
MinDistance float64 `json:"minDistance" desc:"Minimum distance of the simulated swipe"`
MaxDistance float64 `json:"maxDistance" desc:"Maximum distance of the simulated swipe"`
}
func (t *ToolSIMSwipeInArea) Name() option.ActionName {
return option.ACTION_SIMSwipeInArea
}
func (t *ToolSIMSwipeInArea) Description() string {
return "Perform simulated swipe in specified area with direction and random distance"
}
func (t *ToolSIMSwipeInArea) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeInArea)
}
func (t *ToolSIMSwipeInArea) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
arguments := request.GetArguments()
driverExt, err := setupXTDriver(ctx, arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(arguments)
if err != nil {
return nil, err
}
// Validate required parameters
if unifiedReq.Direction == nil {
return nil, fmt.Errorf("direction parameter is required")
}
direction, ok := unifiedReq.Direction.(string)
if !ok {
return nil, fmt.Errorf("direction must be a string")
}
// Validate direction
validDirections := []string{"up", "down", "left", "right"}
if !slices.Contains(validDirections, direction) {
return nil, fmt.Errorf("invalid swipe direction: %s, expected one of: %v",
direction, validDirections)
}
// Get area coordinates - use SIM-prefixed fields
simAreaStartX := unifiedReq.SIMAreaStartX
simAreaStartY := unifiedReq.SIMAreaStartY
simAreaEndX := unifiedReq.SIMAreaEndX
simAreaEndY := unifiedReq.SIMAreaEndY
simMinDistance := unifiedReq.SIMMinDistance
simMaxDistance := unifiedReq.SIMMaxDistance
// Default values
if simMinDistance == 0 {
simMinDistance = 100
}
if simMaxDistance == 0 {
simMaxDistance = 300
}
log.Info().
Str("direction", direction).
Float64("areaStartX", simAreaStartX).
Float64("areaStartY", simAreaStartY).
Float64("areaEndX", simAreaEndX).
Float64("areaEndY", simAreaEndY).
Float64("minDistance", simMinDistance).
Float64("maxDistance", simMaxDistance).
Msg("performing simulated swipe in area")
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeInArea method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeInArea(direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY, simMinDistance, simMaxDistance, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated swipe in area failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeInArea is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated swipe %s in area (%.2f,%.2f)-(%.2f,%.2f)",
direction, simAreaStartX, simAreaStartY, simAreaEndX, simAreaEndY)
returnData := ToolSIMSwipeInArea{
Direction: direction,
AreaStartX: simAreaStartX,
AreaStartY: simAreaStartY,
AreaEndX: simAreaEndX,
AreaEndY: simAreaEndY,
MinDistance: simMinDistance,
MaxDistance: simMaxDistance,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolSIMSwipeInArea) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
// Handle params as map[string]interface{}
if paramsMap, ok := action.Params.(map[string]interface{}); ok {
arguments := map[string]any{}
// Extract direction
if direction, exists := paramsMap["direction"]; exists {
arguments["direction"] = direction
}
// Extract area coordinates and distances - use SIM-prefixed field names
if areaStartX, exists := paramsMap["sim_area_start_x"]; exists {
arguments["sim_area_start_x"] = areaStartX
}
if areaStartY, exists := paramsMap["sim_area_start_y"]; exists {
arguments["sim_area_start_y"] = areaStartY
}
if areaEndX, exists := paramsMap["sim_area_end_x"]; exists {
arguments["sim_area_end_x"] = areaEndX
}
if areaEndY, exists := paramsMap["sim_area_end_y"]; exists {
arguments["sim_area_end_y"] = areaEndY
}
if minDistance, exists := paramsMap["sim_min_distance"]; exists {
arguments["sim_min_distance"] = minDistance
}
if maxDistance, exists := paramsMap["sim_max_distance"]; exists {
arguments["sim_max_distance"] = maxDistance
}
// Add duration and press duration from options
if duration := action.ActionOptions.Duration; duration > 0 {
arguments["duration"] = duration
}
if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 {
arguments["pressDuration"] = pressDuration
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe in area params: %v", action.Params)
}
// ToolSIMSwipeFromPointToPoint implements the sim_swipe_point_to_point tool call.
type ToolSIMSwipeFromPointToPoint struct {
// Return data fields - these define the structure of data returned by this tool
StartX float64 `json:"startX" desc:"Starting X coordinate"`
StartY float64 `json:"startY" desc:"Starting Y coordinate"`
EndX float64 `json:"endX" desc:"Ending X coordinate"`
EndY float64 `json:"endY" desc:"Ending Y coordinate"`
}
func (t *ToolSIMSwipeFromPointToPoint) Name() option.ActionName {
return option.ACTION_SIMSwipeFromPointToPoint
}
func (t *ToolSIMSwipeFromPointToPoint) Description() string {
return "Perform simulated swipe from point to point with human-like touch patterns"
}
func (t *ToolSIMSwipeFromPointToPoint) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SIMSwipeFromPointToPoint)
}
func (t *ToolSIMSwipeFromPointToPoint) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
arguments := request.GetArguments()
driverExt, err := setupXTDriver(ctx, arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(arguments)
if err != nil {
return nil, err
}
// Get coordinates from arguments - use fromX/fromY instead of startX/startY
fromX := unifiedReq.FromX
fromY := unifiedReq.FromY
toX := unifiedReq.ToX
toY := unifiedReq.ToY
log.Info().
Float64("startX", fromX).
Float64("startY", fromY).
Float64("endX", toX).
Float64("endY", toY).
Msg("performing simulated point to point swipe")
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMSwipeFromPointToPoint method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMSwipeFromPointToPoint(fromX, fromY, toX, toY, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated point to point swipe failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMSwipeFromPointToPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated swipe from (%.2f,%.2f) to (%.2f,%.2f)",
fromX, fromY, toX, toY)
returnData := ToolSIMSwipeFromPointToPoint{
StartX: fromX,
StartY: fromY,
EndX: toX,
EndY: toY,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolSIMSwipeFromPointToPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
// Handle params as map[string]interface{}
if paramsMap, ok := action.Params.(map[string]interface{}); ok {
arguments := map[string]any{}
// Extract coordinates - use new field names directly
if fromX, exists := paramsMap["from_x"]; exists {
arguments["from_x"] = fromX
}
if fromY, exists := paramsMap["from_y"]; exists {
arguments["from_y"] = fromY
}
if toX, exists := paramsMap["to_x"]; exists {
arguments["to_x"] = toX
}
if toY, exists := paramsMap["to_y"]; exists {
arguments["to_y"] = toY
}
// Add duration and press duration from options
if duration := action.ActionOptions.Duration; duration > 0 {
arguments["duration"] = duration
}
if pressDuration := action.ActionOptions.PressDuration; pressDuration > 0 {
arguments["pressDuration"] = pressDuration
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM swipe point to point params: %v", action.Params)
}

View File

@@ -6,6 +6,7 @@ import (
"github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
"github.com/rs/zerolog/log"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
@@ -341,3 +342,95 @@ func (t *ToolDoubleTapXY) ConvertActionToCallToolRequest(action option.MobileAct
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid double tap params: %v", action.Params)
}
// ToolSIMClickAtPoint implements the sim_click_at_point tool call.
type ToolSIMClickAtPoint struct {
// Return data fields - these define the structure of data returned by this tool
X float64 `json:"x" desc:"X coordinate where simulated click was performed"`
Y float64 `json:"y" desc:"Y coordinate where simulated click was performed"`
}
func (t *ToolSIMClickAtPoint) Name() option.ActionName {
return option.ACTION_SIMClickAtPoint
}
func (t *ToolSIMClickAtPoint) Description() string {
return "Perform simulated click at specified point with human-like touch patterns"
}
func (t *ToolSIMClickAtPoint) Options() []mcp.ToolOption {
unifiedReq := &option.ActionOptions{}
return unifiedReq.GetMCPOptions(option.ACTION_SIMClickAtPoint)
}
func (t *ToolSIMClickAtPoint) Implement() server.ToolHandlerFunc {
return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) {
arguments := request.GetArguments()
driverExt, err := setupXTDriver(ctx, arguments)
if err != nil {
return nil, fmt.Errorf("setup driver failed: %w", err)
}
unifiedReq, err := parseActionOptions(arguments)
if err != nil {
return nil, err
}
// Validate required parameters
if unifiedReq.X == 0 || unifiedReq.Y == 0 {
return nil, fmt.Errorf("x and y coordinates are required")
}
x := unifiedReq.X
y := unifiedReq.Y
log.Info().
Float64("x", x).
Float64("y", y).
Msg("performing simulated click at point")
// Build all options from request arguments
opts := unifiedReq.Options()
// Call the underlying SIMClickAtPoint method (check if driver supports SIM)
if simDriver, ok := driverExt.IDriver.(SIMSupport); ok {
err = simDriver.SIMClickAtPoint(x, y, opts...)
if err != nil {
return NewMCPErrorResponse(fmt.Sprintf("Simulated click failed: %s", err.Error())), err
}
} else {
return NewMCPErrorResponse("SIMClickAtPoint is not supported by the current driver"), fmt.Errorf("driver does not implement SIMSupport interface")
}
message := fmt.Sprintf("Successfully performed simulated click at (%.2f, %.2f)", x, y)
returnData := ToolSIMClickAtPoint{
X: x,
Y: y,
}
return NewMCPSuccessResponse(message, &returnData), nil
}
}
func (t *ToolSIMClickAtPoint) ConvertActionToCallToolRequest(action option.MobileAction) (mcp.CallToolRequest, error) {
// Handle params as map[string]interface{}
if paramsMap, ok := action.Params.(map[string]interface{}); ok {
arguments := map[string]any{}
// Extract coordinates
if x, exists := paramsMap["x"]; exists {
arguments["x"] = x
}
if y, exists := paramsMap["y"]; exists {
arguments["y"] = y
}
// Add duration from options
if duration := action.ActionOptions.Duration; duration > 0 {
arguments["duration"] = duration
}
return BuildMCPCallToolRequest(t.Name(), arguments, action), nil
}
return mcp.CallToolRequest{}, fmt.Errorf("invalid SIM click at point params: %v", action.Params)
}

View File

@@ -66,9 +66,14 @@ const (
ACTION_TapByCV ActionName = "tap_cv"
ACTION_DoubleTap ActionName = "double_tap" // generic double tap action
ACTION_DoubleTapXY ActionName = "double_tap_xy"
ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates
ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right)
ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY)
ACTION_Swipe ActionName = "swipe" // swipe by direction or coordinates
ACTION_SwipeDirection ActionName = "swipe_direction" // swipe by direction (up, down, left, right)
ACTION_SwipeCoordinate ActionName = "swipe_coordinate" // swipe by coordinates (fromX, fromY, toX, toY)
ACTION_SIMSwipeDirection ActionName = "sim_swipe_direction" // simulated swipe by direction with random distance
ACTION_SIMSwipeInArea ActionName = "sim_swipe_in_area" // simulated swipe in area with direction and distance
ACTION_SIMSwipeFromPointToPoint ActionName = "sim_swipe_point_to_point" // simulated swipe from point to point
ACTION_SIMClickAtPoint ActionName = "sim_click_at_point" // simulated click at point
ACTION_SIMInput ActionName = "sim_input" // simulated text input with segments
ACTION_Drag ActionName = "drag"
ACTION_Input ActionName = "input"
ACTION_PressButton ActionName = "press_button"
@@ -201,9 +206,18 @@ type ActionOptions struct {
PressDuration float64 `json:"press_duration,omitempty" yaml:"press_duration,omitempty" desc:"Press duration in seconds"`
Steps int `json:"steps,omitempty" yaml:"steps,omitempty" desc:"Number of steps for action"`
Direction interface{} `json:"direction,omitempty" yaml:"direction,omitempty" desc:"Direction for swipe operations or custom coordinates"`
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"`
TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"`
Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"`
// SIM specific options with SIM prefix
SIMMinDistance float64 `json:"sim_min_distance,omitempty" yaml:"sim_min_distance,omitempty" desc:"Minimum distance for SIM simulated actions"`
SIMMaxDistance float64 `json:"sim_max_distance,omitempty" yaml:"sim_max_distance,omitempty" desc:"Maximum distance for SIM simulated actions"`
SIMAreaStartX float64 `json:"sim_area_start_x,omitempty" yaml:"sim_area_start_x,omitempty" desc:"Area starting X coordinate for SIM simulated swipe"`
SIMAreaStartY float64 `json:"sim_area_start_y,omitempty" yaml:"sim_area_start_y,omitempty" desc:"Area starting Y coordinate for SIM simulated swipe"`
SIMAreaEndX float64 `json:"sim_area_end_x,omitempty" yaml:"sim_area_end_x,omitempty" desc:"Area ending X coordinate for SIM simulated swipe"`
SIMAreaEndY float64 `json:"sim_area_end_y,omitempty" yaml:"sim_area_end_y,omitempty" desc:"Area ending Y coordinate for SIM simulated swipe"`
Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty" desc:"Timeout in seconds for action execution"`
TimeLimit int `json:"time_limit,omitempty" yaml:"time_limit,omitempty" desc:"Time limit in seconds for action execution, stops gracefully when reached"`
Frequency int `json:"frequency,omitempty" yaml:"frequency,omitempty" desc:"Action frequency"`
ScreenOptions
@@ -649,6 +663,13 @@ func (o *ActionOptions) GetMCPOptions(actionType ActionName) []mcp.ToolOption {
ACTION_Back: {"platform", "serial"},
ACTION_ListPackages: {"platform", "serial"},
ACTION_ClosePopups: {"platform", "serial"},
// SIM specific actions using fromX/fromY for startX/startY and SIM-prefixed fields
ACTION_SIMSwipeDirection: {"platform", "serial", "direction", "fromX", "fromY", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"},
ACTION_SIMSwipeInArea: {"platform", "serial", "direction", "sim_area_start_x", "sim_area_start_y", "sim_area_end_x", "sim_area_end_y", "sim_min_distance", "sim_max_distance", "duration", "pressDuration"},
ACTION_SIMSwipeFromPointToPoint: {"platform", "serial", "fromX", "fromY", "toX", "toY", "duration", "pressDuration"},
ACTION_SIMClickAtPoint: {"platform", "serial", "x", "y", "duration", "pressDuration"},
ACTION_SIMInput: {"platform", "serial", "text", "frequency"},
}
fields := fieldMappings[actionType]