mirror of
https://github.com/httprunner/httprunner.git
synced 2026-05-12 02:21:29 +08:00
merge master
This commit is contained in:
50
code/code.go
50
code/code.go
@@ -66,18 +66,19 @@ var (
|
||||
|
||||
// device related: [50, 70)
|
||||
var (
|
||||
DeviceConnectionError = errors.New("device general connection error") // 50
|
||||
DeviceHTTPDriverError = errors.New("device HTTP driver error") // 51
|
||||
DeviceUSBDriverError = errors.New("device USB driver error") // 52
|
||||
DeviceAppNotInstalled = errors.New("device app not installed") // 59
|
||||
DeviceGetInfoError = errors.New("device get info error") // 60
|
||||
DeviceConfigureError = errors.New("device configure error") // 61
|
||||
DeviceShellExecError = errors.New("device shell exec error") // 62
|
||||
DeviceOfflineError = errors.New("device offline") // 63
|
||||
DeviceInstallFailed = errors.New("device install app failed") // 64
|
||||
DeviceScreenShotError = errors.New("device screenshot error") // 65
|
||||
DeviceCaptureLogError = errors.New("device capture log error") // 66
|
||||
DeviceUIResponseSlow = errors.New("device UI response slow") // 67
|
||||
DeviceConnectionError = errors.New("device general connection error") // 50
|
||||
DeviceHTTPDriverError = errors.New("device HTTP driver error") // 51
|
||||
DeviceUSBDriverError = errors.New("device USB driver error") // 52
|
||||
DeviceUntrustedCertError = errors.New("device app certificate not trusted") // 53
|
||||
DeviceAppNotInstalled = errors.New("device app not installed") // 59
|
||||
DeviceGetInfoError = errors.New("device get info error") // 60
|
||||
DeviceConfigureError = errors.New("device configure error") // 61
|
||||
DeviceShellExecError = errors.New("device shell exec error") // 62
|
||||
DeviceOfflineError = errors.New("device offline") // 63
|
||||
DeviceInstallFailed = errors.New("device install app failed") // 64
|
||||
DeviceScreenShotError = errors.New("device screenshot error") // 65
|
||||
DeviceCaptureLogError = errors.New("device capture log error") // 66
|
||||
DeviceUIResponseSlow = errors.New("device UI response slow") // 67
|
||||
)
|
||||
|
||||
// UI automation related: [70, 80)
|
||||
@@ -176,18 +177,19 @@ var errorsMap = map[error]int{
|
||||
UploadFailed: 49,
|
||||
|
||||
// device related
|
||||
DeviceConnectionError: 50,
|
||||
DeviceHTTPDriverError: 51,
|
||||
DeviceUSBDriverError: 52,
|
||||
DeviceAppNotInstalled: 59,
|
||||
DeviceGetInfoError: 60,
|
||||
DeviceConfigureError: 61,
|
||||
DeviceShellExecError: 62,
|
||||
DeviceOfflineError: 63,
|
||||
DeviceInstallFailed: 64,
|
||||
DeviceScreenShotError: 65,
|
||||
DeviceCaptureLogError: 66,
|
||||
DeviceUIResponseSlow: 67,
|
||||
DeviceConnectionError: 50,
|
||||
DeviceHTTPDriverError: 51,
|
||||
DeviceUSBDriverError: 52,
|
||||
DeviceUntrustedCertError: 53,
|
||||
DeviceAppNotInstalled: 59,
|
||||
DeviceGetInfoError: 60,
|
||||
DeviceConfigureError: 61,
|
||||
DeviceShellExecError: 62,
|
||||
DeviceOfflineError: 63,
|
||||
DeviceInstallFailed: 64,
|
||||
DeviceScreenShotError: 65,
|
||||
DeviceCaptureLogError: 66,
|
||||
DeviceUIResponseSlow: 67,
|
||||
|
||||
// UI automation related
|
||||
MobileUIDriverAppNotInstalled: 68,
|
||||
|
||||
@@ -7,13 +7,14 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
hrp "github.com/httprunner/httprunner/v5"
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/uixt"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// GameElement represents a game element detected in the interface
|
||||
|
||||
@@ -664,14 +664,22 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt
|
||||
tp transport
|
||||
filesize int64
|
||||
)
|
||||
timeout := 8
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
filesize, err = apk.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tp, err = d.createDeviceTransport(5 * time.Minute); err != nil {
|
||||
if tp, err = d.createDeviceTransport(4 * time.Minute); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = tp.Close() }()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = tp.Close()
|
||||
}()
|
||||
cmd := "abb_exec:package\x00install\x00-t"
|
||||
for _, arg := range args {
|
||||
cmd += "\x00" + arg
|
||||
@@ -690,6 +698,9 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt
|
||||
return nil, err
|
||||
}
|
||||
raw, err = tp.ReadBytesAll()
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("installation timed out after %d minutes", timeout)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -6,14 +6,14 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
func TestILLMServiceQuery(t *testing.T) {
|
||||
|
||||
// Create LLM service
|
||||
service, err := NewLLMService(option.OPENAI_GPT_4O)
|
||||
require.NoError(t, err)
|
||||
@@ -80,7 +80,6 @@ func TestILLMServiceQuery(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestILLMServiceIntegration(t *testing.T) {
|
||||
|
||||
// Create LLM service
|
||||
service, err := NewLLMService(option.OPENAI_GPT_4O)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -26,6 +26,7 @@ type WingsService struct {
|
||||
bizId string
|
||||
accessKey string
|
||||
secretKey string
|
||||
history []History // Conversation history for Wings API
|
||||
}
|
||||
|
||||
// NewWingsService creates a new Wings service instance
|
||||
@@ -49,6 +50,7 @@ func NewWingsService() (ILLMService, error) {
|
||||
bizId: bizID,
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
history: []History{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
|
||||
return nil, errors.Wrap(err, "validate planning parameters failed")
|
||||
}
|
||||
|
||||
// Reset history if requested
|
||||
if opts.ResetHistory {
|
||||
w.resetHistory()
|
||||
}
|
||||
|
||||
// Extract screenshot from message
|
||||
screenshot, err := w.extractScreenshotFromMessage(opts.Message)
|
||||
if err != nil {
|
||||
@@ -70,15 +77,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
|
||||
|
||||
// Prepare Wings API request
|
||||
apiRequest := WingsActionRequest{
|
||||
Historys: []interface{}{}, // empty as specified
|
||||
DeviceInfos: []WingsDeviceInfo{
|
||||
deviceInfo,
|
||||
},
|
||||
StepText: opts.UserInstruction,
|
||||
BizId: w.bizId,
|
||||
TextCase: "整体描述:\\n前置条件:\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤:\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言:屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项:\\n",
|
||||
StepType: "automation",
|
||||
DeviceID: deviceInfo.DeviceID,
|
||||
Historys: w.history,
|
||||
DeviceInfos: deviceInfo,
|
||||
StepText: fmt.Sprintf("%s", opts.UserInstruction),
|
||||
BizId: w.bizId,
|
||||
TextCase: fmt.Sprintf("整体描述:\n前置条件:\n操作步骤:\n%s\n停止操作。\n注意事项:\n", opts.UserInstruction),
|
||||
Base: WingsBase{
|
||||
LogID: generateWingsUUID(),
|
||||
},
|
||||
@@ -98,7 +101,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
|
||||
}
|
||||
|
||||
// Check API response status
|
||||
if response.BaseResp.StatusCode != 0 {
|
||||
if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 {
|
||||
err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage)
|
||||
return &PlanningResult{
|
||||
Thought: response.ThoughtChain.Thought,
|
||||
@@ -107,26 +110,48 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
|
||||
}, err
|
||||
}
|
||||
|
||||
// Convert Wings API response to tool calls
|
||||
toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams)
|
||||
if err != nil {
|
||||
return &PlanningResult{
|
||||
Thought: response.ThoughtChain.Thought,
|
||||
Error: err.Error(),
|
||||
ModelName: "wings-api",
|
||||
}, errors.Wrap(err, "convert Wings response to tool calls failed")
|
||||
// Update history with response data
|
||||
newHistoryEntry := History{
|
||||
ThoughtChain: response.ThoughtChain,
|
||||
StepText: response.StepText,
|
||||
StepTextTrans: response.StepTextTrans,
|
||||
OriStepIndex: response.OriStepIndex,
|
||||
DeviceID: deviceInfo[0].DeviceID,
|
||||
AgentType: response.AgentType,
|
||||
ActionResult: "", // Always empty as requested
|
||||
DeviceInfos: &deviceInfo,
|
||||
ActionParams: response.ActionParams,
|
||||
}
|
||||
w.history = append(w.history, newHistoryEntry)
|
||||
var toolCalls []schema.ToolCall
|
||||
if response.StepType != "FINISH" {
|
||||
// Convert Wings API response to tool calls
|
||||
toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams)
|
||||
if err != nil {
|
||||
return &PlanningResult{
|
||||
Thought: response.ThoughtChain.Thought,
|
||||
Error: err.Error(),
|
||||
ModelName: "wings-api",
|
||||
}, errors.Wrap(err, "convert Wings response to tool calls failed")
|
||||
}
|
||||
}
|
||||
|
||||
// No need to update ActionResult as per user request
|
||||
// ActionResult should always be empty
|
||||
|
||||
log.Info().
|
||||
Str("thought", response.ThoughtChain.Thought).
|
||||
Str("action", response.AgentType).
|
||||
Str("action_params", response.ActionParams).
|
||||
Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)).
|
||||
Int("tool_calls_count", len(toolCalls)).
|
||||
Int64("elapsed_ms", elapsed).
|
||||
Msg("Wings API planning completed")
|
||||
|
||||
return &PlanningResult{
|
||||
ToolCalls: toolCalls,
|
||||
Thought: response.ThoughtChain.Thought,
|
||||
Content: response.ThoughtChain.Summary,
|
||||
Thought: response.StepTextTrans,
|
||||
Content: response.StepTextTrans,
|
||||
ModelName: "wings-api",
|
||||
}, nil
|
||||
}
|
||||
@@ -138,28 +163,20 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
|
||||
return nil, errors.Wrap(err, "validate assertion parameters failed")
|
||||
}
|
||||
|
||||
// Clean screenshot data URL prefix
|
||||
cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot)
|
||||
|
||||
// Get device info from context (if available)
|
||||
deviceInfo := w.getDeviceInfoFromScreenshot(ctx, cleanScreenshot)
|
||||
deviceInfos := w.getDeviceInfoFromScreenshot(ctx, opts.Screenshot)
|
||||
|
||||
// Prepare Wings API request for assertion
|
||||
apiRequest := WingsActionRequest{
|
||||
Historys: []interface{}{}, // empty as specified
|
||||
DeviceInfos: []WingsDeviceInfo{
|
||||
deviceInfo,
|
||||
},
|
||||
StepText: opts.Assertion,
|
||||
BizId: w.bizId,
|
||||
TextCase: "整体描述:\\n前置条件:\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤:\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言:屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项:\\n",
|
||||
StepType: "assert", // Different from automation
|
||||
DeviceID: deviceInfo.DeviceID,
|
||||
Historys: []History{},
|
||||
DeviceInfos: deviceInfos,
|
||||
StepText: fmt.Sprintf("断言:%s", opts.Assertion),
|
||||
BizId: w.bizId,
|
||||
TextCase: fmt.Sprintf("整体描述:\n前置条件:\n操作步骤:\n断言: %s\n停止操作。\n注意事项:\n", opts.Assertion),
|
||||
Base: WingsBase{
|
||||
LogID: generateWingsUUID(),
|
||||
},
|
||||
}
|
||||
log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request")
|
||||
|
||||
// Call Wings API
|
||||
startTime := time.Now()
|
||||
@@ -175,7 +192,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
|
||||
}
|
||||
|
||||
// Check API response status
|
||||
if response.BaseResp.StatusCode != 0 {
|
||||
if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 {
|
||||
err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage)
|
||||
return &AssertionResult{
|
||||
Pass: false,
|
||||
@@ -184,6 +201,19 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
|
||||
}, err
|
||||
}
|
||||
|
||||
// Update history with response data
|
||||
newHistoryEntry := History{
|
||||
ThoughtChain: response.ThoughtChain,
|
||||
StepText: response.StepText,
|
||||
StepTextTrans: response.StepTextTrans,
|
||||
OriStepIndex: response.OriStepIndex,
|
||||
DeviceID: response.DeviceId,
|
||||
AgentType: response.AgentType,
|
||||
DeviceInfos: &apiRequest.DeviceInfos,
|
||||
ActionParams: response.ActionParams,
|
||||
}
|
||||
w.history = append(w.history, newHistoryEntry)
|
||||
|
||||
// Parse assertion result from action_params
|
||||
passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain)
|
||||
if err != nil {
|
||||
@@ -194,6 +224,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
|
||||
}, errors.Wrap(err, "parse assertion result failed")
|
||||
}
|
||||
|
||||
// No need to update ActionResult as per user request
|
||||
// ActionResult should always be empty
|
||||
|
||||
log.Info().
|
||||
Bool("passed", passed).
|
||||
Str("thought", assertionThought).
|
||||
@@ -228,13 +261,12 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error {
|
||||
|
||||
// Wings API data structures
|
||||
type WingsActionRequest struct {
|
||||
Historys []interface{} `json:"historys"`
|
||||
Historys []History `json:"historys"`
|
||||
DeviceInfos []WingsDeviceInfo `json:"device_infos"`
|
||||
StepText string `json:"step_text"`
|
||||
BizId string `json:"biz_id"`
|
||||
TextCase string `json:"text_case"`
|
||||
StepType string `json:"step_type"`
|
||||
DeviceID string `json:"device_id"`
|
||||
BizId string `json:"biz_id"`
|
||||
TaskType string `json:"task_type"`
|
||||
Base WingsBase `json:"Base"`
|
||||
}
|
||||
|
||||
@@ -253,10 +285,16 @@ type WingsBase struct {
|
||||
}
|
||||
|
||||
type WingsActionResponse struct {
|
||||
StepType string `json:"step_type"`
|
||||
ActionParams string `json:"action_params"`
|
||||
ThoughtChain WingsThoughtChain `json:"thought_chain"`
|
||||
BaseResp WingsBaseResp `json:"BaseResp"`
|
||||
AgentType string `json:"agent_type"`
|
||||
StepText string `json:"step_text"`
|
||||
StepTextTrans string `json:"step_text_trans"`
|
||||
OriStepIndex int `json:"ori_step_index"`
|
||||
StepType string `json:"step_type"`
|
||||
ActionParams string `json:"action_params"`
|
||||
DeviceId string `json:"device_id"`
|
||||
NextIsFinish bool `json:"next_is_finish"`
|
||||
ThoughtChain WingsThoughtChain `json:"thought_chain"`
|
||||
BaseResp WingsBaseResp `json:"BaseResp"`
|
||||
}
|
||||
|
||||
type WingsThoughtChain struct {
|
||||
@@ -276,6 +314,19 @@ type WingsExtra struct {
|
||||
LogID string `json:"_log_id"`
|
||||
}
|
||||
|
||||
// History structure for request and response
|
||||
type History struct {
|
||||
ThoughtChain WingsThoughtChain `json:"thought_chain"` // 思考结果
|
||||
StepText string `json:"step_text"` // 操作的指令
|
||||
DeviceID string `json:"device_id"` // 操作的设备id
|
||||
AgentType string `json:"agent_type"` // 最终决策的agent类型
|
||||
ActionResult string `json:"action_result"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果
|
||||
DeviceInfos *[]WingsDeviceInfo `json:"device_infos"` // 所有设备的信息
|
||||
ActionParams string `json:"action_params"` // 历史操作解析结果(断言,自动化,物料构造)
|
||||
StepTextTrans string `json:"step_text_trans"` // 归一化的步骤文本(为后续的实际执行解析文本)
|
||||
OriStepIndex int `json:"ori_step_index"` // 原本的执行序列(扩展前、目标导向原始文本步骤)
|
||||
}
|
||||
|
||||
// Action parameter structures
|
||||
type WingsActionParams struct {
|
||||
Type string `json:"Type"`
|
||||
@@ -315,6 +366,11 @@ type WingsTextParams struct {
|
||||
|
||||
// Helper methods
|
||||
|
||||
// resetHistory resets the conversation history
|
||||
func (w *WingsService) resetHistory() {
|
||||
w.history = []History{}
|
||||
}
|
||||
|
||||
// generateWingsUUID generates a random UUID for LogID
|
||||
func generateWingsUUID() string {
|
||||
return uuid.New().String()
|
||||
@@ -328,16 +384,7 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st
|
||||
|
||||
for _, content := range message.MultiContent {
|
||||
if content.Type == schema.ChatMessagePartTypeImageURL && content.ImageURL != nil {
|
||||
// Extract base64 data from data URL
|
||||
screenshot := content.ImageURL.URL
|
||||
if strings.HasPrefix(screenshot, "data:image/") {
|
||||
// Remove data URL prefix
|
||||
parts := strings.Split(screenshot, ",")
|
||||
if len(parts) == 2 {
|
||||
return parts[1], nil
|
||||
}
|
||||
}
|
||||
return screenshot, nil
|
||||
return content.ImageURL.URL, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -345,19 +392,29 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st
|
||||
}
|
||||
|
||||
// getDeviceInfoFromContext gets device info from context with fallback
|
||||
func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo {
|
||||
// use default device info
|
||||
return WingsDeviceInfo{
|
||||
DeviceID: "default-device",
|
||||
NowImage: screenshot,
|
||||
PreImage: screenshot,
|
||||
NowLayoutJSON: "",
|
||||
OperationSystem: "android",
|
||||
func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo {
|
||||
// TODO: Extract device info from context if available
|
||||
|
||||
// Use last history's NowImage as PreImage if history exists
|
||||
preImageUrl := screenshot
|
||||
if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
|
||||
preImageUrl = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImageUrl
|
||||
}
|
||||
|
||||
// use default device info with optimized PreImage
|
||||
return []WingsDeviceInfo{
|
||||
{
|
||||
DeviceID: "default-device",
|
||||
NowImageUrl: screenshot,
|
||||
PreImageUrl: preImageUrl,
|
||||
NowLayoutJSON: "",
|
||||
OperationSystem: "android",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// getDeviceInfoFromScreenshot gets device info from screenshot (for Assert)
|
||||
func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo {
|
||||
func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo {
|
||||
return w.getDeviceInfoFromContext(ctx, screenshot)
|
||||
}
|
||||
|
||||
@@ -390,6 +447,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
|
||||
// Set headers
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Accept", "application/json")
|
||||
httpReq.Header.Add("x-use-ppe", "1")
|
||||
httpReq.Header.Add("x-tt-env", "ppe_refactor_merge")
|
||||
|
||||
// Add authentication headers if using external API
|
||||
if w.accessKey != "" && w.secretKey != "" {
|
||||
@@ -403,7 +462,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
|
||||
|
||||
// Execute HTTP request
|
||||
client := &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
Timeout: 120 * time.Second,
|
||||
}
|
||||
|
||||
resp, err := client.Do(httpReq)
|
||||
@@ -414,6 +473,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
|
||||
|
||||
logID := resp.Header.Get("X-Tt-Logid")
|
||||
log.Info().Str("step_text", request.StepText).
|
||||
Str("image_url", request.DeviceInfos[0].NowImageUrl).
|
||||
Str("log_id", logID).Str("biz_id", request.BizId).
|
||||
Str("url", w.apiURL).Msg("call wings api")
|
||||
|
||||
@@ -439,7 +499,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
|
||||
|
||||
// convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach
|
||||
func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) {
|
||||
if actionParamsStr == "" {
|
||||
if actionParamsStr == "" || actionParamsStr == "FINISH" {
|
||||
return []schema.ToolCall{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -240,12 +240,12 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er
|
||||
return err
|
||||
}
|
||||
// 等待安装完成或超时
|
||||
timeout := 3 * time.Minute
|
||||
timeout := 8 * time.Minute
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(timeout):
|
||||
return fmt.Errorf("installation timed out after %v", timeout)
|
||||
return fmt.Errorf("install via installer timed out after %v", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -251,6 +251,7 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
|
||||
screenResult, err := dExt.GetScreenResult(
|
||||
option.WithScreenShotFileName("ai_assert"),
|
||||
option.WithScreenShotBase64(true),
|
||||
option.WithScreenShotUpload(true),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -267,7 +268,7 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
|
||||
modelCallStartTime := time.Now()
|
||||
assertOpts := &ai.AssertOptions{
|
||||
Assertion: assertion,
|
||||
Screenshot: screenResult.Base64,
|
||||
Screenshot: screenResult.UploadedURL,
|
||||
Size: screenResult.Resolution,
|
||||
}
|
||||
result, err := dExt.LLMService.Assert(context.Background(), assertOpts)
|
||||
@@ -302,6 +303,7 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ..
|
||||
screenResult, err := dExt.GetScreenResult(
|
||||
option.WithScreenShotFileName("ai_planning"),
|
||||
option.WithScreenShotBase64(true),
|
||||
option.WithScreenShotUpload(true),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -321,7 +323,7 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ..
|
||||
{
|
||||
Type: schema.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &schema.ChatMessageImageURL{
|
||||
URL: screenResult.Base64,
|
||||
URL: screenResult.UploadedURL,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -134,6 +134,19 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult
|
||||
}
|
||||
}
|
||||
|
||||
if screenshotOptions.ScreenShotWithUpload {
|
||||
// Upload the screenshot to the server
|
||||
if screenResult.ImagePath != "" && screenResult.bufSource != nil {
|
||||
url, err := uploadScreenshot(screenResult.ImagePath, screenResult.bufSource)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("imagePath", screenResult.ImagePath).Msg("failed to upload screenshot")
|
||||
} else if url != "" {
|
||||
screenResult.UploadedURL = url
|
||||
log.Info().Str("uploadedUrl", url).Msg("screenshot uploaded successfully")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// save screen result to session
|
||||
session := dExt.GetSession()
|
||||
session.screenResults = append(session.screenResults, screenResult)
|
||||
|
||||
@@ -41,7 +41,7 @@ type DriverRequests struct {
|
||||
}
|
||||
|
||||
func NewDriverSession() *DriverSession {
|
||||
timeout := 30 * time.Second
|
||||
timeout := 120 * time.Second
|
||||
session := &DriverSession{
|
||||
ctx: context.Background(),
|
||||
ID: "<SessionNotInit>",
|
||||
@@ -202,8 +202,13 @@ func (s *DriverSession) RequestWithRetry(method string, urlStr string, rawBody [
|
||||
synthesizeEventRetryAdded = true
|
||||
}
|
||||
|
||||
// Notice: use DeviceHTTPDriverError when request driver failed
|
||||
lastError = errors.Wrap(code.DeviceHTTPDriverError, err.Error())
|
||||
// Check if it's already a DeviceOfflineError
|
||||
if errors.Is(err, code.DeviceOfflineError) {
|
||||
lastError = err
|
||||
} else {
|
||||
// Use DeviceHTTPDriverError for other errors
|
||||
lastError = errors.Wrap(code.DeviceHTTPDriverError, err.Error())
|
||||
}
|
||||
|
||||
// If this was the last attempt, break
|
||||
if attempt == maxRetry {
|
||||
@@ -296,6 +301,11 @@ func (s *DriverSession) Request(method string, urlStr string, rawBody []byte, op
|
||||
driverResult.RequestTime = time.Now()
|
||||
var resp *http.Response
|
||||
if resp, err = s.client.Do(req); err != nil {
|
||||
// Check for connection reset or EOF errors and classify as DeviceOfflineError
|
||||
if strings.Contains(err.Error(), "read: connection reset by peer") ||
|
||||
strings.Contains(err.Error(), "EOF") {
|
||||
return nil, errors.Wrap(code.DeviceOfflineError, err.Error())
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package uixt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -18,6 +21,7 @@ import (
|
||||
"github.com/httprunner/httprunner/v5/code"
|
||||
"github.com/httprunner/httprunner/v5/internal/builtin"
|
||||
"github.com/httprunner/httprunner/v5/internal/config"
|
||||
"github.com/httprunner/httprunner/v5/internal/json"
|
||||
"github.com/httprunner/httprunner/v5/uixt/option"
|
||||
)
|
||||
|
||||
@@ -350,34 +354,110 @@ func DownloadFileByUrl(fileUrl string) (filePath string, err error) {
|
||||
// Build the HTTP GET request.
|
||||
req, err := http.NewRequest("GET", fileUrl, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.Wrap(code.NetworkError, err.Error())
|
||||
}
|
||||
|
||||
// Perform the request.
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.Wrap(code.NetworkError, err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check the HTTP status code.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("failed to download file: %s", resp.Status)
|
||||
return "", errors.Wrap(code.NetworkError, fmt.Errorf("failed to download file: %s", resp.Status).Error())
|
||||
}
|
||||
|
||||
// Create the output file.
|
||||
outFile, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.Wrap(code.MobileUIDriverError, err.Error())
|
||||
}
|
||||
defer outFile.Close()
|
||||
|
||||
// Copy the response body to the file.
|
||||
_, err = io.Copy(outFile, resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", errors.Wrap(code.NetworkError, err.Error())
|
||||
}
|
||||
|
||||
log.Info().Str("filePath", filePath).Msg("download file success")
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
// uploadScreenshot uploads a screenshot to the server and returns the URL
|
||||
func uploadScreenshot(imagePath string, imageBuffer *bytes.Buffer) (string, error) {
|
||||
// Create a new buffer for the multipart form
|
||||
var requestBody bytes.Buffer
|
||||
writer := multipart.NewWriter(&requestBody)
|
||||
|
||||
// Create a form file field
|
||||
fileField, err := writer.CreateFormFile("file", filepath.Base(imagePath))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create form file")
|
||||
}
|
||||
|
||||
// Copy the image buffer to the form file field
|
||||
if _, err := io.Copy(fileField, bytes.NewReader(imageBuffer.Bytes())); err != nil {
|
||||
return "", errors.Wrap(err, "failed to copy image data")
|
||||
}
|
||||
|
||||
// Close the multipart writer
|
||||
if err := writer.Close(); err != nil {
|
||||
return "", errors.Wrap(err, "failed to close multipart writer")
|
||||
}
|
||||
|
||||
// Create the HTTP request
|
||||
uploadURL := "https://gtf-eapi-cn.bytedance.com/cn/upload/xxx"
|
||||
req, err := http.NewRequest("POST", uploadURL, &requestBody)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(code.UploadFailed, err.Error())
|
||||
}
|
||||
|
||||
// Set headers
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("accessKey", "ies.vedem.video")
|
||||
req.Header.Set("token", "***REMOVED***")
|
||||
|
||||
// Create HTTP client with HTTP/1.1 support
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
||||
},
|
||||
}
|
||||
|
||||
// Send the request
|
||||
log.Debug().Str("url", uploadURL).Str("imagePath", imagePath).Msg("uploading screenshot")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(code.UploadFailed, err.Error())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read the response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(code.UploadFailed, err.Error())
|
||||
}
|
||||
|
||||
// Parse the response JSON
|
||||
var result struct {
|
||||
StatusCode int `json:"StatusCode"`
|
||||
Data interface{} `json:"Data"`
|
||||
URL string `json:"URL"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
log.Warn().Err(err).Str("response", string(respBody)).Msg("failed to parse upload response")
|
||||
return "", errors.Wrap(code.UploadFailed, "failed to parse response JSON")
|
||||
}
|
||||
|
||||
// Check if the upload was successful
|
||||
if result.StatusCode != 0 {
|
||||
return "", fmt.Errorf("upload failed with status code: %d", result.StatusCode)
|
||||
}
|
||||
|
||||
log.Debug().Str("url", result.URL).Msg("screenshot uploaded successfully")
|
||||
return result.URL, nil
|
||||
}
|
||||
|
||||
BIN
uixt/evalite
BIN
uixt/evalite
Binary file not shown.
@@ -437,6 +437,13 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) {
|
||||
// 超时两分钟
|
||||
_, err = wd.Session.POST(data, "/wings/apps/launch", option.WithTimeout(120))
|
||||
if err != nil {
|
||||
// Check for untrusted certificate error
|
||||
errMsg := err.Error()
|
||||
if strings.Contains(errMsg, "has not been explicitly trusted by the user") ||
|
||||
strings.Contains(errMsg, "invalid code signature") ||
|
||||
strings.Contains(errMsg, "inadequate entitlements") {
|
||||
return errors.Wrap(code.DeviceUntrustedCertError, "App certificate not trusted: "+bundleId)
|
||||
}
|
||||
return errors.Wrap(err, "wda app launch failed")
|
||||
}
|
||||
return nil
|
||||
@@ -444,10 +451,17 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) {
|
||||
|
||||
func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) {
|
||||
log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunchUnattached")
|
||||
// [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)]
|
||||
// [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)]]
|
||||
data := map[string]interface{}{"bundleId": bundleId}
|
||||
_, err = wd.Session.POST(data, "/wda/apps/launchUnattached")
|
||||
if err != nil {
|
||||
// Check for untrusted certificate error
|
||||
errMsg := err.Error()
|
||||
if strings.Contains(errMsg, "has not been explicitly trusted by the user") ||
|
||||
strings.Contains(errMsg, "invalid code signature") ||
|
||||
strings.Contains(errMsg, "inadequate entitlements") {
|
||||
return errors.Wrap(code.DeviceUntrustedCertError, "App certificate not trusted: "+bundleId)
|
||||
}
|
||||
return errors.Wrap(err, "wda app launchUnattached failed")
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user