diff --git a/code/code.go b/code/code.go index 54da9ae0..d5636f72 100644 --- a/code/code.go +++ b/code/code.go @@ -66,18 +66,19 @@ var ( // device related: [50, 70) var ( - DeviceConnectionError = errors.New("device general connection error") // 50 - DeviceHTTPDriverError = errors.New("device HTTP driver error") // 51 - DeviceUSBDriverError = errors.New("device USB driver error") // 52 - DeviceAppNotInstalled = errors.New("device app not installed") // 59 - DeviceGetInfoError = errors.New("device get info error") // 60 - DeviceConfigureError = errors.New("device configure error") // 61 - DeviceShellExecError = errors.New("device shell exec error") // 62 - DeviceOfflineError = errors.New("device offline") // 63 - DeviceInstallFailed = errors.New("device install app failed") // 64 - DeviceScreenShotError = errors.New("device screenshot error") // 65 - DeviceCaptureLogError = errors.New("device capture log error") // 66 - DeviceUIResponseSlow = errors.New("device UI response slow") // 67 + DeviceConnectionError = errors.New("device general connection error") // 50 + DeviceHTTPDriverError = errors.New("device HTTP driver error") // 51 + DeviceUSBDriverError = errors.New("device USB driver error") // 52 + DeviceUntrustedCertError = errors.New("device app certificate not trusted") // 53 + DeviceAppNotInstalled = errors.New("device app not installed") // 59 + DeviceGetInfoError = errors.New("device get info error") // 60 + DeviceConfigureError = errors.New("device configure error") // 61 + DeviceShellExecError = errors.New("device shell exec error") // 62 + DeviceOfflineError = errors.New("device offline") // 63 + DeviceInstallFailed = errors.New("device install app failed") // 64 + DeviceScreenShotError = errors.New("device screenshot error") // 65 + DeviceCaptureLogError = errors.New("device capture log error") // 66 + DeviceUIResponseSlow = errors.New("device UI response slow") // 67 ) // UI automation related: [70, 80) @@ -176,18 +177,19 @@ var errorsMap = map[error]int{ UploadFailed: 49, // device related - DeviceConnectionError: 50, - DeviceHTTPDriverError: 51, - DeviceUSBDriverError: 52, - DeviceAppNotInstalled: 59, - DeviceGetInfoError: 60, - DeviceConfigureError: 61, - DeviceShellExecError: 62, - DeviceOfflineError: 63, - DeviceInstallFailed: 64, - DeviceScreenShotError: 65, - DeviceCaptureLogError: 66, - DeviceUIResponseSlow: 67, + DeviceConnectionError: 50, + DeviceHTTPDriverError: 51, + DeviceUSBDriverError: 52, + DeviceUntrustedCertError: 53, + DeviceAppNotInstalled: 59, + DeviceGetInfoError: 60, + DeviceConfigureError: 61, + DeviceShellExecError: 62, + DeviceOfflineError: 63, + DeviceInstallFailed: 64, + DeviceScreenShotError: 65, + DeviceCaptureLogError: 66, + DeviceUIResponseSlow: 67, // UI automation related MobileUIDriverAppNotInstalled: 68, diff --git a/examples/game/llk/main.go b/examples/game/llk/main.go index c8e5f201..ccf28ae5 100644 --- a/examples/game/llk/main.go +++ b/examples/game/llk/main.go @@ -7,13 +7,14 @@ import ( "fmt" "path/filepath" + "github.com/rs/zerolog/log" + hrp "github.com/httprunner/httprunner/v5" "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/config" "github.com/httprunner/httprunner/v5/uixt" "github.com/httprunner/httprunner/v5/uixt/option" - "github.com/rs/zerolog/log" ) // GameElement represents a game element detected in the interface diff --git a/internal/version/VERSION b/internal/version/VERSION index 3c60993c..a1341323 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -v5.0.0-250813 +v5.0.0-250815 diff --git a/pkg/gadb/device.go b/pkg/gadb/device.go index caf7e448..c6857fee 100644 --- a/pkg/gadb/device.go +++ b/pkg/gadb/device.go @@ -664,14 +664,22 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt tp transport filesize int64 ) + timeout := 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Minute) + defer cancel() + filesize, err = apk.Seek(0, io.SeekEnd) if err != nil { return nil, err } - if tp, err = d.createDeviceTransport(5 * time.Minute); err != nil { + if tp, err = d.createDeviceTransport(4 * time.Minute); err != nil { return nil, err } defer func() { _ = tp.Close() }() + go func() { + <-ctx.Done() + _ = tp.Close() + }() cmd := "abb_exec:package\x00install\x00-t" for _, arg := range args { cmd += "\x00" + arg @@ -690,6 +698,9 @@ func (d *Device) installViaABBExec(apk io.ReadSeeker, args ...string) (raw []byt return nil, err } raw, err = tp.ReadBytesAll() + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("installation timed out after %d minutes", timeout) + } return } diff --git a/uixt/ai/ai_test.go b/uixt/ai/ai_test.go index 0a387784..75aa5af8 100644 --- a/uixt/ai/ai_test.go +++ b/uixt/ai/ai_test.go @@ -6,14 +6,14 @@ import ( "context" "testing" - "github.com/httprunner/httprunner/v5/internal/builtin" - "github.com/httprunner/httprunner/v5/uixt/option" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/httprunner/httprunner/v5/internal/builtin" + "github.com/httprunner/httprunner/v5/uixt/option" ) func TestILLMServiceQuery(t *testing.T) { - // Create LLM service service, err := NewLLMService(option.OPENAI_GPT_4O) require.NoError(t, err) @@ -80,7 +80,6 @@ func TestILLMServiceQuery(t *testing.T) { } func TestILLMServiceIntegration(t *testing.T) { - // Create LLM service service, err := NewLLMService(option.OPENAI_GPT_4O) require.NoError(t, err) diff --git a/uixt/ai/wings_service.go b/uixt/ai/wings_service.go index 7e4f29f5..4424f98a 100644 --- a/uixt/ai/wings_service.go +++ b/uixt/ai/wings_service.go @@ -26,6 +26,7 @@ type WingsService struct { bizId string accessKey string secretKey string + history []History // Conversation history for Wings API } // NewWingsService creates a new Wings service instance @@ -49,6 +50,7 @@ func NewWingsService() (ILLMService, error) { bizId: bizID, accessKey: accessKey, secretKey: secretKey, + history: []History{}, }, nil } @@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni return nil, errors.Wrap(err, "validate planning parameters failed") } + // Reset history if requested + if opts.ResetHistory { + w.resetHistory() + } + // Extract screenshot from message screenshot, err := w.extractScreenshotFromMessage(opts.Message) if err != nil { @@ -70,15 +77,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni // Prepare Wings API request apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.UserInstruction, - BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", - StepType: "automation", - DeviceID: deviceInfo.DeviceID, + Historys: w.history, + DeviceInfos: deviceInfo, + StepText: fmt.Sprintf("%s", opts.UserInstruction), + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s\n停止操作。\n注意事项：\n", opts.UserInstruction), Base: WingsBase{ LogID: generateWingsUUID(), }, @@ -98,7 +101,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &PlanningResult{ Thought: response.ThoughtChain.Thought, @@ -107,26 +110,48 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni }, err } - // Convert Wings API response to tool calls - toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams) - if err != nil { - return &PlanningResult{ - Thought: response.ThoughtChain.Thought, - Error: err.Error(), - ModelName: "wings-api", - }, errors.Wrap(err, "convert Wings response to tool calls failed") + // Update history with response data + newHistoryEntry := History{ + ThoughtChain: response.ThoughtChain, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: response.OriStepIndex, + DeviceID: deviceInfo[0].DeviceID, + AgentType: response.AgentType, + ActionResult: "", // Always empty as requested + DeviceInfos: &deviceInfo, + ActionParams: response.ActionParams, } + w.history = append(w.history, newHistoryEntry) + var toolCalls []schema.ToolCall + if response.StepType != "FINISH" { + // Convert Wings API response to tool calls + toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams) + if err != nil { + return &PlanningResult{ + Thought: response.ThoughtChain.Thought, + Error: err.Error(), + ModelName: "wings-api", + }, errors.Wrap(err, "convert Wings response to tool calls failed") + } + } + + // No need to update ActionResult as per user request + // ActionResult should always be empty log.Info(). Str("thought", response.ThoughtChain.Thought). + Str("action", response.AgentType). + Str("action_params", response.ActionParams). + Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)). Int("tool_calls_count", len(toolCalls)). Int64("elapsed_ms", elapsed). Msg("Wings API planning completed") return &PlanningResult{ ToolCalls: toolCalls, - Thought: response.ThoughtChain.Thought, - Content: response.ThoughtChain.Summary, + Thought: response.StepTextTrans, + Content: response.StepTextTrans, ModelName: "wings-api", }, nil } @@ -138,28 +163,20 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert return nil, errors.Wrap(err, "validate assertion parameters failed") } - // Clean screenshot data URL prefix - cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot) - // Get device info from context (if available) - deviceInfo := w.getDeviceInfoFromScreenshot(ctx, cleanScreenshot) + deviceInfos := w.getDeviceInfoFromScreenshot(ctx, opts.Screenshot) // Prepare Wings API request for assertion apiRequest := WingsActionRequest{ - Historys: []interface{}{}, // empty as specified - DeviceInfos: []WingsDeviceInfo{ - deviceInfo, - }, - StepText: opts.Assertion, - BizId: w.bizId, - TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n", - StepType: "assert", // Different from automation - DeviceID: deviceInfo.DeviceID, + Historys: []History{}, + DeviceInfos: deviceInfos, + StepText: fmt.Sprintf("断言:%s", opts.Assertion), + BizId: w.bizId, + TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion), Base: WingsBase{ LogID: generateWingsUUID(), }, } - log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request") // Call Wings API startTime := time.Now() @@ -175,7 +192,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert } // Check API response status - if response.BaseResp.StatusCode != 0 { + if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 { err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage) return &AssertionResult{ Pass: false, @@ -184,6 +201,19 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, err } + // Update history with response data + newHistoryEntry := History{ + ThoughtChain: response.ThoughtChain, + StepText: response.StepText, + StepTextTrans: response.StepTextTrans, + OriStepIndex: response.OriStepIndex, + DeviceID: response.DeviceId, + AgentType: response.AgentType, + DeviceInfos: &apiRequest.DeviceInfos, + ActionParams: response.ActionParams, + } + w.history = append(w.history, newHistoryEntry) + // Parse assertion result from action_params passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain) if err != nil { @@ -194,6 +224,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert }, errors.Wrap(err, "parse assertion result failed") } + // No need to update ActionResult as per user request + // ActionResult should always be empty + log.Info(). Bool("passed", passed). Str("thought", assertionThought). @@ -228,13 +261,12 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error { // Wings API data structures type WingsActionRequest struct { - Historys []interface{} `json:"historys"` + Historys []History `json:"historys"` DeviceInfos []WingsDeviceInfo `json:"device_infos"` StepText string `json:"step_text"` - BizId string `json:"biz_id"` TextCase string `json:"text_case"` - StepType string `json:"step_type"` - DeviceID string `json:"device_id"` + BizId string `json:"biz_id"` + TaskType string `json:"task_type"` Base WingsBase `json:"Base"` } @@ -253,10 +285,16 @@ type WingsBase struct { } type WingsActionResponse struct { - StepType string `json:"step_type"` - ActionParams string `json:"action_params"` - ThoughtChain WingsThoughtChain `json:"thought_chain"` - BaseResp WingsBaseResp `json:"BaseResp"` + AgentType string `json:"agent_type"` + StepText string `json:"step_text"` + StepTextTrans string `json:"step_text_trans"` + OriStepIndex int `json:"ori_step_index"` + StepType string `json:"step_type"` + ActionParams string `json:"action_params"` + DeviceId string `json:"device_id"` + NextIsFinish bool `json:"next_is_finish"` + ThoughtChain WingsThoughtChain `json:"thought_chain"` + BaseResp WingsBaseResp `json:"BaseResp"` } type WingsThoughtChain struct { @@ -276,6 +314,19 @@ type WingsExtra struct { LogID string `json:"_log_id"` } +// History structure for request and response +type History struct { + ThoughtChain WingsThoughtChain `json:"thought_chain"` // 思考结果 + StepText string `json:"step_text"` // 操作的指令 + DeviceID string `json:"device_id"` // 操作的设备id + AgentType string `json:"agent_type"` // 最终决策的agent类型 + ActionResult string `json:"action_result"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果 + DeviceInfos *[]WingsDeviceInfo `json:"device_infos"` // 所有设备的信息 + ActionParams string `json:"action_params"` // 历史操作解析结果(断言，自动化，物料构造) + StepTextTrans string `json:"step_text_trans"` // 归一化的步骤文本(为后续的实际执行解析文本) + OriStepIndex int `json:"ori_step_index"` // 原本的执行序列（扩展前、目标导向原始文本步骤） +} + // Action parameter structures type WingsActionParams struct { Type string `json:"Type"` @@ -315,6 +366,11 @@ type WingsTextParams struct { // Helper methods +// resetHistory resets the conversation history +func (w *WingsService) resetHistory() { + w.history = []History{} +} + // generateWingsUUID generates a random UUID for LogID func generateWingsUUID() string { return uuid.New().String() @@ -328,16 +384,7 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st for _, content := range message.MultiContent { if content.Type == schema.ChatMessagePartTypeImageURL && content.ImageURL != nil { - // Extract base64 data from data URL - screenshot := content.ImageURL.URL - if strings.HasPrefix(screenshot, "data:image/") { - // Remove data URL prefix - parts := strings.Split(screenshot, ",") - if len(parts) == 2 { - return parts[1], nil - } - } - return screenshot, nil + return content.ImageURL.URL, nil } } @@ -345,19 +392,29 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st } // getDeviceInfoFromContext gets device info from context with fallback -func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo { - // use default device info - return WingsDeviceInfo{ - DeviceID: "default-device", - NowImage: screenshot, - PreImage: screenshot, - NowLayoutJSON: "", - OperationSystem: "android", +func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo { + // TODO: Extract device info from context if available + + // Use last history's NowImage as PreImage if history exists + preImageUrl := screenshot + if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 { + preImageUrl = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImageUrl + } + + // use default device info with optimized PreImage + return []WingsDeviceInfo{ + { + DeviceID: "default-device", + NowImageUrl: screenshot, + PreImageUrl: preImageUrl, + NowLayoutJSON: "", + OperationSystem: "android", + }, } } // getDeviceInfoFromScreenshot gets device info from screenshot (for Assert) -func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo { +func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo { return w.getDeviceInfoFromContext(ctx, screenshot) } @@ -390,6 +447,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Set headers httpReq.Header.Set("Content-Type", "application/json") httpReq.Header.Set("Accept", "application/json") + httpReq.Header.Add("x-use-ppe", "1") + httpReq.Header.Add("x-tt-env", "ppe_refactor_merge") // Add authentication headers if using external API if w.accessKey != "" && w.secretKey != "" { @@ -403,7 +462,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // Execute HTTP request client := &http.Client{ - Timeout: 60 * time.Second, + Timeout: 120 * time.Second, } resp, err := client.Do(httpReq) @@ -413,7 +472,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ defer resp.Body.Close() logID := resp.Header.Get("X-Tt-Logid") - log.Info().Str("step_text", request.StepText).Str("log_id", logID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") + log.Info().Str("step_text", request.StepText).Str("image_url", request.DeviceInfos[0].NowImageUrl).Str("log_id", logID).Str("biz_id", request.BizId).Str("url", w.apiURL).Msg("call wings api") // Read response body responseBody, err := io.ReadAll(resp.Body) @@ -437,7 +496,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ // convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) { - if actionParamsStr == "" { + if actionParamsStr == "" || actionParamsStr == "FINISH" { return []schema.ToolCall{}, nil } diff --git a/uixt/android_device.go b/uixt/android_device.go index efb243e8..0a47eb95 100644 --- a/uixt/android_device.go +++ b/uixt/android_device.go @@ -240,12 +240,12 @@ func (dev *AndroidDevice) installViaInstaller(apkPath string, args ...string) er return err } // 等待安装完成或超时 - timeout := 3 * time.Minute + timeout := 8 * time.Minute select { case err := <-done: return err case <-time.After(timeout): - return fmt.Errorf("installation timed out after %v", timeout) + return fmt.Errorf("install via installer timed out after %v", timeout) } } diff --git a/uixt/driver_ext_ai.go b/uixt/driver_ext_ai.go index 185f41fc..c849e6a6 100644 --- a/uixt/driver_ext_ai.go +++ b/uixt/driver_ext_ai.go @@ -251,6 +251,7 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* screenResult, err := dExt.GetScreenResult( option.WithScreenShotFileName("ai_assert"), option.WithScreenShotBase64(true), + option.WithScreenShotUpload(true), ) if err != nil { return nil, err @@ -267,7 +268,7 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (* modelCallStartTime := time.Now() assertOpts := &ai.AssertOptions{ Assertion: assertion, - Screenshot: screenResult.Base64, + Screenshot: screenResult.UploadedURL, Size: screenResult.Resolution, } result, err := dExt.LLMService.Assert(context.Background(), assertOpts) @@ -302,6 +303,7 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts .. screenResult, err := dExt.GetScreenResult( option.WithScreenShotFileName("ai_planning"), option.WithScreenShotBase64(true), + option.WithScreenShotUpload(true), ) if err != nil { return nil, err @@ -321,7 +323,7 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts .. { Type: schema.ChatMessagePartTypeImageURL, ImageURL: &schema.ChatMessageImageURL{ - URL: screenResult.Base64, + URL: screenResult.UploadedURL, }, }, }, diff --git a/uixt/driver_ext_screenshot.go b/uixt/driver_ext_screenshot.go index d84c4e1b..b1662d3b 100644 --- a/uixt/driver_ext_screenshot.go +++ b/uixt/driver_ext_screenshot.go @@ -134,6 +134,19 @@ func (dExt *XTDriver) GetScreenResult(opts ...option.ActionOption) (screenResult } } + if screenshotOptions.ScreenShotWithUpload { + // Upload the screenshot to the server + if screenResult.ImagePath != "" && screenResult.bufSource != nil { + url, err := uploadScreenshot(screenResult.ImagePath, screenResult.bufSource) + if err != nil { + log.Warn().Err(err).Str("imagePath", screenResult.ImagePath).Msg("failed to upload screenshot") + } else if url != "" { + screenResult.UploadedURL = url + log.Info().Str("uploadedUrl", url).Msg("screenshot uploaded successfully") + } + } + } + // save screen result to session session := dExt.GetSession() session.screenResults = append(session.screenResults, screenResult) diff --git a/uixt/driver_session.go b/uixt/driver_session.go index 5a837bb1..29fd433b 100644 --- a/uixt/driver_session.go +++ b/uixt/driver_session.go @@ -41,7 +41,7 @@ type DriverRequests struct { } func NewDriverSession() *DriverSession { - timeout := 30 * time.Second + timeout := 120 * time.Second session := &DriverSession{ ctx: context.Background(), ID: "", @@ -202,8 +202,13 @@ func (s *DriverSession) RequestWithRetry(method string, urlStr string, rawBody [ synthesizeEventRetryAdded = true } - // Notice: use DeviceHTTPDriverError when request driver failed - lastError = errors.Wrap(code.DeviceHTTPDriverError, err.Error()) + // Check if it's already a DeviceOfflineError + if errors.Is(err, code.DeviceOfflineError) { + lastError = err + } else { + // Use DeviceHTTPDriverError for other errors + lastError = errors.Wrap(code.DeviceHTTPDriverError, err.Error()) + } // If this was the last attempt, break if attempt == maxRetry { @@ -296,6 +301,11 @@ func (s *DriverSession) Request(method string, urlStr string, rawBody []byte, op driverResult.RequestTime = time.Now() var resp *http.Response if resp, err = s.client.Do(req); err != nil { + // Check for connection reset or EOF errors and classify as DeviceOfflineError + if strings.Contains(err.Error(), "read: connection reset by peer") || + strings.Contains(err.Error(), "EOF") { + return nil, errors.Wrap(code.DeviceOfflineError, err.Error()) + } return nil, err } defer func() { diff --git a/uixt/driver_utils.go b/uixt/driver_utils.go index b643d582..f583c84d 100644 --- a/uixt/driver_utils.go +++ b/uixt/driver_utils.go @@ -1,11 +1,14 @@ package uixt import ( + "bytes" "context" "crypto/md5" + "crypto/tls" "fmt" "io" "math/rand/v2" + "mime/multipart" "net/http" "os" "path/filepath" @@ -18,6 +21,7 @@ import ( "github.com/httprunner/httprunner/v5/code" "github.com/httprunner/httprunner/v5/internal/builtin" "github.com/httprunner/httprunner/v5/internal/config" + "github.com/httprunner/httprunner/v5/internal/json" "github.com/httprunner/httprunner/v5/uixt/option" ) @@ -350,34 +354,110 @@ func DownloadFileByUrl(fileUrl string) (filePath string, err error) { // Build the HTTP GET request. req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { - return "", err + return "", errors.Wrap(code.NetworkError, err.Error()) } // Perform the request. resp, err := client.Do(req) if err != nil { - return "", err + return "", errors.Wrap(code.NetworkError, err.Error()) } defer resp.Body.Close() // Check the HTTP status code. if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download file: %s", resp.Status) + return "", errors.Wrap(code.NetworkError, fmt.Errorf("failed to download file: %s", resp.Status).Error()) } // Create the output file. outFile, err := os.Create(filePath) if err != nil { - return "", err + return "", errors.Wrap(code.MobileUIDriverError, err.Error()) } defer outFile.Close() // Copy the response body to the file. _, err = io.Copy(outFile, resp.Body) if err != nil { - return "", err + return "", errors.Wrap(code.NetworkError, err.Error()) } log.Info().Str("filePath", filePath).Msg("download file success") return filePath, nil } + +// uploadScreenshot uploads a screenshot to the server and returns the URL +func uploadScreenshot(imagePath string, imageBuffer *bytes.Buffer) (string, error) { + // Create a new buffer for the multipart form + var requestBody bytes.Buffer + writer := multipart.NewWriter(&requestBody) + + // Create a form file field + fileField, err := writer.CreateFormFile("file", filepath.Base(imagePath)) + if err != nil { + return "", errors.Wrap(err, "failed to create form file") + } + + // Copy the image buffer to the form file field + if _, err := io.Copy(fileField, bytes.NewReader(imageBuffer.Bytes())); err != nil { + return "", errors.Wrap(err, "failed to copy image data") + } + + // Close the multipart writer + if err := writer.Close(); err != nil { + return "", errors.Wrap(err, "failed to close multipart writer") + } + + // Create the HTTP request + uploadURL := "https://gtf-eapi-cn.bytedance.com/cn/upload/xxx" + req, err := http.NewRequest("POST", uploadURL, &requestBody) + if err != nil { + return "", errors.Wrap(code.UploadFailed, err.Error()) + } + + // Set headers + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("accessKey", "ies.vedem.video") + req.Header.Set("token", "***REMOVED***") + + // Create HTTP client with HTTP/1.1 support + client := &http.Client{ + Transport: &http.Transport{ + TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), + }, + } + + // Send the request + log.Debug().Str("url", uploadURL).Str("imagePath", imagePath).Msg("uploading screenshot") + resp, err := client.Do(req) + if err != nil { + return "", errors.Wrap(code.UploadFailed, err.Error()) + } + defer resp.Body.Close() + + // Read the response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(code.UploadFailed, err.Error()) + } + + // Parse the response JSON + var result struct { + StatusCode int `json:"StatusCode"` + Data interface{} `json:"Data"` + URL string `json:"URL"` + } + + if err := json.Unmarshal(respBody, &result); err != nil { + log.Warn().Err(err).Str("response", string(respBody)).Msg("failed to parse upload response") + return "", errors.Wrap(code.UploadFailed, "failed to parse response JSON") + } + + // Check if the upload was successful + if result.StatusCode != 0 { + return "", fmt.Errorf("upload failed with status code: %d", result.StatusCode) + } + + log.Debug().Str("url", result.URL).Msg("screenshot uploaded successfully") + return result.URL, nil +} diff --git a/uixt/evalite b/uixt/evalite index ec074fa3..8437a1eb 100644 Binary files a/uixt/evalite and b/uixt/evalite differ diff --git a/uixt/ios_driver_wda.go b/uixt/ios_driver_wda.go index d62a069c..b86c2fc3 100644 --- a/uixt/ios_driver_wda.go +++ b/uixt/ios_driver_wda.go @@ -437,6 +437,13 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) { // 超时两分钟 _, err = wd.Session.POST(data, "/wings/apps/launch", option.WithTimeout(120)) if err != nil { + // Check for untrusted certificate error + errMsg := err.Error() + if strings.Contains(errMsg, "has not been explicitly trusted by the user") || + strings.Contains(errMsg, "invalid code signature") || + strings.Contains(errMsg, "inadequate entitlements") { + return errors.Wrap(code.DeviceUntrustedCertError, "App certificate not trusted: "+bundleId) + } return errors.Wrap(err, "wda app launch failed") } return nil @@ -444,10 +451,17 @@ func (wd *WDADriver) AppLaunch(bundleId string) (err error) { func (wd *WDADriver) AppLaunchUnattached(bundleId string) (err error) { log.Info().Str("bundleId", bundleId).Msg("WDADriver.AppLaunchUnattached") - // [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)] + // [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)]] data := map[string]interface{}{"bundleId": bundleId} _, err = wd.Session.POST(data, "/wda/apps/launchUnattached") if err != nil { + // Check for untrusted certificate error + errMsg := err.Error() + if strings.Contains(errMsg, "has not been explicitly trusted by the user") || + strings.Contains(errMsg, "invalid code signature") || + strings.Contains(errMsg, "inadequate entitlements") { + return errors.Wrap(code.DeviceUntrustedCertError, "App certificate not trusted: "+bundleId) + } return errors.Wrap(err, "wda app launchUnattached failed") } return nil