GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-07-12 16:01:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge master

Browse Source
This commit is contained in:
lilong.129
2025-08-15 14:56:41 +08:00
parent 158d6d9bdb ea36c92899
commit 917e46a2e3
12 changed files with 299 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
uixt/ai/ai_test.go
View File

@@ -6,14 +6,14 @@ import (
"context"
"testing"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/httprunner/httprunner/v5/internal/builtin"
"github.com/httprunner/httprunner/v5/uixt/option"
)
func TestILLMServiceQuery(t *testing.T) {
// Create LLM service
service, err := NewLLMService(option.OPENAI_GPT_4O)
require.NoError(t, err)
@@ -80,7 +80,6 @@ func TestILLMServiceQuery(t *testing.T) {
}
func TestILLMServiceIntegration(t *testing.T) {
// Create LLM service
service, err := NewLLMService(option.OPENAI_GPT_4O)
require.NoError(t, err)

188
uixt/ai/wings_service.go
View File

@@ -26,6 +26,7 @@ type WingsService struct {
bizId string
accessKey string
secretKey string
history []History // Conversation history for Wings API
}
// NewWingsService creates a new Wings service instance
@@ -49,6 +50,7 @@ func NewWingsService() (ILLMService, error) {
bizId: bizID,
accessKey: accessKey,
secretKey: secretKey,
history: []History{},
}, nil
}
@@ -59,6 +61,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
return nil, errors.Wrap(err, "validate planning parameters failed")
}
// Reset history if requested
if opts.ResetHistory {
w.resetHistory()
}
// Extract screenshot from message
screenshot, err := w.extractScreenshotFromMessage(opts.Message)
if err != nil {
@@ -70,15 +77,11 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
// Prepare Wings API request
apiRequest := WingsActionRequest{
Historys: []interface{}{}, // empty as specified
DeviceInfos: []WingsDeviceInfo{
deviceInfo,
},
StepText: opts.UserInstruction,
BizId: w.bizId,
TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n",
StepType: "automation",
DeviceID: deviceInfo.DeviceID,
Historys: w.history,
DeviceInfos: deviceInfo,
StepText: fmt.Sprintf("%s", opts.UserInstruction),
BizId: w.bizId,
TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n%s\n停止操作。\n注意事项：\n", opts.UserInstruction),
Base: WingsBase{
LogID: generateWingsUUID(),
},
@@ -98,7 +101,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
}
// Check API response status
if response.BaseResp.StatusCode != 0 {
if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 {
err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage)
return &PlanningResult{
Thought: response.ThoughtChain.Thought,
@@ -107,26 +110,48 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
}, err
}
// Convert Wings API response to tool calls
toolCalls, err := w.convertWingsResponseToToolCalls(response.ActionParams)
if err != nil {
return &PlanningResult{
Thought: response.ThoughtChain.Thought,
Error: err.Error(),
ModelName: "wings-api",
}, errors.Wrap(err, "convert Wings response to tool calls failed")
// Update history with response data
newHistoryEntry := History{
ThoughtChain: response.ThoughtChain,
StepText: response.StepText,
StepTextTrans: response.StepTextTrans,
OriStepIndex: response.OriStepIndex,
DeviceID: deviceInfo[0].DeviceID,
AgentType: response.AgentType,
ActionResult: "", // Always empty as requested
DeviceInfos: &deviceInfo,
ActionParams: response.ActionParams,
}
w.history = append(w.history, newHistoryEntry)
var toolCalls []schema.ToolCall
if response.StepType != "FINISH" {
// Convert Wings API response to tool calls
toolCalls, err = w.convertWingsResponseToToolCalls(response.ActionParams)
if err != nil {
return &PlanningResult{
Thought: response.ThoughtChain.Thought,
Error: err.Error(),
ModelName: "wings-api",
}, errors.Wrap(err, "convert Wings response to tool calls failed")
}
}
// No need to update ActionResult as per user request
// ActionResult should always be empty
log.Info().
Str("thought", response.ThoughtChain.Thought).
Str("action", response.AgentType).
Str("action_params", response.ActionParams).
Str("log_id", fmt.Sprintf("%v", response.BaseResp.Extra)).
Int("tool_calls_count", len(toolCalls)).
Int64("elapsed_ms", elapsed).
Msg("Wings API planning completed")
return &PlanningResult{
ToolCalls: toolCalls,
Thought: response.ThoughtChain.Thought,
Content: response.ThoughtChain.Summary,
Thought: response.StepTextTrans,
Content: response.StepTextTrans,
ModelName: "wings-api",
}, nil
}
@@ -138,28 +163,20 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
return nil, errors.Wrap(err, "validate assertion parameters failed")
}
// Clean screenshot data URL prefix
cleanScreenshot := w.cleanScreenshotDataURL(opts.Screenshot)
// Get device info from context (if available)
deviceInfo := w.getDeviceInfoFromScreenshot(ctx, cleanScreenshot)
deviceInfos := w.getDeviceInfoFromScreenshot(ctx, opts.Screenshot)
// Prepare Wings API request for assertion
apiRequest := WingsActionRequest{
Historys: []interface{}{}, // empty as specified
DeviceInfos: []WingsDeviceInfo{
deviceInfo,
},
StepText: opts.Assertion,
BizId: w.bizId,
TextCase: "整体描述：\\n前置条件：\\n获取 1 台设备 A。\\n获取 1 个[万粉创作者]账号a。\\n获取 2 个[普通]账号 b、c。\\n账号 a 和账号 b 互相关注。\\n账号 a 和账号 c 互相关注。\\n账号 a 给账号 b 设置备注为 “11131b”。\\n账号 a 给账号 c 设置备注为 “11131c”。\\n账号 a 创建一个粉丝群 m。\\n 账号 a 修改粉丝群 m 名称为“11131群”。\\n 账号 a 邀请账号 b 加入粉丝群 m。\\n账号 a 邀请账号 c 加入粉丝群 m。\\n账号 a 给群聊 m 发送一条文字消息。\\n设备 A 打开抖音 app。\\n设备 A 登录账号 a。\\n设备 A 退出抖音 app。\\n操作步骤：\\n账号a打开抖音app。\\n点击“消息”。\\n点击“11131群”cell。\\n点击“聊天信息页入口”按钮。\\n点击“分享公开群”按钮。\\n点击文字“群口令”。\\n断言：屏幕中存在文字“口令复制成功”。\\n停止操作。\\n注意事项：\\n",
StepType: "assert", // Different from automation
DeviceID: deviceInfo.DeviceID,
Historys: []History{},
DeviceInfos: deviceInfos,
StepText: fmt.Sprintf("断言:%s", opts.Assertion),
BizId: w.bizId,
TextCase: fmt.Sprintf("整体描述：\n前置条件：\n操作步骤：\n断言: %s\n停止操作。\n注意事项：\n", opts.Assertion),
Base: WingsBase{
LogID: generateWingsUUID(),
},
}
log.Info().Interface("apiRequest", apiRequest).Msg("Wings API request")
// Call Wings API
startTime := time.Now()
@@ -175,7 +192,7 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
}
// Check API response status
if response.BaseResp.StatusCode != 0 {
if response.BaseResp.StatusCode != 0 && response.BaseResp.StatusCode != 200 {
err = fmt.Errorf("API returned error: %s", response.BaseResp.StatusMessage)
return &AssertionResult{
Pass: false,
@@ -184,6 +201,19 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
}, err
}
// Update history with response data
newHistoryEntry := History{
ThoughtChain: response.ThoughtChain,
StepText: response.StepText,
StepTextTrans: response.StepTextTrans,
OriStepIndex: response.OriStepIndex,
DeviceID: response.DeviceId,
AgentType: response.AgentType,
DeviceInfos: &apiRequest.DeviceInfos,
ActionParams: response.ActionParams,
}
w.history = append(w.history, newHistoryEntry)
// Parse assertion result from action_params
passed, assertionThought, err := w.parseAssertionResult(response.ActionParams, response.ThoughtChain)
if err != nil {
@@ -194,6 +224,9 @@ func (w *WingsService) Assert(ctx context.Context, opts *AssertOptions) (*Assert
}, errors.Wrap(err, "parse assertion result failed")
}
// No need to update ActionResult as per user request
// ActionResult should always be empty
log.Info().
Bool("passed", passed).
Str("thought", assertionThought).
@@ -228,13 +261,12 @@ func (w *WingsService) RegisterTools(tools []*schema.ToolInfo) error {
// Wings API data structures
type WingsActionRequest struct {
Historys []interface{} `json:"historys"`
Historys []History `json:"historys"`
DeviceInfos []WingsDeviceInfo `json:"device_infos"`
StepText string `json:"step_text"`
BizId string `json:"biz_id"`
TextCase string `json:"text_case"`
StepType string `json:"step_type"`
DeviceID string `json:"device_id"`
BizId string `json:"biz_id"`
TaskType string `json:"task_type"`
Base WingsBase `json:"Base"`
}
@@ -253,10 +285,16 @@ type WingsBase struct {
}
type WingsActionResponse struct {
StepType string `json:"step_type"`
ActionParams string `json:"action_params"`
ThoughtChain WingsThoughtChain `json:"thought_chain"`
BaseResp WingsBaseResp `json:"BaseResp"`
AgentType string `json:"agent_type"`
StepText string `json:"step_text"`
StepTextTrans string `json:"step_text_trans"`
OriStepIndex int `json:"ori_step_index"`
StepType string `json:"step_type"`
ActionParams string `json:"action_params"`
DeviceId string `json:"device_id"`
NextIsFinish bool `json:"next_is_finish"`
ThoughtChain WingsThoughtChain `json:"thought_chain"`
BaseResp WingsBaseResp `json:"BaseResp"`
}
type WingsThoughtChain struct {
@@ -276,6 +314,19 @@ type WingsExtra struct {
LogID string `json:"_log_id"`
}
// History structure for request and response
type History struct {
ThoughtChain WingsThoughtChain `json:"thought_chain"` // 思考结果
StepText string `json:"step_text"` // 操作的指令
DeviceID string `json:"device_id"` // 操作的设备id
AgentType string `json:"agent_type"` // 最终决策的agent类型
ActionResult string `json:"action_result"` // 操作结果, 断言=断言结果, 自动化=自动化操作是否成功, 物料构造=物料构造结果
DeviceInfos *[]WingsDeviceInfo `json:"device_infos"` // 所有设备的信息
ActionParams string `json:"action_params"` // 历史操作解析结果(断言，自动化，物料构造)
StepTextTrans string `json:"step_text_trans"` // 归一化的步骤文本(为后续的实际执行解析文本)
OriStepIndex int `json:"ori_step_index"` // 原本的执行序列（扩展前、目标导向原始文本步骤）
}
// Action parameter structures
type WingsActionParams struct {
Type string `json:"Type"`
@@ -315,6 +366,11 @@ type WingsTextParams struct {
// Helper methods
// resetHistory resets the conversation history
func (w *WingsService) resetHistory() {
w.history = []History{}
}
// generateWingsUUID generates a random UUID for LogID
func generateWingsUUID() string {
return uuid.New().String()
@@ -328,16 +384,7 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st
for _, content := range message.MultiContent {
if content.Type == schema.ChatMessagePartTypeImageURL && content.ImageURL != nil {
// Extract base64 data from data URL
screenshot := content.ImageURL.URL
if strings.HasPrefix(screenshot, "data:image/") {
// Remove data URL prefix
parts := strings.Split(screenshot, ",")
if len(parts) == 2 {
return parts[1], nil
}
}
return screenshot, nil
return content.ImageURL.URL, nil
}
}
@@ -345,19 +392,29 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st
}
// getDeviceInfoFromContext gets device info from context with fallback
func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) WingsDeviceInfo {
// use default device info
return WingsDeviceInfo{
DeviceID: "default-device",
NowImage: screenshot,
PreImage: screenshot,
NowLayoutJSON: "",
OperationSystem: "android",
func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo {
// TODO: Extract device info from context if available
// Use last history's NowImage as PreImage if history exists
preImageUrl := screenshot
if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
preImageUrl = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImageUrl
}
// use default device info with optimized PreImage
return []WingsDeviceInfo{
{
DeviceID: "default-device",
NowImageUrl: screenshot,
PreImageUrl: preImageUrl,
NowLayoutJSON: "",
OperationSystem: "android",
},
}
}
// getDeviceInfoFromScreenshot gets device info from screenshot (for Assert)
func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) WingsDeviceInfo {
func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo {
return w.getDeviceInfoFromContext(ctx, screenshot)
}
@@ -390,6 +447,8 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
// Set headers
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Accept", "application/json")
httpReq.Header.Add("x-use-ppe", "1")
httpReq.Header.Add("x-tt-env", "ppe_refactor_merge")
// Add authentication headers if using external API
if w.accessKey != "" && w.secretKey != "" {
@@ -403,7 +462,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
// Execute HTTP request
client := &http.Client{
Timeout: 60 * time.Second,
Timeout: 120 * time.Second,
}
resp, err := client.Do(httpReq)
@@ -414,6 +473,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
logID := resp.Header.Get("X-Tt-Logid")
log.Info().Str("step_text", request.StepText).
Str("image_url", request.DeviceInfos[0].NowImageUrl).
Str("log_id", logID).Str("biz_id", request.BizId).
Str("url", w.apiURL).Msg("call wings api")
@@ -439,7 +499,7 @@ func (w *WingsService) callWingsAPI(ctx context.Context, request WingsActionRequ
// convertWingsResponseToToolCalls converts Wings API response to tool calls using generic approach
func (w *WingsService) convertWingsResponseToToolCalls(actionParamsStr string) ([]schema.ToolCall, error) {
if actionParamsStr == "" {
if actionParamsStr == "" || actionParamsStr == "FINISH" {
return []schema.ToolCall{}, nil
}