refactor: update AI result handling to differentiate content and thought based on result types in report generation

This commit is contained in:
lilong.129
2025-06-24 16:01:50 +08:00
parent 4522bb2dbf
commit d0ceeb6c51
4 changed files with 29 additions and 16 deletions

View File

@@ -1 +1 @@
v5.0.0-beta-2506241536 v5.0.0-beta-2506241601

View File

@@ -2442,9 +2442,19 @@ const htmlTemplate = `<!DOCTYPE html>
{{if $action.AIResult}} {{if $action.AIResult}}
<div class="sub-action-item"> <div class="sub-action-item">
<div class="validator-ai-content"> <div class="validator-ai-content">
<!-- Display AI Thought --> <!-- Display AI Thought from specific result types -->
{{if $action.AIResult.Thought}} {{if eq $action.AIResult.Type "query"}}
<div class="thought">{{$action.AIResult.Thought}}</div> {{if $action.AIResult.QueryResult.Thought}}
<div class="thought">{{$action.AIResult.QueryResult.Thought}}</div>
{{end}}
{{else if eq $action.AIResult.Type "action"}}
{{if $action.AIResult.PlanningResult.Thought}}
<div class="thought">{{$action.AIResult.PlanningResult.Thought}}</div>
{{end}}
{{else if eq $action.AIResult.Type "assert"}}
{{if $action.AIResult.AssertionResult.Thought}}
<div class="thought">{{$action.AIResult.AssertionResult.Thought}}</div>
{{end}}
{{end}} {{end}}
<!-- AI Operation Layout: Screenshot left, Analysis right --> <!-- AI Operation Layout: Screenshot left, Analysis right -->
@@ -2514,8 +2524,19 @@ const htmlTemplate = `<!DOCTYPE html>
{{if $action.AIResult.Resolution}} {{if $action.AIResult.Resolution}}
<div class="model-info">📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}</div> <div class="model-info">📐 Resolution: {{$action.AIResult.Resolution.Width}}x{{$action.AIResult.Resolution.Height}}</div>
{{end}} {{end}}
{{if $action.AIResult.Content}} {{/* Display Content from specific result types */}}
<div class="model-info">💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.Content}}</div> {{if eq $action.AIResult.Type "query"}}
{{if $action.AIResult.QueryResult.Content}}
<div class="model-info">💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.QueryResult.Content}}</div>
{{end}}
{{else if eq $action.AIResult.Type "action"}}
{{if $action.AIResult.PlanningResult.Content}}
<div class="model-info">💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.PlanningResult.Content}}</div>
{{end}}
{{else if eq $action.AIResult.Type "assert"}}
{{if $action.AIResult.AssertionResult.Content}}
<div class="model-info">💬 {{title $action.AIResult.Type}} Result: {{$action.AIResult.AssertionResult.Content}}</div>
{{end}}
{{end}} {{end}}
</div> </div>
</div> </div>

View File

@@ -156,8 +156,6 @@ func (dExt *XTDriver) AIAction(ctx context.Context, prompt string, opts ...optio
ImagePath: screenResult.ImagePath, ImagePath: screenResult.ImagePath,
Resolution: &screenResult.Resolution, Resolution: &screenResult.Resolution,
PlanningResult: &planningResult.PlanningResult, PlanningResult: &planningResult.PlanningResult,
Thought: planningResult.Thought,
Content: planningResult.Content,
} }
if err != nil { if err != nil {
@@ -341,9 +339,7 @@ type AIExecutionResult struct {
AssertionResult *ai.AssertionResult `json:"assertion_result,omitempty"` // for ai_assert operations AssertionResult *ai.AssertionResult `json:"assertion_result,omitempty"` // for ai_assert operations
// Common fields // Common fields
Thought string `json:"thought,omitempty"` // AI reasoning/thought process Error string `json:"error,omitempty"` // error message if operation failed
Content string `json:"content,omitempty"` // operation result content
Error string `json:"error,omitempty"` // error message if operation failed
} }
// SubActionResult represents a sub-action within a start_to_goal action // SubActionResult represents a sub-action within a start_to_goal action
@@ -408,8 +404,6 @@ func (dExt *XTDriver) AIQuery(text string, opts ...option.ActionOption) (*AIExec
ImagePath: screenResult.ImagePath, // screenshot path ImagePath: screenResult.ImagePath, // screenshot path
Resolution: &screenResult.Resolution, // screen resolution Resolution: &screenResult.Resolution, // screen resolution
QueryResult: result, // query-specific result QueryResult: result, // query-specific result
Thought: result.Thought, // AI reasoning
Content: result.Content, // query result content
} }
return aiResult, nil return aiResult, nil
} }
@@ -458,7 +452,6 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
ImagePath: screenResult.ImagePath, ImagePath: screenResult.ImagePath,
Resolution: &screenResult.Resolution, Resolution: &screenResult.Resolution,
AssertionResult: result, AssertionResult: result,
Thought: result.Thought,
} }
if err != nil { if err != nil {
@@ -471,6 +464,5 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
return aiResult, errors.New(result.Thought) return aiResult, errors.New(result.Thought)
} }
aiResult.Content = "Assertion passed"
return aiResult, nil return aiResult, nil
} }

View File

@@ -174,7 +174,7 @@ func (t *ToolAIQuery) Implement() server.ToolHandlerFunc {
message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt) message := fmt.Sprintf("Successfully queried information with prompt: %s", unifiedReq.Prompt)
returnData := ToolAIQuery{ returnData := ToolAIQuery{
Prompt: unifiedReq.Prompt, Prompt: unifiedReq.Prompt,
Result: queryResult.Content, Result: queryResult.QueryResult.Content,
} }
return NewMCPSuccessResponse(message, &returnData), nil return NewMCPSuccessResponse(message, &returnData), nil