feat: 兼容base64

uixt/ai/wings_service.go

@@ -73,7 +73,7 @@ func (w *WingsService) Plan(ctx context.Context, opts *PlanningOptions) (*Planni
 	}
 
 	// Get device info from context (if available)
-	deviceInfo := w.getDeviceInfoFromContext(ctx, screenshot)
+	deviceInfo := w.getDeviceInfoFromScreenshot(ctx, screenshot)
 
 	// Prepare Wings API request
 	apiRequest := WingsActionRequest{
@@ -391,12 +391,17 @@ func (w *WingsService) extractScreenshotFromMessage(message *schema.Message) (st
 	return "", errors.New("no image found in message")
 }
 
-// getDeviceInfoFromContext gets device info from context with fallback
+// getDeviceInfoFromBase gets device info from base64 screenshot
-func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot string) []WingsDeviceInfo {
+func (w *WingsService) getDeviceInfoFromBase64(screenshotBase64 string) []WingsDeviceInfo {
 	// TODO: Extract device info from context if available
 
 	// Use last history's NowImage as PreImage if history exists
-	preImageUrl := screenshot
+	preImage := screenshotBase64
+	if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
+		preImage = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImage
+	}
+
+	preImageUrl := ""
 	if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
 		preImageUrl = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImageUrl
 	}
@@ -405,7 +410,38 @@ func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot st
 	return []WingsDeviceInfo{
 		{
 			DeviceID:        "default-device",
-			NowImageUrl:     screenshot,
+			NowImage:        screenshotBase64,
+			NowImageUrl:     "",
+			PreImage:        preImage,
+			PreImageUrl:     preImageUrl,
+			NowLayoutJSON:   "",
+			OperationSystem: "android",
+		},
+	}
+}
+
+// getDeviceInfoFromUrl gets device info from url screenshot
+func (w *WingsService) getDeviceInfoFromUrl(screenshotUrl string) []WingsDeviceInfo {
+	// TODO: Extract device info from context if available
+
+	// Use last history's NowImage as PreImage if history exists
+	preImage := ""
+	if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
+		preImage = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImage
+	}
+
+	preImageUrl := screenshotUrl
+	if len(w.history) > 0 && w.history[len(w.history)-1].DeviceInfos != nil && len(*w.history[len(w.history)-1].DeviceInfos) > 0 {
+		preImageUrl = (*w.history[len(w.history)-1].DeviceInfos)[0].NowImageUrl
+	}
+
+	// use default device info with optimized PreImage
+	return []WingsDeviceInfo{
+		{
+			DeviceID:        "default-device",
+			NowImage:        "",
+			NowImageUrl:     screenshotUrl,
+			PreImage:        preImage,
 			PreImageUrl:     preImageUrl,
 			NowLayoutJSON:   "",
 			OperationSystem: "android",
@@ -415,7 +451,14 @@ func (w *WingsService) getDeviceInfoFromContext(_ context.Context, screenshot st
 
 // getDeviceInfoFromScreenshot gets device info from screenshot (for Assert)
 func (w *WingsService) getDeviceInfoFromScreenshot(ctx context.Context, screenshot string) []WingsDeviceInfo {
-	return w.getDeviceInfoFromContext(ctx, screenshot)
+	if strings.HasPrefix(screenshot, "data:image/") {
+		// Remove data URL prefix like "data:image/jpeg;base64,"
+		parts := strings.Split(screenshot, ",")
+		if len(parts) == 2 {
+			return w.getDeviceInfoFromBase64(parts[1])
+		}
+	}
+	return w.getDeviceInfoFromUrl(screenshot)
 }
 
 // cleanScreenshotDataURL removes data URL prefix from screenshot string

uixt/driver_ext_ai.go

@@ -247,12 +247,15 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
 		return nil, errors.New("LLM service is not initialized")
 	}
 
+	// Parse action options to get ResetHistory setting
+	options := option.NewActionOptions(opts...)
+	screenOptions := []option.ActionOption{option.WithScreenShotFileName("ai_action"), option.WithScreenShotBase64(true)}
+	if options.ScreenShotWithUpload {
+		screenOptions = append(screenOptions, option.WithScreenShotUpload(true))
+	}
+
 	// Step 1: Take screenshot and convert to base64
-	screenResult, err := dExt.GetScreenResult(
+	screenResult, err := dExt.GetScreenResult(screenOptions...)
-		option.WithScreenShotFileName("ai_assert"),
-		option.WithScreenShotBase64(true),
-		option.WithScreenShotUpload(true),
-	)
 	if err != nil {
 		return nil, err
 	}
@@ -263,12 +266,17 @@ func (dExt *XTDriver) AIAssert(assertion string, opts ...option.ActionOption) (*
 		ImagePath:         screenResult.ImagePath,
 		Resolution:        &screenResult.Resolution,
 	}
-
+	var imageURL string
+	if screenResult.UploadedURL != "" {
+		imageURL = screenResult.UploadedURL
+	} else {
+		imageURL = screenResult.Base64
+	}
 	// Step 2: Call model and measure time
 	modelCallStartTime := time.Now()
 	assertOpts := &ai.AssertOptions{
 		Assertion:  assertion,
-		Screenshot: screenResult.UploadedURL,
+		Screenshot: imageURL,
 		Size:       screenResult.Resolution,
 	}
 	result, err := dExt.LLMService.Assert(context.Background(), assertOpts)
@@ -298,9 +306,8 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ..
 	// Parse action options to get ResetHistory setting
 	options := option.NewActionOptions(opts...)
 	resetHistory := options.ResetHistory
-	actionOptions := option.NewActionOptions(opts...)
 	screenOptions := []option.ActionOption{option.WithScreenShotFileName("ai_action"), option.WithScreenShotBase64(true)}
-	if actionOptions.ScreenShotWithUpload {
+	if options.ScreenShotWithUpload {
 		screenOptions = append(screenOptions, option.WithScreenShotUpload(true))
 	}
 
@@ -320,7 +327,7 @@ func (dExt *XTDriver) PlanNextAction(ctx context.Context, prompt string, opts ..
 	if screenResult.UploadedURL != "" {
 		imageURL = screenResult.UploadedURL
 	} else {
-		imageURL = screenResult.ImagePath
+		imageURL = screenResult.Base64
 	}
 	planningOpts := &ai.PlanningOptions{
 		UserInstruction: prompt,