🚧 WIP(custom): optimize gallery db sync logic and force refresh gallery

src/main/apis/core/datastore/index.ts

@@ -107,8 +107,8 @@ class GalleryDB {
     console.log('init gallery db')
   }
 
-  static getInstance(): DBStore {
+  static getInstance(forceRefresh: boolean = false): DBStore {
-    if (!GalleryDB.#instance) {
+    if (!GalleryDB.#instance || forceRefresh) {
       GalleryDB.#instance = new DBStore(DB_PATH, 'gallery')
     }
     return GalleryDB.#instance

src/main/utils/configPaths.ts

@@ -100,6 +100,8 @@ export interface IConfigStruct {
     autoImport: boolean
     autoImportPicBed: string[]
     galleryPicBedFilter: string[]
+    enableSecondUploader?: boolean
+    lastSyncTime?: number
   }
   needReload: boolean
   picgoPlugins: IPicGoPlugins
@@ -189,6 +191,7 @@ export const configPaths = {
     autoImportPicBed: 'settings.autoImportPicBed',
     galleryPicBedFilter: 'settings.galleryPicBedFilter',
     enableSecondUploader: 'settings.enableSecondUploader',
+    lastSyncTime: 'settings.lastSyncTime',
   },
   needReload: 'needReload',
   picgoPlugins: 'picgoPlugins',

src/main/utils/syncSettings.ts

@@ -2,6 +2,7 @@ import os from 'node:os'
 import path from 'node:path'
 
 import db from '@core/datastore'
+import { GalleryDB } from '@core/datastore'
 import logger from '@core/picgo/logger'
 import { Octokit } from '@octokit/rest'
 import axios from 'axios'
@@ -10,15 +11,15 @@ import fs from 'fs-extra'
 import { HttpsProxyAgent } from 'hpagent'
 import { AuthType, createClient, WebDAVClientOptions } from 'webdav'
 
-import type { ISyncConfig } from '#/types/types'
+import type { IGalleryDBFile, IGalleryDBGalleryItem, ISyncConfig } from '#/types/types'
 import { extractData, zipData } from '~/utils/common'
 import { formatEndpoint } from '~/utils/common'
 import { configPaths } from '~/utils/configPaths'
 
 const STORE_PATH = app.getPath('userData')
 const tempDir = path.join(os.tmpdir(), `piclist-sync-tmp`)
-const db1 = path.join(tempDir, 'db1')
+const localDBPath = path.join(tempDir, 'db1')
-const db2 = path.join(tempDir, 'db2')
+const remoteDBPath = path.join(tempDir, 'db2')
 const dbMerged = path.join(tempDir, 'db-merged')
 const galleryDBList = ['piclist.db', 'piclist.bak.db']
 
@@ -30,37 +31,50 @@ const uploadOrUpdateMsg = (fileName: string, isUpdate: boolean = true) =>
   isUpdate ? `update ${fileName} from PicList` : `upload ${fileName} from PicList`
 
 const emptyDir = async (): Promise<void> => {
-  await fs.emptyDir(tempDir)
+  for (const dir of [tempDir, localDBPath, remoteDBPath, dbMerged]) {
-  await fs.emptyDir(db1)
+    await fs.emptyDir(dir)
-  await fs.emptyDir(db2)
+  }
-  await fs.emptyDir(dbMerged)
 }
 
 const mergeGalleryDB = async (targetFile: string) => {
+  const lastSyncTime = db.get(configPaths.settings.lastSyncTime) || 0
   try {
-    const db1Data = await extractData(path.join(db1, targetFile))
+    const localDBData = (await extractData(path.join(localDBPath, targetFile))) as IGalleryDBFile
-    const db2Data = await extractData(path.join(db2, targetFile))
+    const remoteDBData = (await extractData(path.join(remoteDBPath, targetFile))) as IGalleryDBFile
-    const mergedData: any = {
+    const localMap = new Map(localDBData.gallery.map((item: IGalleryDBGalleryItem) => [item.id, item]))
-      gallery: [],
+    const remoteMap = new Map(remoteDBData.gallery.map((item: IGalleryDBGalleryItem) => [item.id, item]))
-      __gallery_KEY__: {},
+    const mergedGalleryMap = new Map<string, any>()
-    }
+
-    const db1Ids = new Set<string>(Object.keys(db1Data.__gallery_KEY__ || {}))
+    for (const [id, localItem] of localMap) {
-    const db2Ids = new Set<string>(Object.keys(db2Data.__gallery_KEY__ || {}))
+      const remoteItem = remoteMap.get(id)
-    const idSet = new Set<string>([...db1Ids, ...db2Ids])
+      if (!remoteItem) {
-    for (const id of idSet) {
+        mergedGalleryMap.set(id, localItem)
-      if (db2Ids.has(id)) {
+      } else {
-        mergedData.gallery.push(db2Data.gallery.find((item: any) => item.id === id))
+        const newest = (localItem.updatedAt || 0) >= (remoteItem.updatedAt || 0) ? localItem : remoteItem
-      } else if (db1Ids.has(id)) {
+        mergedGalleryMap.set(id, newest)
-        mergedData.gallery.push(db1Data.gallery.find((item: any) => item.id === id))
       }
     }
-    for (const item of mergedData.gallery) {
+    for (const [id, remoteItem] of remoteMap) {
-      mergedData.__gallery_KEY__[item.id] = 1
+      if (!localMap.has(id) && (remoteItem.updatedAt || 0) >= lastSyncTime) {
+        console.log('newer in remote:', JSON.stringify(remoteItem))
+        mergedGalleryMap.set(id, remoteItem)
+      }
     }
-    await zipData(mergedData, path.join(dbMerged, targetFile))
+
-    await fs.copyFile(path.join(dbMerged, targetFile), path.join(STORE_PATH, targetFile))
+    const galleryKeyObj: Record<string, number> = {}
+    mergedGalleryMap.forEach((_, id) => {
+      galleryKeyObj[id] = 1
+    })
+    const mergedData = {
+      gallery: Array.from(mergedGalleryMap.values()),
+      __gallery_KEY__: galleryKeyObj,
+    }
+    const targetFilePath = path.join(dbMerged, targetFile)
+    await zipData(mergedData, targetFilePath)
+    await fs.copyFile(targetFilePath, path.join(STORE_PATH, targetFile))
   } catch (err: any) {
     logger.error('merge gallery db failed:', String(err))
+    throw new Error('merge gallery db failed')
   }
 }
 
@@ -353,7 +367,7 @@ async function downloadAndWriteFile(url: string, localFilePath: string, config:
 }
 
 async function downloadRemoteToLocal(syncConfig: ISyncConfig, fileName: string, galleryMode = false) {
-  const storePath = galleryMode ? db2 : STORE_PATH
+  const storePath = galleryMode ? remoteDBPath : STORE_PATH
   const localFilePath = path.join(storePath, fileName)
   const { username, repo, branch, token, proxy, type } = syncConfig
   try {
@@ -582,6 +596,8 @@ async function syncGallery(): Promise<number> {
         await uploadLocalToRemote(syncConfig, file)
         logger.info(`gallery db ${file} not exist in cloud, upload local file instead`)
         successCount++
+        db.set(configPaths.settings.lastSyncTime, Date.now())
+        GalleryDB.getInstance(true)
         continue
       }
     } catch (err: any) {
@@ -589,9 +605,11 @@ async function syncGallery(): Promise<number> {
       continue
     }
     await downloadRemoteToLocal(syncConfig, file, true)
-    await fs.copyFile(path.join(STORE_PATH, file), path.join(db1, file))
+    await fs.copyFile(path.join(STORE_PATH, file), path.join(localDBPath, file))
     await mergeGalleryDB(file)
     await updateLocalToRemote(syncConfig, file)
+    db.set(configPaths.settings.lastSyncTime, Date.now())
+    GalleryDB.getInstance(true) // refresh gallery db instance
     logger.info(`sync gallery db ${file} success`)
     successCount++
   }

src/renderer/i18n/locales/en.json

@@ -717,10 +717,10 @@
         "syncConfigNote": "The files to be synced are configuration files",
         "syncConfigProxy": "Proxy",
         "syncConfiguration": "Sync Configuration",
-        "syncEndpointConfig": "Sync Endpoint Configuration",
+        "syncEndpointConfig": "Endpoint Configuration",
         "syncResult": { "failed": "Sync Failed", "success": "Sync Successful" },
         "title": "Sync",
-        "upDownloadSettings": "Upload and Download Settings",
+        "upDownloadSettings": "Sync Config & Gallery",
         "uploadSettings": "Upload Settings",
         "webdav": {
           "authType": "WebDAV Auth Type",

src/renderer/i18n/locales/zh-CN.json

@@ -712,10 +712,10 @@
         "syncConfigNote": "同步的文件为配置文件",
         "syncConfigProxy": "代理",
         "syncConfiguration": "同步配置",
-        "syncEndpointConfig": "同步方案配置",
+        "syncEndpointConfig": "平台设置",
         "syncResult": { "failed": "同步失败", "success": "同步成功" },
-        "title": "配置/同步",
+        "title": "配置/相册同步",
-        "upDownloadSettings": "上传下载配置文件",
+        "upDownloadSettings": "同步配置和相册",
         "uploadSettings": "上传配置",
         "webdav": {
           "authType": "WebDAV 认证类型",

src/renderer/i18n/locales/zh-TW.json

@@ -712,10 +712,10 @@
         "syncConfigNote": "同步的文件為配置文件",
         "syncConfigProxy": "代理",
         "syncConfiguration": "同步配置",
-        "syncEndpointConfig": "同步方案配置",
+        "syncEndpointConfig": "平台配置",
         "syncResult": { "failed": "同步失敗", "success": "同步成功" },
-        "title": "配置/同步",
+        "title": "配置/相冊同步",
-        "upDownloadSettings": "上傳下載配置文件",
+        "upDownloadSettings": "同步配置和相冊",
         "uploadSettings": "上傳配置",
         "webdav": {
           "authType": "WebDAV 認證類型",

src/universal/types/types.ts

@@ -523,3 +523,13 @@ export interface IHTTPProxy {
   port: number
   protocol: string
 }
+
+export interface IGalleryDBGalleryItem {
+  id: string
+  updatedAt?: number
+  [propName: string]: any
+}
+export interface IGalleryDBFile {
+  gallery: IGalleryDBGalleryItem[]
+  __gallery_KEY__: Record<string, number>
+}