✨ Feature: add concurrency limit for download

src/main/manage/apis/aliyun.ts

@@ -1,7 +1,6 @@
 import axios from 'axios'
-import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common'
+import { hmacSha1Base64, getFileMimeType, formatError, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
 import { ipcMain, IpcMainEvent } from 'electron'
-import fs from 'fs-extra'
 import { XMLParser } from 'fast-xml-parser'
 import OSS from 'ali-oss'
 import path from 'path'
@@ -548,19 +547,13 @@ class AliyunApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
-    // fileArray = [{
-    //   bucketName: string,
-    //   region: string,
-    //   key: string,
-    //  fileName: string
-    // }]
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName } = item
       const client = this.getNewCtx(region, bucketName)
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -575,8 +568,21 @@ class AliyunApi {
       const preSignedUrl = client.signatureUrl(key, {
         expires: 60 * 60 * 48
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error: any) => {
+      this.logger.error(formatError(error, { class: 'AliyunApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/github.ts

@@ -1,10 +1,10 @@
 import got from 'got'
 import { ManageLogger } from '../utils/logger'
-import { isImage } from '~/renderer/manage/utils/common'
+import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
-import { gotUpload, trimPath, gotDownload, getAgent, getOptions } from '../utils/common'
+import { gotUpload, trimPath, NewDownloader, getAgent, getOptions, ConcurrencyPromisePool, formatError } from '../utils/common'
 import UpDownTaskQueue,
 {
   commonTaskStatus
@@ -17,6 +17,7 @@ class GithubApi {
   username: string
   logger: ManageLogger
   proxy: any
+  proxyStr: string | undefined
   baseUrl = 'https://api.github.com'
   commonHeaders : IStringKeyMap
 
@@ -25,6 +26,7 @@ class GithubApi {
     this.token = token.startsWith('Bearer ') ? token : `Bearer ${token}`.trim()
     this.username = username
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.commonHeaders = {
       Authorization: this.token,
       Accept: 'application/vnd.github+json'
@@ -388,13 +390,13 @@ class GithubApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName: repo, customUrl: branch, key, fileName, githubPrivate, githubUrl } = item
       const id = `${repo}-${branch}-${key}-${fileName}`
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       if (instance.getDownloadTask(id)) {
         continue
       }
@@ -405,7 +407,7 @@ class GithubApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      let downloadUrl
+      let downloadUrl: string
       if (githubPrivate) {
         const preSignedUrl = await this.getPreSignedUrl({
           bucketName: repo,
@@ -418,17 +420,28 @@ class GithubApi {
       } else {
         downloadUrl = githubUrl
       }
-      gotDownload(
-        instance,
-        downloadUrl,
-        fileStream,
-        id,
-        savedFilePath,
-        this.logger,
-        undefined,
-        getAgent(this.proxy)
-      )
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(
+          instance,
+          downloadUrl,
+          id,
+          savedFilePath,
+          this.logger,
+          this.proxyStr
+        )
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'GithubApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/imgur.ts

@@ -1,10 +1,10 @@
 import got from 'got'
 import ManageLogger from '../utils/logger'
-import { getAgent, getOptions, gotDownload, gotUpload, getFileMimeType } from '../utils/common'
+import { getAgent, getOptions, NewDownloader, gotUpload, getFileMimeType, ConcurrencyPromisePool, formatError } from '../utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
-import { isImage } from '~/renderer/manage/utils/common'
+import { formatHttpProxy, isImage } from '~/renderer/manage/utils/common'
 import path from 'path'
 import UpDownTaskQueue,
 {
@@ -18,6 +18,7 @@ class ImgurApi {
   accessToken: string
   proxy: any
   logger: ManageLogger
+  proxyStr: string | undefined
   tokenHeaders: any
   idHeaders: any
   baseUrl = 'https://api.imgur.com/3'
@@ -26,6 +27,7 @@ class ImgurApi {
     this.userName = userName
     this.accessToken = accessToken.startsWith('Bearer ') ? accessToken : `Bearer ${accessToken}`
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.logger = logger
     this.tokenHeaders = {
       Authorization: this.accessToken
@@ -227,13 +229,13 @@ class ImgurApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, githubUrl: url } = item
       const id = `${bucketName}-${region}-${key}-${fileName}`
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       if (instance.getDownloadTask(id)) {
         continue
       }
@@ -244,17 +246,28 @@ class ImgurApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      gotDownload(
-        instance,
-        url,
-        fileStream,
-        id,
-        savedFilePath,
-        this.logger,
-        undefined,
-        getAgent(this.proxy)
-      )
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(
+          instance,
+          url,
+          id,
+          savedFilePath,
+          this.logger,
+          this.proxyStr
+        )
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'ImgurApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/qiniu.ts

@@ -1,6 +1,5 @@
 import axios from 'axios'
-import { hmacSha1Base64, getFileMimeType, gotDownload, formatError } from '../utils/common'
-import fs from 'fs-extra'
+import { hmacSha1Base64, getFileMimeType, NewDownloader, formatError, ConcurrencyPromisePool } from '../utils/common'
 import qiniu from 'qiniu/index'
 import path from 'path'
 import { isImage } from '~/renderer/manage/utils/common'
@@ -627,12 +626,12 @@ class QiniuApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -645,8 +644,21 @@ class QiniuApi {
         targetFilePath: savedFilePath
       })
       const preSignedUrl = await this.getPreSignedUrl({ key, expires: 36000, customUrl })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'QiniuApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/s3plist.ts

@@ -18,8 +18,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import https from 'https'
 import http from 'http'
 import { ManageLogger } from '../utils/logger'
-import { formatEndpoint, formatError, getAgent, getFileMimeType, gotDownload } from '../utils/common'
-import { isImage } from '@/manage/utils/common'
+import { formatEndpoint, formatError, getAgent, getFileMimeType, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
+import { isImage, formatHttpProxy } from '@/manage/utils/common'
 import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
@@ -49,6 +49,7 @@ class S3plistApi {
   baseOptions: S3plistApiOptions
   logger: ManageLogger
   agent: any
+  proxy: string | undefined
 
   constructor (
     accessKeyId: string,
@@ -73,6 +74,7 @@ class S3plistApi {
     } as S3plistApiOptions
     this.logger = logger
     this.agent = this.setAgent(proxy, sslEnabled)
+    this.proxy = formatHttpProxy(proxy, 'string') as string | undefined
   }
 
   setAgent (proxy: string | undefined, sslEnabled: boolean) : HttpProxyAgent | HttpsProxyAgent | undefined {
@@ -129,32 +131,43 @@ class S3plistApi {
     const options = Object.assign({}, this.baseOptions) as S3ClientConfig
     options.region = 'us-east-1'
     const result = [] as IStringKeyMap[]
+    const endpoint = options.endpoint as string || '' as string
     try {
       const client = new S3Client(options)
       const command = new ListBucketsCommand({})
       const data = await client.send(command)
       if (data.$metadata.httpStatusCode === 200) {
         if (data.Buckets) {
-          for (let i = 0; i < data.Buckets.length; i++) {
-            const bucket = data.Buckets[i]
-            const bucketName = bucket.Name
-            const command = new GetBucketLocationCommand({
-              Bucket: bucketName
+          if (endpoint.indexOf('cloudflarestorage') !== -1) {
+            data.Buckets.forEach((bucket) => {
+              result.push({
+                Name: bucket.Name,
+                CreationDate: bucket.CreationDate,
+                Location: 'auto'
+              })
             })
-            const bucketConfig = await client.send(command)
-            if (bucketConfig.$metadata.httpStatusCode === 200) {
-              result.push({
-                Name: bucketName,
-                CreationDate: bucket.CreationDate,
-                Location: bucketConfig.LocationConstraint || 'us-east-1'
-              })
-            } else {
-              this.logParam(bucketConfig, 'getBucketList')
-              result.push({
-                Name: bucketName,
-                CreationDate: bucket.CreationDate,
-                Location: 'us-east-1'
+          } else {
+            for (let i = 0; i < data.Buckets.length; i++) {
+              const bucket = data.Buckets[i]
+              const bucketName = bucket.Name
+              const command = new GetBucketLocationCommand({
+                Bucket: bucketName
               })
+              const bucketConfig = await client.send(command)
+              if (bucketConfig.$metadata.httpStatusCode === 200) {
+                result.push({
+                  Name: bucketName,
+                  CreationDate: bucket.CreationDate,
+                  Location: bucketConfig.LocationConstraint?.toLowerCase() || 'us-east-1'
+                })
+              } else {
+                this.logParam(bucketConfig, 'getBucketList')
+                result.push({
+                  Name: bucketName,
+                  CreationDate: bucket.CreationDate,
+                  Location: 'us-east-1'
+                })
+              }
             }
           }
         }
@@ -573,18 +586,12 @@ class S3plistApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
-    // fileArray = [{
-    //   bucketName: string,
-    //   region: string,
-    //   key: string,
-    //  fileName: string
-    // }]
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}-${savedFilePath}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -603,8 +610,21 @@ class S3plistApi {
         expires: 36000,
         customUrl
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxy)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logParam(error, 'downloadBucketFile')
+    })
     return true
   }
 }

src/main/manage/apis/smms.ts

@@ -5,7 +5,7 @@ import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
 import FormData from 'form-data'
 import fs from 'fs-extra'
-import { getFileMimeType, gotUpload, gotDownload } from '../utils/common'
+import { getFileMimeType, gotUpload, NewDownloader, ConcurrencyPromisePool, formatError } from '../utils/common'
 import path from 'path'
 import UpDownTaskQueue, { commonTaskStatus } from '../datastore/upDownTaskQueue'
 import { ManageLogger } from '../utils/logger'
@@ -227,12 +227,12 @@ class SmmsApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, downloadUrl: preSignedUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -244,8 +244,21 @@ class SmmsApi {
         sourceFileName: fileName,
         targetFilePath: savedFilePath
       })
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'SmmsApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/upyun.ts

@@ -1,6 +1,6 @@
 // @ts-ignore
 import Upyun from 'upyun'
-import { md5, hmacSha1Base64, getFileMimeType, gotDownload, gotUpload } from '../utils/common'
+import { md5, hmacSha1Base64, getFileMimeType, NewDownloader, gotUpload, ConcurrencyPromisePool, formatError } from '../utils/common'
 import { isImage } from '~/renderer/manage/utils/common'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
@@ -359,12 +359,12 @@ class UpyunApi {
    * @param configMap
    */
   async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
-    const { downloadPath, fileArray } = configMap
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
     const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
     for (const item of fileArray) {
       const { bucketName, region, key, fileName, customUrl } = item
       const savedFilePath = path.join(downloadPath, fileName)
-      const fileStream = fs.createWriteStream(savedFilePath)
       const id = `${bucketName}-${region}-${key}`
       if (instance.getDownloadTask(id)) {
         continue
@@ -377,8 +377,21 @@ class UpyunApi {
         targetFilePath: savedFilePath
       })
       const preSignedUrl = `${customUrl}/${key}`
-      gotDownload(instance, preSignedUrl, fileStream, id, savedFilePath, this.logger)
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
     }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logger.error(formatError(error, { class: 'UpyunApi', method: 'downloadBucketFile' }))
+    })
     return true
   }
 }

src/main/manage/apis/webdavplist.ts

@@ -1,12 +1,19 @@
 import ManageLogger from '../utils/logger'
-import { createClient, WebDAVClient, FileStat } from 'webdav'
-import { formatError, formatEndpoint, getInnerAgent } from '../utils/common'
-import { isImage } from '@/manage/utils/common'
+import { createClient, WebDAVClient, FileStat, ProgressEvent } from 'webdav'
+import { formatError, formatEndpoint, getInnerAgent, NewDownloader, ConcurrencyPromisePool } from '../utils/common'
+import { formatHttpProxy, isImage } from '@/manage/utils/common'
 import http from 'http'
 import https from 'https'
 import windowManager from 'apis/app/window/windowManager'
 import { IWindowList } from '#/types/enum'
 import { ipcMain, IpcMainEvent } from 'electron'
+import UpDownTaskQueue,
+{
+  uploadTaskSpecialStatus,
+  commonTaskStatus
+} from '../datastore/upDownTaskQueue'
+import fs from 'fs-extra'
+import path from 'path'
 
 class WebdavplistApi {
   endpoint: string
@@ -14,6 +21,7 @@ class WebdavplistApi {
   password: string
   sslEnabled: boolean
   proxy: string | undefined
+  proxyStr: string | undefined
   logger: ManageLogger
   agent: https.Agent | http.Agent
   ctx: WebDAVClient
@@ -24,6 +32,7 @@ class WebdavplistApi {
     this.password = password
     this.sslEnabled = sslEnabled
     this.proxy = proxy
+    this.proxyStr = formatHttpProxy(proxy, 'string') as string | undefined
     this.logger = logger
     this.agent = getInnerAgent(proxy, sslEnabled).agent
     this.ctx = createClient(
@@ -125,6 +134,173 @@ class WebdavplistApi {
     window.webContents.send('refreshFileTransferList', result)
     ipcMain.removeAllListeners('cancelLoadingFileList')
   }
+
+  async renameBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { oldKey, newKey } = configMap
+    let result = false
+    try {
+      await this.ctx.moveFile(oldKey, newKey)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'renameBucketFile')
+    }
+    return result
+  }
+
+  async deleteBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.deleteFile(key)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'deleteBucketFile')
+    }
+    return result
+  }
+
+  async deleteBucketFolder (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.deleteFile(key)
+      result = true
+    } catch (error) {
+      this.logParam(error, 'deleteBucketFolder')
+    }
+    return result
+  }
+
+  async getPreSignedUrl (configMap: IStringKeyMap): Promise<string> {
+    const { key } = configMap
+    let result = ''
+    try {
+      const res = this.ctx.getFileDownloadLink(key)
+      result = res
+    } catch (error) {
+      this.logParam(error, 'getPreSignedUrl')
+    }
+    return result
+  }
+
+  async uploadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { fileArray } = configMap
+    const instance = UpDownTaskQueue.getInstance()
+    for (const item of fileArray) {
+      const { alias, bucketName, region, key, filePath, fileName } = item
+      const id = `${alias}-${bucketName}-${key}-${filePath}`
+      if (instance.getUploadTask(id)) {
+        continue
+      }
+      instance.addUploadTask({
+        id,
+        progress: 0,
+        status: commonTaskStatus.queuing,
+        sourceFileName: fileName,
+        sourceFilePath: filePath,
+        targetFilePath: key,
+        targetFileBucket: bucketName,
+        targetFileRegion: region,
+        noProgress: true
+      })
+      this.ctx.putFileContents(
+        key,
+        fs.createReadStream(filePath),
+        {
+          overwrite: true,
+          onUploadProgress: (progressEvent: ProgressEvent) => {
+            instance.updateUploadTask({
+              id,
+              progress: Math.floor((progressEvent.loaded / progressEvent.total) * 100),
+              status: uploadTaskSpecialStatus.uploading
+            })
+          }
+        }
+      ).then((res: boolean) => {
+        if (res) {
+          instance.updateUploadTask({
+            id,
+            progress: 100,
+            status: uploadTaskSpecialStatus.uploaded,
+            finishTime: new Date().toLocaleString()
+          })
+        } else {
+          instance.updateUploadTask({
+            id,
+            progress: 0,
+            status: commonTaskStatus.failed,
+            finishTime: new Date().toLocaleString()
+          })
+        }
+      }).catch((error: any) => {
+        this.logParam(error, 'uploadBucketFile')
+        instance.updateUploadTask({
+          id,
+          progress: 0,
+          status: commonTaskStatus.failed,
+          finishTime: new Date().toLocaleString()
+        })
+      })
+    }
+    return true
+  }
+
+  async createBucketFolder (configMap: IStringKeyMap): Promise<boolean> {
+    const { key } = configMap
+    let result = false
+    try {
+      await this.ctx.createDirectory(key, {
+        recursive: true
+      })
+      result = true
+    } catch (error) {
+      this.logParam(error, 'createBucketFolder')
+    }
+    return result
+  }
+
+  async downloadBucketFile (configMap: IStringKeyMap): Promise<boolean> {
+    const { downloadPath, fileArray, maxDownloadFileCount } = configMap
+    const instance = UpDownTaskQueue.getInstance()
+    const promises = [] as any
+    for (const item of fileArray) {
+      const { alias, bucketName, region, key, fileName } = item
+      const savedFilePath = path.join(downloadPath, fileName)
+      const id = `${alias}-${bucketName}-${region}-${key}`
+      if (instance.getDownloadTask(id)) {
+        continue
+      }
+      instance.addDownloadTask({
+        id,
+        progress: 0,
+        status: commonTaskStatus.queuing,
+        sourceFileName: fileName,
+        targetFilePath: savedFilePath
+      })
+      const preSignedUrl = await this.getPreSignedUrl({
+        key
+      })
+      const base64Str = Buffer.from(`${this.username}:${this.password}`).toString('base64')
+      const headers = {
+        Authorization: `Basic ${base64Str}`
+      }
+      promises.push(() => new Promise((resolve, reject) => {
+        NewDownloader(instance, preSignedUrl, id, savedFilePath, this.logger, this.proxyStr, headers)
+          .then((res: boolean) => {
+            if (res) {
+              resolve(res)
+            } else {
+              reject(res)
+            }
+          })
+      }))
+    }
+    const pool = new ConcurrencyPromisePool(maxDownloadFileCount)
+    pool.all(promises).catch((error) => {
+      this.logParam(error, 'downloadBucketFile')
+    })
+    return true
+  }
 }
 
 export default WebdavplistApi

src/main/manage/manageApi.ts

@@ -387,6 +387,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           const res = await client.deleteBucketFile(param!)
@@ -411,6 +412,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'upyun':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.deleteBucketFolder(param!)
@@ -433,6 +435,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'qiniu':
       case 'upyun':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.renameBucketFile(param!)
@@ -458,6 +461,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           const res = await client.downloadBucketFile(param!)
@@ -489,6 +493,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'upyun':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.createBucketFolder(param!)
@@ -514,6 +519,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'github':
       case 'imgur':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.uploadBucketFile(param!)
@@ -536,6 +542,7 @@ export class ManageApi extends EventEmitter implements ManageApiType {
       case 'qiniu':
       case 'github':
       case 's3plist':
+      case 'webdavplist':
         try {
           client = this.createClient() as any
           return await client.getPreSignedUrl(param!)

src/main/manage/utils/common.ts

@@ -18,6 +18,7 @@ import { formatHttpProxy, IHTTPProxy } from '@/manage/utils/common'
 import { HttpsProxyAgent, HttpProxyAgent } from 'hpagent'
 import http from 'http'
 import https from 'https'
+import Downloader from 'nodejs-file-downloader'
 
 export const getFSFile = async (
   filePath: string,
@@ -90,52 +91,57 @@ export const md5 = (str: string, code: 'hex' | 'base64'): string => crypto.creat
 
 export const hmacSha1Base64 = (secretKey: string, stringToSign: string) : string => crypto.createHmac('sha1', secretKey).update(Buffer.from(stringToSign, 'utf8')).digest('base64')
 
-export const gotDownload = async (
+export const NewDownloader = async (
   instance: UpDownTaskQueue,
   preSignedUrl: string,
-  fileStream: fs.WriteStream,
   id : string,
   savedFilePath: string,
   logger?: ManageLogger,
-  param?: any,
-  agent: any = {}
-) => {
-  got(
-    preSignedUrl,
-    {
-      isStream: true,
-      throwHttpErrors: false,
-      searchParams: param,
-      agent: agent || {}
-    }
-  )
-    .on('downloadProgress', (progress: any) => {
+  proxy?: string,
+  headers?: any
+) : Promise<boolean> => {
+  const options = {
+    url: preSignedUrl,
+    directory: path.dirname(savedFilePath),
+    fileName: path.basename(savedFilePath),
+    cloneFiles: false,
+    onProgress: (percentage: string) => {
       instance.updateDownloadTask({
         id,
-        progress: Math.floor(progress.percent * 100),
+        progress: Math.floor(Number(percentage)),
         status: downloadTaskSpecialStatus.downloading
       })
+    },
+    maxAttempts: 3
+  } as any
+  if (proxy) {
+    options.proxy = proxy
+  }
+  if (headers) {
+    options.headers = headers
+  }
+  const downloader = new Downloader(options)
+  try {
+    await downloader.download()
+    instance.updateDownloadTask({
+      id,
+      progress: 100,
+      status: downloadTaskSpecialStatus.downloaded,
+      finishTime: new Date().toLocaleString()
     })
-    .pipe(fileStream)
-    .on('close', () => {
-      instance.updateDownloadTask({
-        id,
-        progress: 100,
-        status: downloadTaskSpecialStatus.downloaded,
-        finishTime: new Date().toLocaleString()
-      })
-    })
-    .on('error', (err: any) => {
-      logger && logger.error(formatError(err, { method: 'gotDownload' }))
-      fs.remove(savedFilePath)
-      instance.updateDownloadTask({
-        id,
-        progress: 0,
-        status: commonTaskStatus.failed,
-        response: formatError(err, { method: 'gotDownload' }),
-        finishTime: new Date().toLocaleString()
-      })
+    return true
+  } catch (e: any) {
+    logger && logger.error(formatError(e, { method: 'NewDownloader' }))
+    fs.remove(savedFilePath)
+    instance.updateDownloadTask({
+      id,
+      progress: 0,
+      status: commonTaskStatus.failed,
+      response: formatError(e, { method: 'NewDownloader' }),
+      finishTime: new Date().toLocaleString()
     })
+    return false
+  }
 }
 
 export const gotUpload = async (
@@ -320,3 +326,45 @@ export const formatEndpoint = (endpoint: string, sslEnabled: boolean): string =>
     : sslEnabled
       ? endpoint.replace('http://', 'https://')
       : endpoint.replace('https://', 'http://')
+
+export class ConcurrencyPromisePool {
+  limit: number
+  queue: any[]
+  runningNum: number
+  results: any[]
+
+  constructor (limit: number) {
+    this.limit = limit
+    this.queue = []
+    this.runningNum = 0
+    this.results = []
+  }
+
+  all (promises: any[] = []) {
+    return new Promise((resolve, reject) => {
+      for (const promise of promises) {
+        this._run(promise, resolve, reject)
+      }
+    })
+  }
+
+  _run (promise: any, resolve: any, reject: any) {
+    if (this.runningNum >= this.limit) {
+      this.queue.push(promise)
+      return
+    }
+    this.runningNum += 1
+    promise()
+      .then((res: any) => {
+        this.results.push(res)
+        --this.runningNum
+        if (this.queue.length === 0 && this.runningNum === 0) {
+          return resolve(this.results)
+        }
+        if (this.queue.length > 0) {
+          this._run(this.queue.shift(), resolve, reject)
+        }
+      })
+      .catch(reject)
+  }
+}