Feature: downloaded file or folder can keey folder structure now

This commit is contained in:
萌萌哒赫萝
2023-02-27 23:11:43 +08:00
parent 750ea58845
commit 1e9c87dc22
17 changed files with 666 additions and 34 deletions
+49 -1
View File
@@ -13,6 +13,7 @@ import UpDownTaskQueue,
commonTaskStatus
} from '../datastore/upDownTaskQueue'
import { ManageLogger } from '../utils/logger'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
// 坑爹阿里云 返回数据类型标注和实际各种不一致
class AliyunApi {
@@ -211,6 +212,53 @@ class AliyunApi {
return res && res.res.status === 200
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, bucketConfig: { Location: region }, prefix, cancelToken } = configMap
const slicedPrefix = prefix.slice(1)
const urlPrefix = configMap.customUrl || `https://${bucket}.${region}.aliyuncs.com`
let marker
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as any
const result = {
fullList: <any>[],
success: false,
finished: false
}
const client = this.getNewCtx(region, bucket)
do {
res = await client.listV2({
prefix: slicedPrefix === '' ? undefined : slicedPrefix,
'max-keys': '1000',
'continuation-token': marker
}, {
timeout: this.timeOut
})
if (res && res.res.statusCode === 200) {
res.objects && res.objects.forEach((item: OSS.ObjectMeta) => {
item.size !== 0 && result.fullList.push(this.formatFile(item, slicedPrefix, urlPrefix))
})
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
marker = res.nextContinuationToken
} while (res.isTruncated === true && !cancelTask[0])
result.success = !cancelTask[0]
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, bucketConfig: { Location: region }, prefix, cancelToken } = configMap
@@ -256,7 +304,7 @@ class AliyunApi {
}
marker = res.nextContinuationToken
} while (res.isTruncated === true && !cancelTask[0])
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+52
View File
@@ -11,6 +11,7 @@ import UpDownTaskQueue,
} from '../datastore/upDownTaskQueue'
import fs from 'fs-extra'
import path from 'path'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
class GithubApi {
token: string
@@ -147,6 +148,57 @@ class GithubApi {
return result
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: repo, customUrl: branch, prefix, cancelToken, cdnUrl } = configMap
const slicedPrefix = prefix.replace(/^\//, '').replace(/\/$/, '')
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as any
const result = {
fullList: <any>[],
success: false,
finished: false
}
const treeQueue = [slicedPrefix]
while (treeQueue.length) {
if (cancelTask[0]) {
result.finished = true
return result
}
const currentPrefix = treeQueue[0]
res = await got(
`${this.baseUrl}/repos/${this.username}/${repo}/git/trees/${branch}:${treeQueue.shift()}`,
getOptions('GET', this.commonHeaders, {}, 'json', undefined, undefined, this.proxy)
) as any
if (res && res.statusCode === 200) {
const { tree } = res.body
tree.forEach((item: any) => {
if (item.type === 'tree') {
treeQueue.push(`${currentPrefix}/${item.path}`)
} else {
result.fullList.push(this.formatFile(item, currentPrefix, branch, repo, cdnUrl))
}
})
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
}
result.success = true
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: repo, customUrl: branch, prefix, cancelToken, cdnUrl } = configMap
+2 -2
View File
@@ -147,9 +147,9 @@ class ImgurApi {
return
}
initPage++
} while (res.body.data.length > 0)
} while (res.body.data.length > 0 && !cancelTask[0])
}
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+58 -1
View File
@@ -12,6 +12,7 @@ import UpDownTaskQueue,
commonTaskStatus
} from '../datastore/upDownTaskQueue'
import { ManageLogger } from '../utils/logger'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
class QiniuApi {
mac: qiniu.auth.digest.Mac
@@ -246,6 +247,62 @@ class QiniuApi {
}
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, prefix, cancelToken, customUrl: urlPrefix } = configMap
let marker = undefined as any
const slicedPrefix = prefix.slice(1)
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as any
const result = {
fullList: <any>[],
success: false,
finished: false
}
const config = new qiniu.conf.Config()
const bucketManager = new qiniu.rs.BucketManager(this.mac, config)
do {
res = await new Promise((resolve, reject) => {
bucketManager.listPrefix(bucket, {
prefix: slicedPrefix === '' ? undefined : slicedPrefix,
marker,
limit: 1000
}, (err: any, respBody: any, respInfo: any) => {
if (err) {
reject(err)
} else {
resolve({
respBody,
respInfo
})
}
})
})
if (res && res.respInfo.statusCode === 200) {
res.respBody && res.respBody.items && res.respBody.items.forEach((item: any) => {
item.fsize !== 0 && result.fullList.push(this.formatFile(item, slicedPrefix, urlPrefix))
})
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
marker = res.respBody.marker
} while (res.respBody && res.respBody.marker && !cancelTask[0])
result.success = !cancelTask[0]
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, prefix, cancelToken, customUrl: urlPrefix } = configMap
@@ -300,7 +357,7 @@ class QiniuApi {
}
marker = res.respBody.marker
} while (res.respBody && res.respBody.marker && !cancelTask[0])
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+64 -4
View File
@@ -31,6 +31,7 @@ import UpDownTaskQueue,
} from '../datastore/upDownTaskQueue'
import fs from 'fs-extra'
import path from 'path'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
interface S3plistApiOptions {
credentials: {
@@ -129,9 +130,9 @@ class S3plistApi {
*/
async getBucketList (): Promise<any> {
const options = Object.assign({}, this.baseOptions) as S3ClientConfig
options.region = 'us-east-1'
const result = [] as IStringKeyMap[]
const endpoint = options.endpoint as string || '' as string
options.region = endpoint.indexOf('cloudflarestorage') !== -1 ? 'auto' : 'us-east-1'
try {
const client = new S3Client(options)
const command = new ListBucketsCommand({})
@@ -180,6 +181,64 @@ class S3plistApi {
return result
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, bucketConfig: { Location: region }, prefix, cancelToken } = configMap
const slicedPrefix = prefix.slice(1)
const urlPrefix = configMap.customUrl || `https://${bucket}.s3.amazonaws.com`
let marker
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as ListObjectsV2CommandOutput
const result = {
fullList: <any>[],
success: false,
finished: false
}
try {
do {
const options = Object.assign({}, this.baseOptions) as S3ClientConfig
options.region = region || 'us-east-1'
const client = new S3Client(options)
const command = new ListObjectsV2Command({
Bucket: bucket,
Prefix: slicedPrefix === '' ? undefined : slicedPrefix,
MaxKeys: 1000,
ContinuationToken: marker
})
res = await client.send(command)
if (res.$metadata.httpStatusCode === 200) {
res.Contents && res.Contents.forEach((item: _Object) => {
result.fullList.push(this.formatFile(item, slicedPrefix, urlPrefix))
})
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
this.logParam(res, 'getBucketListRecursively')
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
marker = res.NextContinuationToken
} while (res.IsTruncated && !cancelTask[0])
} catch (error) {
this.logParam(error, 'getBucketListRecursively')
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
result.success = !cancelTask[0]
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, bucketConfig: { Location: region }, prefix, cancelToken } = configMap
@@ -221,7 +280,7 @@ class S3plistApi {
})
window.webContents.send('refreshFileTransferList', result)
} else {
this.logParam(res, 'getBucketFileList')
this.logParam(res, 'getBucketListBackstage')
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
@@ -230,12 +289,13 @@ class S3plistApi {
marker = res.NextContinuationToken
} while (res.IsTruncated && !cancelTask[0])
} catch (error) {
this.logParam(error, 'getBucketFileList')
this.logParam(error, 'getBucketListBackstage')
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
return
}
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+1 -1
View File
@@ -100,7 +100,7 @@ class SmmsApi {
}
marker++
} while (!cancelTask[0] && res && res.status === 200 && res.data && res.data.success && res.data.CurrentPage < res.data.TotalPages)
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+50 -1
View File
@@ -14,6 +14,7 @@ import UpDownTaskQueue,
downloadTaskSpecialStatus
} from '../datastore/upDownTaskQueue'
import { ManageLogger } from '../utils/logger'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
class TcyunApi {
ctx: COS
@@ -115,6 +116,53 @@ class TcyunApi {
return res && res.statusCode === 200
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const bucket = configMap.bucketName
const region = configMap.bucketConfig.Location
const prefix = configMap.prefix as string
const slicedPrefix = prefix.slice(1, prefix.length)
const urlPrefix = configMap.customUrl || `https://${bucket}.cos.${region}.myqcloud.com`
let marker
const cancelToken = configMap.cancelToken as string
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as COS.GetBucketResult
const result = {
fullList: <any>[],
success: false,
finished: false
}
do {
res = await this.ctx.getBucket({
Bucket: bucket,
Region: region,
Prefix: slicedPrefix === '' ? undefined : slicedPrefix,
Marker: marker
})
if (res && res.statusCode === 200) {
res.Contents.forEach((item: COS.CosObject) =>
parseInt(item.Size) !== 0 && result.fullList.push(this.formatFile(item, slicedPrefix, urlPrefix)))
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
marker = res.NextMarker
} while (res.IsTruncated === 'true' && !cancelTask[0])
result.success = !cancelTask[0]
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise < any > {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const bucket = configMap.bucketName
@@ -159,7 +207,7 @@ class TcyunApi {
}
marker = res.NextMarker
} while (res.IsTruncated === 'true' && !cancelTask[0])
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
@@ -481,6 +529,7 @@ class TcyunApi {
sourceFileName: fileName,
targetFilePath: path.join(downloadPath, fileName)
})
fs.ensureDirSync(path.dirname(path.join(downloadPath, fileName)))
this.ctx.downloadFile({
Bucket: bucketName,
Region: region,
+54 -1
View File
@@ -14,6 +14,7 @@ import UpDownTaskQueue,
commonTaskStatus
} from '../datastore/upDownTaskQueue'
import { ManageLogger } from '../utils/logger'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
class UpyunApi {
ser: Upyun.Service
@@ -91,6 +92,58 @@ class UpyunApi {
return this.bucket
}
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, prefix, cancelToken } = configMap
const slicedPrefix = prefix.slice(1)
const urlPrefix = configMap.customUrl || `http://${bucket}.test.upcdn.net`
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as any
const result = {
fullList: <any>[],
success: false,
finished: false
}
const folderQueue = [prefix]
const getFolderFile = async (folder: any) => {
let marker = ''
const key = folder
do {
res = await this.cli.listDir(key, {
limit: 10000,
iter: marker
})
if (res) {
res.files && res.files.forEach((item: any) => {
item.type === 'F' && folderQueue.push(`${slicedPrefix}${item.name}/`)
item.type === 'N' && result.fullList.push(this.formatFile(item, folder, urlPrefix))
})
window.webContents.send(refreshDownloadFileTransferList, result)
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
marker = res.next
} while (!cancelTask[0] && res.next !== this.stopMarker)
}
while (folderQueue.length) {
const folder = folderQueue.shift()
await getFolderFile(folder)
}
result.success = !cancelTask[0]
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { bucketName: bucket, prefix, cancelToken } = configMap
@@ -129,7 +182,7 @@ class UpyunApi {
}
marker = res.next
} while (!cancelTask[0] && res.next !== this.stopMarker)
result.success = true
result.success = !cancelTask[0]
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
+51
View File
@@ -14,6 +14,7 @@ import UpDownTaskQueue,
} from '../datastore/upDownTaskQueue'
import fs from 'fs-extra'
import path from 'path'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
class WebdavplistApi {
endpoint: string
@@ -85,6 +86,55 @@ class WebdavplistApi {
isRequestSuccess = (code: number) => code >= 200 && code < 300
async getBucketListRecursively (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { prefix, customUrl, cancelToken } = configMap
const urlPrefix = customUrl || this.endpoint
const cancelTask = [false]
ipcMain.on(cancelDownloadLoadingFileList, (_evt: IpcMainEvent, token: string) => {
if (token === cancelToken) {
cancelTask[0] = true
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
})
let res = {} as any
const result = {
fullList: <any>[],
success: false,
finished: false
}
try {
res = await this.ctx.getDirectoryContents(prefix, {
deep: true,
details: true
})
if (this.isRequestSuccess(res.status)) {
if (res.data && res.data.length) {
res.data.forEach((item: FileStat) => {
if (item.type !== 'directory') {
result.fullList.push(this.formatFile(item, urlPrefix))
}
})
}
} else {
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
} catch (error) {
this.logParam(error, 'getBucketListRecursively')
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return
}
result.success = true
result.finished = true
window.webContents.send(refreshDownloadFileTransferList, result)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
}
async getBucketListBackstage (configMap: IStringKeyMap): Promise<any> {
const window = windowManager.get(IWindowList.SETTING_WINDOW)!
const { prefix, customUrl, cancelToken } = configMap
@@ -128,6 +178,7 @@ class WebdavplistApi {
result.finished = true
window.webContents.send('refreshFileTransferList', result)
ipcMain.removeAllListeners('cancelLoadingFileList')
return
}
result.success = true
result.finished = true
+5
View File
@@ -56,6 +56,11 @@ export const manageIpcList = {
return manage.getBucketListBackstage(param)
})
ipcMain.on('getBucketListRecursively', async (_evt: IpcMainInvokeEvent, currentPicBed: string, param: IStringKeyMap) => {
const manage = new ManageApi(currentPicBed)
return manage.getBucketListRecursively(param)
})
ipcMain.handle('openFileSelectDialog', async () => {
const res = await dialog.showOpenDialog({
properties: ['openFile', 'multiSelections']
+39
View File
@@ -18,6 +18,7 @@ import API from './apis/api'
import windowManager from 'apis/app/window/windowManager'
import { IWindowList } from '#/types/enum'
import { ipcMain } from 'electron'
import { cancelDownloadLoadingFileList, refreshDownloadFileTransferList } from '@/manage/utils/static'
export class ManageApi extends EventEmitter implements ManageApiType {
private _config!: Partial<ManageConfigType>
@@ -293,6 +294,44 @@ export class ManageApi extends EventEmitter implements ManageApiType {
}
}
async getBucketListRecursively (
param?: IStringKeyMap
): Promise<IStringKeyMap | ManageError> {
let client
let window
const defaultResult = {
fullList: [],
success: false,
finished: true
}
switch (this.currentPicBedConfig.picBedName) {
case 'tcyun':
case 'aliyun':
case 'qiniu':
case 'upyun':
case 'smms':
case 'github':
case 'imgur':
case 's3plist':
case 'webdavplist':
try {
client = this.createClient() as any
return await client.getBucketListRecursively(param!)
} catch (error: any) {
this.errorMsg(error, this.getMsgParam('getBucketListRecursively'))
window = windowManager.get(IWindowList.SETTING_WINDOW)!
window.webContents.send(refreshDownloadFileTransferList, defaultResult)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return {}
}
default:
window = windowManager.get(IWindowList.SETTING_WINDOW)!
window.webContents.send(refreshDownloadFileTransferList, defaultResult)
ipcMain.removeAllListeners(cancelDownloadLoadingFileList)
return {}
}
}
/**
* 后台更新bucket文件列表
* @param param