From 668f7b6df048ce5b756d0313b62fd051ce655439 Mon Sep 17 00:00:00 2001 From: HFO4 <912394456@qq.com> Date: Sun, 27 Feb 2022 14:36:26 +0800 Subject: [PATCH] Feat: chunked file uploader class --- .../FileManager/Navigator/Navigator.js | 1 - src/component/Uploader/UseUpload.js | 2 +- src/component/Uploader/core/api/index.ts | 18 +++++ src/component/Uploader/core/index.ts | 3 +- src/component/Uploader/core/types.ts | 4 +- src/component/Uploader/core/uploader/base.ts | 78 +++++++++++-------- src/component/Uploader/core/uploader/chunk.ts | 44 +++++++++++ .../Uploader/core/uploader/folder.ts | 3 - src/component/Uploader/core/uploader/local.ts | 4 +- src/component/Uploader/core/utils/helper.ts | 33 ++++++++ src/component/Uploader/core/utils/index.ts | 11 +-- src/component/Uploader/core/utils/pool.ts | 2 +- tsconfig.json | 2 +- 13 files changed, 155 insertions(+), 50 deletions(-) create mode 100644 src/component/Uploader/core/api/index.ts create mode 100644 src/component/Uploader/core/uploader/chunk.ts delete mode 100644 src/component/Uploader/core/uploader/folder.ts create mode 100644 src/component/Uploader/core/utils/helper.ts diff --git a/src/component/FileManager/Navigator/Navigator.js b/src/component/FileManager/Navigator/Navigator.js index face04c..ebce790 100644 --- a/src/component/FileManager/Navigator/Navigator.js +++ b/src/component/FileManager/Navigator/Navigator.js @@ -201,7 +201,6 @@ class NavigatorComponent extends Component { type: response.data.policy.type, maxSize: response.data.policy.max_size, allowedSuffix: response.data.policy.file_type, - chunkSize: response.data.policy.chunk_size, }); } }) diff --git a/src/component/Uploader/UseUpload.js b/src/component/Uploader/UseUpload.js index 63f29f0..cf9d93a 100644 --- a/src/component/Uploader/UseUpload.js +++ b/src/component/Uploader/UseUpload.js @@ -16,7 +16,7 @@ export function useUpload(uploader) { setError(err); setStatus(uploader.status); }, - onProgress: () => {}, + onProgress: (data) => {}, }); /* eslint-enable @typescript-eslint/no-empty-function */ if (status === Status.added) { diff --git a/src/component/Uploader/core/api/index.ts b/src/component/Uploader/core/api/index.ts new file mode 100644 index 0000000..63d8a58 --- /dev/null +++ b/src/component/Uploader/core/api/index.ts @@ -0,0 +1,18 @@ +import { UploadCredential, UploadSessionRequest } from "../types"; +import { requestAPI } from "../utils"; +import { CreateUploadSessionError } from "../errors"; + +export async function createUploadSession( + req: UploadSessionRequest +): Promise { + const res = await requestAPI("file/upload", { + method: "put", + data: req, + }); + + if (res.data.code !== 0) { + throw new CreateUploadSessionError(res.data); + } + + return res.data.data; +} diff --git a/src/component/Uploader/core/index.ts b/src/component/Uploader/core/index.ts index df2976b..0f69e3d 100644 --- a/src/component/Uploader/core/index.ts +++ b/src/component/Uploader/core/index.ts @@ -2,7 +2,6 @@ import { Policy, PolicyType, Task, TaskType } from "./types"; import Logger, { LogLevel } from "./logger"; import { UnknownPolicyError, UploaderError, UploaderErrorName } from "./errors"; import Base from "./uploader/base"; -import Folder from "./uploader/folder"; import Local from "./uploader/local"; import { Pool } from "./utils/pool"; @@ -39,7 +38,7 @@ export default class UploadManager { dispatchUploader(task: Task): Base { if (task.type == TaskType.folder) { - return new Folder(task, this); + //return new Folder(task, this); } switch (task.policy.type) { diff --git a/src/component/Uploader/core/types.ts b/src/component/Uploader/core/types.ts index ebf48a4..effe5d2 100644 --- a/src/component/Uploader/core/types.ts +++ b/src/component/Uploader/core/types.ts @@ -9,7 +9,6 @@ export interface Policy { allowedSuffix: Nullable; maxSize: number; type: PolicyType; - chunkSize: number; } export enum TaskType { @@ -24,6 +23,7 @@ export interface Task { dst: string; file: File; child?: Task[]; + session?: UploadCredential; } type Nullable = T | null; @@ -45,4 +45,6 @@ export interface UploadSessionRequest { export interface UploadCredential { sessionID: string; + expires: number; + chunk_size: number; } diff --git a/src/component/Uploader/core/uploader/base.ts b/src/component/Uploader/core/uploader/base.ts index 8d2a437..6a886a0 100644 --- a/src/component/Uploader/core/uploader/base.ts +++ b/src/component/Uploader/core/uploader/base.ts @@ -1,17 +1,17 @@ // 所有 Uploader 的基类 -import { Task, UploadCredential, UploadSessionRequest } from "../types"; +import { Task } from "../types"; import UploadManager from "../index"; import Logger from "../logger"; import { validate } from "../utils/validator"; -import { CancelToken, requestAPI } from "../utils/request"; +import { CancelToken } from "../utils/request"; import { CancelTokenSource } from "axios"; -import { CreateUploadSessionError } from "../errors"; +import { createUploadSession } from "../api"; export enum Status { added, initialized, - preparing, queued, + preparing, processing, finishing, finished, @@ -22,7 +22,19 @@ export enum Status { export interface UploadHandlers { onTransition: (newStatus: Status) => void; onError: (err: Error) => void; - onProgress: () => void; + onProgress: (data: UploadProgress) => void; +} + +export interface UploadProgress { + total: ProgressCompose; + chunks?: ProgressCompose[]; +} + +export interface ProgressCompose { + size: number; + loaded: number; + percent: number; + fromCache?: boolean; } export default abstract class Base { @@ -35,9 +47,9 @@ export default abstract class Base { protected logger: Logger; protected subscriber: UploadHandlers; - // 用于取消请求 - private cancelToken: CancelTokenSource = CancelToken.source(); + protected cancelToken: CancelTokenSource = CancelToken.source(); + protected progress: UploadProgress; constructor(public task: Task, protected manager: UploadManager) { this.logger = new Logger( @@ -51,7 +63,7 @@ export default abstract class Base { /* eslint-disable @typescript-eslint/no-empty-function */ onTransition: (newStatus: Status) => {}, onError: (err: Error) => {}, - onProgress: () => {}, + onProgress: (data: UploadProgress) => {}, /* eslint-enable @typescript-eslint/no-empty-function */ }; } @@ -67,25 +79,38 @@ export default abstract class Base { try { validate(this.task.file, this.task.policy); } catch (e) { - this.logger.info("File validate failed with error:", e); + this.logger.error("File validate failed with error:", e); this.setError(e); return; } this.logger.info("Enqueued in manager pool"); + this.transit(Status.queued); this.manager.pool.enqueue(this).catch((e) => { this.logger.info("Upload task failed with error:", e); + // TODO: delete upload session this.setError(e); }); }; - public upload = async () => { + public run = async () => { this.logger.info("Start upload task, create upload session..."); this.transit(Status.preparing); - const uploadSession = await this.createUploadSession(); - console.log(uploadSession); + this.task.session = await createUploadSession({ + path: this.task.dst, + size: this.task.file.size, + name: this.task.file.name, + policy_id: this.task.policy.id, + last_modified: this.task.file.lastModified, + }); + this.logger.info("Upload session created:", this.task.session); + + this.transit(Status.processing); + await this.upload(); }; + public abstract async upload(): Promise; + public cancel = async () => { this.cancelToken.cancel(); // TODO: delete upload session @@ -102,25 +127,16 @@ export default abstract class Base { this.subscriber.onTransition(status); } - private async createUploadSession(): Promise { - const req: UploadSessionRequest = { - path: this.task.dst, - size: this.task.file.size, - name: this.task.file.name, - policy_id: this.task.policy.id, - last_modified: this.task.file.lastModified, + public getProgressInfoItem( + loaded: number, + size: number, + fromCache?: boolean + ): ProgressCompose { + return { + size, + loaded, + percent: (loaded / size) * 100, + ...(fromCache == null ? {} : { fromCache }), }; - - const res = await requestAPI("file/upload/session", { - method: "put", - cancelToken: this.cancelToken.token, - data: req, - }); - - if (res.data.code !== 0) { - throw new CreateUploadSessionError(res.data); - } - - return res.data.data; } } diff --git a/src/component/Uploader/core/uploader/chunk.ts b/src/component/Uploader/core/uploader/chunk.ts new file mode 100644 index 0000000..cf8f5e5 --- /dev/null +++ b/src/component/Uploader/core/uploader/chunk.ts @@ -0,0 +1,44 @@ +import Base from "./base"; +import * as utils from "../utils"; + +export interface ChunkLoaded { + chunks: number[]; +} + +export default class Chunk extends Base { + protected chunks: Blob[]; + protected loaded: ChunkLoaded; + + public upload = async () => { + this.logger.info("Starting uploading file chunks."); + this.initBeforeUploadChunks(); + }; + + private initBeforeUploadChunks() { + this.chunks = utils.getChunks( + this.task.file, + this.task.session?.chunk_size + ); + this.loaded = { + chunks: this.chunks.map(() => 0), + }; + this.notifyResumeProgress(); + } + + private notifyResumeProgress() { + this.progress = { + total: this.getProgressInfoItem( + utils.sum(this.loaded.chunks), + this.task.file.size + 1 // 防止在 complete 未调用的时候进度显示 100% + ), + chunks: this.chunks.map((chunk, index) => { + return this.getProgressInfoItem( + this.loaded.chunks[index], + chunk.size, + false + ); + }), + }; + this.subscriber.onProgress(this.progress); + } +} diff --git a/src/component/Uploader/core/uploader/folder.ts b/src/component/Uploader/core/uploader/folder.ts deleted file mode 100644 index 9f5c178..0000000 --- a/src/component/Uploader/core/uploader/folder.ts +++ /dev/null @@ -1,3 +0,0 @@ -import Base from "./base"; - -export default class Folder extends Base {} diff --git a/src/component/Uploader/core/uploader/local.ts b/src/component/Uploader/core/uploader/local.ts index 8109096..88d09cc 100644 --- a/src/component/Uploader/core/uploader/local.ts +++ b/src/component/Uploader/core/uploader/local.ts @@ -1,3 +1,3 @@ -import Base from "./base"; +import Chunk from "./chunk"; -export default class Local extends Base {} +export default class Local extends Chunk {} diff --git a/src/component/Uploader/core/utils/helper.ts b/src/component/Uploader/core/utils/helper.ts new file mode 100644 index 0000000..f9b3044 --- /dev/null +++ b/src/component/Uploader/core/utils/helper.ts @@ -0,0 +1,33 @@ +export const sizeToString = (bytes: number): string => { + if (bytes === 0) return "0 B"; + const k = 1024; + const sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return (bytes / Math.pow(k, i)).toFixed(1) + " " + sizes[i]; +}; + +// 文件分块 +export function getChunks( + file: File, + chunkByteSize: number | undefined +): Blob[] { + // 如果 chunkByteSize 比文件大或为0，则直接取文件的大小 + if (!chunkByteSize || chunkByteSize > file.size || chunkByteSize == 0) { + chunkByteSize = file.size; + } + + const chunks: Blob[] = []; + const count = Math.ceil(file.size / chunkByteSize); + for (let i = 0; i < count; i++) { + const chunk = file.slice( + chunkByteSize * i, + i === count - 1 ? file.size : chunkByteSize * (i + 1) + ); + chunks.push(chunk); + } + return chunks; +} + +export function sum(list: number[]) { + return list.reduce((data, loaded) => data + loaded, 0); +} diff --git a/src/component/Uploader/core/utils/index.ts b/src/component/Uploader/core/utils/index.ts index ed7f078..7ca10e3 100644 --- a/src/component/Uploader/core/utils/index.ts +++ b/src/component/Uploader/core/utils/index.ts @@ -1,7 +1,4 @@ -export const sizeToString = (bytes: number): string => { - if (bytes === 0) return "0 B"; - const k = 1024; - const sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return (bytes / Math.pow(k, i)).toFixed(1) + " " + sizes[i]; -}; +export * from "./pool"; +export * from "./helper"; +export * from "./validator"; +export * from "./request"; diff --git a/src/component/Uploader/core/utils/pool.ts b/src/component/Uploader/core/utils/pool.ts index 4b00e7a..a66dd2c 100644 --- a/src/component/Uploader/core/utils/pool.ts +++ b/src/component/Uploader/core/utils/pool.ts @@ -31,7 +31,7 @@ export class Pool { run(item: QueueContent) { this.queue = this.queue.filter((v) => v !== item); this.processing.push(item); - item.uploader.upload().then( + item.uploader.run().then( () => { item.resolve(); this.release(item); diff --git a/tsconfig.json b/tsconfig.json index 0131634..5ccefd6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,7 +15,7 @@ "jsx": "react", "removeComments": true, "strictFunctionTypes": true, - "strictPropertyInitialization": true, + "strictPropertyInitialization": false, "strictBindCallApply": true, "noImplicitReturns": true, "alwaysStrict": true,