Feat: chunked file uploader class

src/component/FileManager/Navigator/Navigator.js

@@ -201,7 +201,6 @@ class NavigatorComponent extends Component {
                         type: response.data.policy.type,
                         maxSize: response.data.policy.max_size,
                         allowedSuffix: response.data.policy.file_type,
-                        chunkSize: response.data.policy.chunk_size,
                     });
                 }
             })

src/component/Uploader/UseUpload.js

@@ -16,7 +16,7 @@ export function useUpload(uploader) {
                 setError(err);
                 setStatus(uploader.status);
             },
-            onProgress: () => {},
+            onProgress: (data) => {},
         });
         /* eslint-enable @typescript-eslint/no-empty-function */
         if (status === Status.added) {

src/component/Uploader/core/api/index.ts

@@ -0,0 +1,18 @@
+import { UploadCredential, UploadSessionRequest } from "../types";
+import { requestAPI } from "../utils";
+import { CreateUploadSessionError } from "../errors";
+
+export async function createUploadSession(
+    req: UploadSessionRequest
+): Promise<UploadCredential> {
+    const res = await requestAPI<UploadCredential>("file/upload", {
+        method: "put",
+        data: req,
+    });
+
+    if (res.data.code !== 0) {
+        throw new CreateUploadSessionError(res.data);
+    }
+
+    return res.data.data;
+}

src/component/Uploader/core/index.ts

@@ -2,7 +2,6 @@ import { Policy, PolicyType, Task, TaskType } from "./types";
 import Logger, { LogLevel } from "./logger";
 import { UnknownPolicyError, UploaderError, UploaderErrorName } from "./errors";
 import Base from "./uploader/base";
-import Folder from "./uploader/folder";
 import Local from "./uploader/local";
 import { Pool } from "./utils/pool";
 
@@ -39,7 +38,7 @@ export default class UploadManager {
 
     dispatchUploader(task: Task): Base {
         if (task.type == TaskType.folder) {
-            return new Folder(task, this);
+            //return new Folder(task, this);
         }
 
         switch (task.policy.type) {

src/component/Uploader/core/types.ts

@@ -9,7 +9,6 @@ export interface Policy {
     allowedSuffix: Nullable<string[]>;
     maxSize: number;
     type: PolicyType;
-    chunkSize: number;
 }
 
 export enum TaskType {
@@ -24,6 +23,7 @@ export interface Task {
     dst: string;
     file: File;
     child?: Task[];
+    session?: UploadCredential;
 }
 
 type Nullable<T> = T | null;
@@ -45,4 +45,6 @@ export interface UploadSessionRequest {
 
 export interface UploadCredential {
     sessionID: string;
+    expires: number;
+    chunk_size: number;
 }

src/component/Uploader/core/uploader/base.ts

@@ -1,17 +1,17 @@
 // 所有 Uploader 的基类
-import { Task, UploadCredential, UploadSessionRequest } from "../types";
+import { Task } from "../types";
 import UploadManager from "../index";
 import Logger from "../logger";
 import { validate } from "../utils/validator";
-import { CancelToken, requestAPI } from "../utils/request";
+import { CancelToken } from "../utils/request";
 import { CancelTokenSource } from "axios";
-import { CreateUploadSessionError } from "../errors";
+import { createUploadSession } from "../api";
 
 export enum Status {
     added,
     initialized,
-    preparing,
     queued,
+    preparing,
     processing,
     finishing,
     finished,
@@ -22,7 +22,19 @@ export enum Status {
 export interface UploadHandlers {
     onTransition: (newStatus: Status) => void;
     onError: (err: Error) => void;
-    onProgress: () => void;
+    onProgress: (data: UploadProgress) => void;
+}
+
+export interface UploadProgress {
+    total: ProgressCompose;
+    chunks?: ProgressCompose[];
+}
+
+export interface ProgressCompose {
+    size: number;
+    loaded: number;
+    percent: number;
+    fromCache?: boolean;
 }
 
 export default abstract class Base {
@@ -35,9 +47,9 @@ export default abstract class Base {
 
     protected logger: Logger;
     protected subscriber: UploadHandlers;
-
     // 用于取消请求
-    private cancelToken: CancelTokenSource = CancelToken.source();
+    protected cancelToken: CancelTokenSource = CancelToken.source();
+    protected progress: UploadProgress;
 
     constructor(public task: Task, protected manager: UploadManager) {
         this.logger = new Logger(
@@ -51,7 +63,7 @@ export default abstract class Base {
             /* eslint-disable @typescript-eslint/no-empty-function */
             onTransition: (newStatus: Status) => {},
             onError: (err: Error) => {},
-            onProgress: () => {},
+            onProgress: (data: UploadProgress) => {},
             /* eslint-enable @typescript-eslint/no-empty-function */
         };
     }
@@ -67,25 +79,38 @@ export default abstract class Base {
         try {
             validate(this.task.file, this.task.policy);
         } catch (e) {
-            this.logger.info("File validate failed with error:", e);
+            this.logger.error("File validate failed with error:", e);
             this.setError(e);
             return;
         }
 
         this.logger.info("Enqueued in manager pool");
+        this.transit(Status.queued);
         this.manager.pool.enqueue(this).catch((e) => {
             this.logger.info("Upload task failed with error:", e);
+            // TODO: delete upload session
             this.setError(e);
         });
     };
 
-    public upload = async () => {
+    public run = async () => {
         this.logger.info("Start upload task, create upload session...");
         this.transit(Status.preparing);
-        const uploadSession = await this.createUploadSession();
-        console.log(uploadSession);
+        this.task.session = await createUploadSession({
+            path: this.task.dst,
+            size: this.task.file.size,
+            name: this.task.file.name,
+            policy_id: this.task.policy.id,
+            last_modified: this.task.file.lastModified,
+        });
+        this.logger.info("Upload session created:", this.task.session);
+
+        this.transit(Status.processing);
+        await this.upload();
     };
 
+    public abstract async upload(): Promise<any>;
+
     public cancel = async () => {
         this.cancelToken.cancel();
         // TODO: delete upload session
@@ -102,25 +127,16 @@ export default abstract class Base {
         this.subscriber.onTransition(status);
     }
 
-    private async createUploadSession(): Promise<UploadCredential> {
-        const req: UploadSessionRequest = {
-            path: this.task.dst,
-            size: this.task.file.size,
-            name: this.task.file.name,
-            policy_id: this.task.policy.id,
-            last_modified: this.task.file.lastModified,
+    public getProgressInfoItem(
+        loaded: number,
+        size: number,
+        fromCache?: boolean
+    ): ProgressCompose {
+        return {
+            size,
+            loaded,
+            percent: (loaded / size) * 100,
+            ...(fromCache == null ? {} : { fromCache }),
         };
-
-        const res = await requestAPI<UploadCredential>("file/upload/session", {
-            method: "put",
-            cancelToken: this.cancelToken.token,
-            data: req,
-        });
-
-        if (res.data.code !== 0) {
-            throw new CreateUploadSessionError(res.data);
-        }
-
-        return res.data.data;
     }
 }

src/component/Uploader/core/uploader/chunk.ts

@@ -0,0 +1,44 @@
+import Base from "./base";
+import * as utils from "../utils";
+
+export interface ChunkLoaded {
+    chunks: number[];
+}
+
+export default class Chunk extends Base {
+    protected chunks: Blob[];
+    protected loaded: ChunkLoaded;
+
+    public upload = async () => {
+        this.logger.info("Starting uploading file chunks.");
+        this.initBeforeUploadChunks();
+    };
+
+    private initBeforeUploadChunks() {
+        this.chunks = utils.getChunks(
+            this.task.file,
+            this.task.session?.chunk_size
+        );
+        this.loaded = {
+            chunks: this.chunks.map(() => 0),
+        };
+        this.notifyResumeProgress();
+    }
+
+    private notifyResumeProgress() {
+        this.progress = {
+            total: this.getProgressInfoItem(
+                utils.sum(this.loaded.chunks),
+                this.task.file.size + 1 // 防止在 complete 未调用的时候进度显示 100%
+            ),
+            chunks: this.chunks.map((chunk, index) => {
+                return this.getProgressInfoItem(
+                    this.loaded.chunks[index],
+                    chunk.size,
+                    false
+                );
+            }),
+        };
+        this.subscriber.onProgress(this.progress);
+    }
+}

src/component/Uploader/core/uploader/folder.ts

@@ -1,3 +0,0 @@
-import Base from "./base";
-
-export default class Folder extends Base {}

src/component/Uploader/core/uploader/local.ts

@@ -1,3 +1,3 @@
-import Base from "./base";
+import Chunk from "./chunk";
 
-export default class Local extends Base {}
+export default class Local extends Chunk {}

src/component/Uploader/core/utils/helper.ts

@@ -0,0 +1,33 @@
+export const sizeToString = (bytes: number): string => {
+    if (bytes === 0) return "0 B";
+    const k = 1024;
+    const sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return (bytes / Math.pow(k, i)).toFixed(1) + " " + sizes[i];
+};
+
+// 文件分块
+export function getChunks(
+    file: File,
+    chunkByteSize: number | undefined
+): Blob[] {
+    // 如果 chunkByteSize 比文件大或为0，则直接取文件的大小
+    if (!chunkByteSize || chunkByteSize > file.size || chunkByteSize == 0) {
+        chunkByteSize = file.size;
+    }
+
+    const chunks: Blob[] = [];
+    const count = Math.ceil(file.size / chunkByteSize);
+    for (let i = 0; i < count; i++) {
+        const chunk = file.slice(
+            chunkByteSize * i,
+            i === count - 1 ? file.size : chunkByteSize * (i + 1)
+        );
+        chunks.push(chunk);
+    }
+    return chunks;
+}
+
+export function sum(list: number[]) {
+    return list.reduce((data, loaded) => data + loaded, 0);
+}

src/component/Uploader/core/utils/index.ts

@@ -1,7 +1,4 @@
-export const sizeToString = (bytes: number): string => {
-    if (bytes === 0) return "0 B";
-    const k = 1024;
-    const sizes = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
-    const i = Math.floor(Math.log(bytes) / Math.log(k));
-    return (bytes / Math.pow(k, i)).toFixed(1) + " " + sizes[i];
-};
+export * from "./pool";
+export * from "./helper";
+export * from "./validator";
+export * from "./request";

src/component/Uploader/core/utils/pool.ts

@@ -31,7 +31,7 @@ export class Pool {
     run(item: QueueContent) {
         this.queue = this.queue.filter((v) => v !== item);
         this.processing.push(item);
-        item.uploader.upload().then(
+        item.uploader.run().then(
             () => {
                 item.resolve();
                 this.release(item);

tsconfig.json

@@ -15,7 +15,7 @@
     "jsx": "react",
     "removeComments": true,
     "strictFunctionTypes": true,
-    "strictPropertyInitialization": true,
+    "strictPropertyInitialization": false,
     "strictBindCallApply": true,
     "noImplicitReturns": true,
     "alwaysStrict": true,