From 6c20818d042ae8a328eed795f72dc5bf23b9e9ee Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:01:49 +0200 Subject: [PATCH 1/6] Add basic storage info to Upload model --- packages/file-store/index.ts | 1 + packages/s3-store/index.ts | 4 ++++ packages/utils/src/models/DataStore.ts | 2 +- packages/utils/src/models/Upload.ts | 9 ++++++++- test/stores.test.ts | 2 ++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/file-store/index.ts b/packages/file-store/index.ts index 83b05871..0d00b632 100644 --- a/packages/file-store/index.ts +++ b/packages/file-store/index.ts @@ -164,6 +164,7 @@ export class FileStore extends DataStore { offset: stats.size, metadata: file.metadata, creation_date: file.creation_date, + storage: {type: 'file', path: file_path}, }) ) }) diff --git a/packages/s3-store/index.ts b/packages/s3-store/index.ts index 5a5c9023..88b2b632 100644 --- a/packages/s3-store/index.ts +++ b/packages/s3-store/index.ts @@ -188,6 +188,7 @@ export class S3Store extends DataStore { offset: Number.parseInt(file.offset, 10), metadata: file.metadata, creation_date: file.creation_date, + storage: file.storage, }), } await this.cache.set(id, metadata) @@ -530,6 +531,7 @@ export class S3Store extends DataStore { upload.creation_date = new Date().toISOString() const res = await this.client.createMultipartUpload(request) + upload.storage = {type: 's3', path: res.Key as string, bucket: this.bucket} await this.saveMetadata(upload, res.UploadId as string) log(`[${upload.id}] multipart upload created (${res.UploadId})`) @@ -614,6 +616,7 @@ export class S3Store extends DataStore { offset: metadata.file.size as number, size: metadata.file.size, metadata: metadata.file.metadata, + storage: metadata.file.storage, }) } @@ -627,6 +630,7 @@ export class S3Store extends DataStore { ...metadata.file, offset: offset + (incompletePartSize ?? 0), size: metadata.file.size, + storage: metadata.file.storage, }) } diff --git a/packages/utils/src/models/DataStore.ts b/packages/utils/src/models/DataStore.ts index c33c51ca..fb05b614 100644 --- a/packages/utils/src/models/DataStore.ts +++ b/packages/utils/src/models/DataStore.ts @@ -50,7 +50,7 @@ export class DataStore extends EventEmitter { * the upload. */ async getUpload(id: string): Promise { - return new Upload({id, size: 0, offset: 0}) + return new Upload({id, size: 0, offset: 0, storage: {type: 'datastore', path: ''}}) } /** diff --git a/packages/utils/src/models/Upload.ts b/packages/utils/src/models/Upload.ts index 1b4706b6..49a6447c 100644 --- a/packages/utils/src/models/Upload.ts +++ b/packages/utils/src/models/Upload.ts @@ -3,15 +3,21 @@ type TUpload = { size?: number offset: number metadata?: Record + storage?: { + type: string + path: string + bucket?: string + } creation_date?: string } export class Upload { id: TUpload['id'] metadata: TUpload['metadata'] - size?: TUpload['size'] + size: TUpload['size'] offset: TUpload['offset'] creation_date: TUpload['creation_date'] + storage: TUpload['storage'] constructor(upload: TUpload) { if (!upload.id) { @@ -22,6 +28,7 @@ export class Upload { this.size = upload.size this.offset = upload.offset this.metadata = upload.metadata + this.storage = upload.storage this.creation_date = upload.creation_date ?? new Date().toISOString() } diff --git a/test/stores.test.ts b/test/stores.test.ts index fcdcdb8d..bdf97d30 100644 --- a/test/stores.test.ts +++ b/test/stores.test.ts @@ -42,6 +42,8 @@ export const shouldCreateUploads = function () { it('should resolve to file', async function () { const newFile = await this.datastore.create(file) + assert.ok(newFile.storage.path) + assert.ok(newFile.storage.type) assert.equal(newFile instanceof Upload, true) }) From 83a599150a35501cc57a110a2a969b3f85c6534d Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:11:52 +0200 Subject: [PATCH 2/6] gcs-store --- packages/gcs-store/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/gcs-store/index.ts b/packages/gcs-store/index.ts index 805d0822..9ad9814f 100644 --- a/packages/gcs-store/index.ts +++ b/packages/gcs-store/index.ts @@ -33,6 +33,8 @@ export class GCSStore extends DataStore { const gcs_file = this.bucket.file(file.id) + file.storage = {type: 'gcs', path: file.id, bucket: this.bucket.name} + const options = { metadata: { metadata: { @@ -41,6 +43,7 @@ export class GCSStore extends DataStore { sizeIsDeferred: `${file.sizeIsDeferred}`, offset: file.offset, metadata: JSON.stringify(file.metadata), + storage: JSON.stringify(file.storage), }, }, } @@ -82,6 +85,7 @@ export class GCSStore extends DataStore { sizeIsDeferred: `${upload.sizeIsDeferred}`, offset, metadata: JSON.stringify(upload.metadata), + storage: JSON.stringify(upload.storage), }, }, } @@ -150,6 +154,7 @@ export class GCSStore extends DataStore { size: size ? Number.parseInt(size, 10) : size, offset: Number.parseInt(metadata.size, 10), // `size` is set by GCS metadata: meta ? JSON.parse(meta) : undefined, + storage: {type: 'gcs', path: id, bucket: this.bucket.name}, }) ) }) From e365497e6cb8588d0fc2dd42f12e2cf9318c1816 Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:11:57 +0200 Subject: [PATCH 3/6] fix file-store --- packages/file-store/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/file-store/index.ts b/packages/file-store/index.ts index 0d00b632..163adc2d 100644 --- a/packages/file-store/index.ts +++ b/packages/file-store/index.ts @@ -60,11 +60,14 @@ export class FileStore extends DataStore { */ async create(file: Upload): Promise { const dirs = file.id.split('/').slice(0, -1) + const filePath = path.join(this.directory, file.id) await fsProm.mkdir(path.join(this.directory, ...dirs), {recursive: true}) - await fsProm.writeFile(path.join(this.directory, file.id), '') + await fsProm.writeFile(filePath, '') await this.configstore.set(file.id, file) + file.storage = {type: 'file', path: filePath} + return file } From 9c1762c2ea3e0978576052947b4dda6a68410209 Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:17:54 +0200 Subject: [PATCH 4/6] Fix another test --- packages/server/test/DeleteHandler.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/test/DeleteHandler.test.ts b/packages/server/test/DeleteHandler.test.ts index 4c0fe2d6..c6d024ad 100644 --- a/packages/server/test/DeleteHandler.test.ts +++ b/packages/server/test/DeleteHandler.test.ts @@ -79,6 +79,7 @@ describe('DeleteHandler', () => { creation_date: undefined, offset: 1000, size: 1000, + storage: {type: 'test', path: `${path}/abc`}, }) await assert.rejects(() => handler.send(req, res, context), {status_code: 400}) }) From 318ca6d68d440021b8352a1ae19287d83740cc29 Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:17:57 +0200 Subject: [PATCH 5/6] Add changeset --- .changeset/hungry-games-compete.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changeset/hungry-games-compete.md diff --git a/.changeset/hungry-games-compete.md b/.changeset/hungry-games-compete.md new file mode 100644 index 00000000..09bb2340 --- /dev/null +++ b/.changeset/hungry-games-compete.md @@ -0,0 +1,9 @@ +--- +'@tus/file-store': minor +'@tus/gcs-store': minor +'@tus/s3-store': minor +'@tus/utils': minor +--- + +Add basic storage information to the Upload model. You can now access `upload.storage` +which has `type` (`file`, `s3`, `gcs`), `path`, and when applicable `bucket`. From 1f7343a7f9951217e100ab13e8dde1400d128a7a Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 6 Jun 2024 12:29:40 +0200 Subject: [PATCH 6/6] Also test in onUploadFinish --- packages/server/test/Server.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/test/Server.test.ts b/packages/server/test/Server.test.ts index 129794be..491b7aa3 100644 --- a/packages/server/test/Server.test.ts +++ b/packages/server/test/Server.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable no-throw-literal */ import 'should' @@ -496,7 +497,9 @@ describe('Server', () => { const server = new Server({ path: '/test/output', datastore: new FileStore({directory}), - onUploadFinish() { + onUploadFinish(_, __, upload) { + assert.ok(upload.storage!.path, 'should have storage.path') + assert.ok(upload.storage!.type, 'should have storage.type') throw {body: 'no', status_code: 500} }, })