From cfc4cbbb3df1cfe1a6c56cae7fc1efffd44971d9 Mon Sep 17 00:00:00 2001 From: tada5hi Date: Fri, 14 Jun 2024 15:19:04 +0200 Subject: [PATCH] fix(storage): use primary key as minio bucket name + persist minio storage in docker-compose --- docker-compose.yml | 4 ++++ .../server-storage/src/domains/bucket/utils.ts | 8 ++++++++ .../controllers/bucket-file/handlers/delete.ts | 4 ++-- .../controllers/bucket-file/handlers/stream.ts | 4 ++-- .../http/controllers/bucket/handlers/create.ts | 6 +++--- .../http/controllers/bucket/handlers/delete.ts | 8 +++++--- .../http/controllers/bucket/handlers/stream.ts | 4 ++-- .../http/controllers/bucket/handlers/update.ts | 10 ++++++---- .../http/controllers/bucket/handlers/upload.ts | 6 ++++-- .../test/unit/bucket-name.spec.ts | 18 ++++++++++++++++++ 10 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 packages/server-storage/test/unit/bucket-name.spec.ts diff --git a/docker-compose.yml b/docker-compose.yml index abb0fcc93..a330e1223 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,8 @@ version: '3.9' volumes: mysql: driver: local + minio: + driver: local services: mysql: image: mysql:latest @@ -58,6 +60,8 @@ services: environment: RABBITMQ_DEFAULT_USER: root RABBITMQ_DEFAULT_PASS: start123 + volumes: + - minio:/data networks: hub: vault: diff --git a/packages/server-storage/src/domains/bucket/utils.ts b/packages/server-storage/src/domains/bucket/utils.ts index 640045594..70dd47aab 100644 --- a/packages/server-storage/src/domains/bucket/utils.ts +++ b/packages/server-storage/src/domains/bucket/utils.ts @@ -12,3 +12,11 @@ export function isBucketOwnedByActor(entity: BucketEntity, actor: Actor) { return entity.actor_type === actor.type && entity.actor_id === actor.id; } + +export function toBucketName(input: string) : string { + input = input + .toLowerCase() + .replace(/[^a-z0-9.-]/g, ''); + + return input.slice(0, Math.min(63, input.length)); +} diff --git a/packages/server-storage/src/http/controllers/bucket-file/handlers/delete.ts b/packages/server-storage/src/http/controllers/bucket-file/handlers/delete.ts index 9d1703dba..8cf64d53f 100644 --- a/packages/server-storage/src/http/controllers/bucket-file/handlers/delete.ts +++ b/packages/server-storage/src/http/controllers/bucket-file/handlers/delete.ts @@ -17,7 +17,7 @@ import { BucketFileEntity, getActorFromRequest, isBucketFileOwnedByActor, - isBucketOwnedByActor, + isBucketOwnedByActor, toBucketName, } from '../../../../domains'; export async function executeBucketFileRouteDeleteHandler(req: Request, res: Response) : Promise { @@ -52,7 +52,7 @@ export async function executeBucketFileRouteDeleteHandler(req: Request, res: Res } const minio = useMinio(); - await minio.removeObject(entity.bucket.name, entity.hash); + await minio.removeObject(toBucketName(entity.bucket.id), entity.hash); const { id: entityId } = entity; diff --git a/packages/server-storage/src/http/controllers/bucket-file/handlers/stream.ts b/packages/server-storage/src/http/controllers/bucket-file/handlers/stream.ts index bb4ded403..d49655fe2 100644 --- a/packages/server-storage/src/http/controllers/bucket-file/handlers/stream.ts +++ b/packages/server-storage/src/http/controllers/bucket-file/handlers/stream.ts @@ -11,7 +11,7 @@ import { useRequestParam } from 'routup'; import { useDataSource } from 'typeorm-extension'; import { useMinio } from '../../../../core'; import { - BucketFileEntity, + BucketFileEntity, toBucketName, } from '../../../../domains'; export async function executeBucketFileRouteStreamHandler(req: Request, res: Response) : Promise { @@ -38,7 +38,7 @@ export async function executeBucketFileRouteStreamHandler(req: Request, res: Res // setResponseHeaderAttachment(res, entity.path); const minio = useMinio(); - const stream = await minio.getObject(entity.bucket.name, entity.hash); + const stream = await minio.getObject(toBucketName(entity.bucket.id), entity.hash); stream.pipe(res); } diff --git a/packages/server-storage/src/http/controllers/bucket/handlers/create.ts b/packages/server-storage/src/http/controllers/bucket/handlers/create.ts index 486b64027..70a657d1f 100644 --- a/packages/server-storage/src/http/controllers/bucket/handlers/create.ts +++ b/packages/server-storage/src/http/controllers/bucket/handlers/create.ts @@ -12,7 +12,7 @@ import { sendCreated } from 'routup'; import { useDataSource } from 'typeorm-extension'; import { useRequestEnv } from '@privateaim/server-http-kit'; import { useMinio } from '../../../../core'; -import { BucketEntity, getActorFromRequest } from '../../../../domains'; +import { BucketEntity, getActorFromRequest, toBucketName } from '../../../../domains'; import { runBucketValidation } from '../utils/validation'; export async function executeBucketRouteCreateHandler(req: Request, res: Response) : Promise { @@ -40,9 +40,9 @@ export async function executeBucketRouteCreateHandler(req: Request, res: Respons const minio = useMinio(); if (entity.region) { - await minio.makeBucket(entity.name, entity.region); + await minio.makeBucket(toBucketName(entity.id), entity.region); } else { - await minio.makeBucket(entity.name); + await minio.makeBucket(toBucketName(entity.id)); } return sendCreated(res, entity); diff --git a/packages/server-storage/src/http/controllers/bucket/handlers/delete.ts b/packages/server-storage/src/http/controllers/bucket/handlers/delete.ts index 996b1511a..da1ee3aa3 100644 --- a/packages/server-storage/src/http/controllers/bucket/handlers/delete.ts +++ b/packages/server-storage/src/http/controllers/bucket/handlers/delete.ts @@ -14,7 +14,9 @@ import { sendAccepted, useRequestParam } from 'routup'; import { useDataSource } from 'typeorm-extension'; import { useRequestEnv } from '@privateaim/server-http-kit'; import { useMinio } from '../../../../core'; -import { BucketEntity, getActorFromRequest, isBucketOwnedByActor } from '../../../../domains'; +import { + BucketEntity, getActorFromRequest, isBucketOwnedByActor, toBucketName, +} from '../../../../domains'; export async function executeBucketRouteDeleteHandler(req: Request, res: Response) : Promise { const id = useRequestParam(req, 'id'); @@ -45,10 +47,10 @@ export async function executeBucketRouteDeleteHandler(req: Request, res: Respons } } - const { id: entityId, name: entityName } = entity; + const { id: entityId } = entity; const minio = useMinio(); - await minio.removeBucket(entityName); + await minio.removeBucket(toBucketName(entityId)); await repository.remove(entity); diff --git a/packages/server-storage/src/http/controllers/bucket/handlers/stream.ts b/packages/server-storage/src/http/controllers/bucket/handlers/stream.ts index 672167ca3..eb3ecc9b7 100644 --- a/packages/server-storage/src/http/controllers/bucket/handlers/stream.ts +++ b/packages/server-storage/src/http/controllers/bucket/handlers/stream.ts @@ -14,7 +14,7 @@ import tar from 'tar-stream'; import { useDataSource } from 'typeorm-extension'; import { streamToBuffer, useMinio } from '../../../../core'; import { - BucketEntity, BucketFileEntity, + BucketEntity, BucketFileEntity, toBucketName, } from '../../../../domains'; async function packFile( @@ -87,7 +87,7 @@ export async function executeBucketRouteStreamHandler(req: Request, res: Respons const pack = tar.pack(); pack.pipe(res); - await packFiles(pack, entity.name, files); + await packFiles(pack, toBucketName(entity.id), files); pack.finalize(); } diff --git a/packages/server-storage/src/http/controllers/bucket/handlers/update.ts b/packages/server-storage/src/http/controllers/bucket/handlers/update.ts index 3fb2e93e9..6fc10ca10 100644 --- a/packages/server-storage/src/http/controllers/bucket/handlers/update.ts +++ b/packages/server-storage/src/http/controllers/bucket/handlers/update.ts @@ -14,7 +14,9 @@ import { sendAccepted, useRequestParam } from 'routup'; import { useDataSource } from 'typeorm-extension'; import { useRequestEnv } from '@privateaim/server-http-kit'; import { useMinio } from '../../../../core'; -import { BucketEntity, getActorFromRequest, isBucketOwnedByActor } from '../../../../domains'; +import { + BucketEntity, getActorFromRequest, isBucketOwnedByActor, toBucketName, +} from '../../../../domains'; import { runBucketValidation } from '../utils/validation'; export async function executeBucketRouteUpdateHandler(req: Request, res: Response) : Promise { @@ -55,12 +57,12 @@ export async function executeBucketRouteUpdateHandler(req: Request, res: Respons await repository.save(entity); const minio = useMinio(); - const hasBucket = await minio.bucketExists(entity.name); + const hasBucket = await minio.bucketExists(entity.id); if (!hasBucket) { if (entity.region) { - await minio.makeBucket(entity.name, entity.region); + await minio.makeBucket(toBucketName(entity.id), entity.region); } else { - await minio.makeBucket(entity.name); + await minio.makeBucket(toBucketName(entity.id)); } } diff --git a/packages/server-storage/src/http/controllers/bucket/handlers/upload.ts b/packages/server-storage/src/http/controllers/bucket/handlers/upload.ts index 5d4a27e0a..2b03156ad 100644 --- a/packages/server-storage/src/http/controllers/bucket/handlers/upload.ts +++ b/packages/server-storage/src/http/controllers/bucket/handlers/upload.ts @@ -14,7 +14,9 @@ import type { Request, Response } from 'routup'; import { useDataSource } from 'typeorm-extension'; import { useRequestEnv } from '@privateaim/server-http-kit'; import { streamToBuffer, useMinio } from '../../../../core'; -import { BucketEntity, BucketFileEntity, getActorFromRequest } from '../../../../domains'; +import { + BucketEntity, BucketFileEntity, getActorFromRequest, toBucketName, +} from '../../../../domains'; export async function uploadRequestFiles(req: Request, bucketName: string) { const minio = useMinio(); @@ -104,7 +106,7 @@ export async function executeBucketRouteUploadHandler(req: Request, res: Respons throw new NotFoundError(); } - let files = await uploadRequestFiles(req, entity.name); + let files = await uploadRequestFiles(req, toBucketName(entity.id)); if (files.length > 0) { files = files.map((file) => ({ diff --git a/packages/server-storage/test/unit/bucket-name.spec.ts b/packages/server-storage/test/unit/bucket-name.spec.ts new file mode 100644 index 000000000..be927257d --- /dev/null +++ b/packages/server-storage/test/unit/bucket-name.spec.ts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024. + * Author Peter Placzek (tada5hi) + * For the full copyright and license information, + * view the LICENSE file that was distributed with this source code. + */ + +import { toBucketName } from '../../src/domains'; + +describe('src/domains/bucket', () => { + it('should serialize bucket name', () => { + let output = toBucketName('805735EC-6A00-4368-88CD-83C818806f7A'); + expect(output).toEqual('805735ec-6a00-4368-88cd-83c818806f7a'); + + output = toBucketName('805735ec-6a00-4368-88cd-83c818806f7a-83c818806f7a-83c818806f7a-foo'); + expect(output).toEqual('805735ec-6a00-4368-88cd-83c818806f7a-83c818806f7a-83c818806f7a-'); + }); +});