feat: minio client management (env, singleton, ...) (#115)

* feat: minio client management (env, singleton, ...)

* feat: bucket file upload handler + crud operations for minio bucket

* feat: restrict bucket(-file) creation to robot/user

* feat: remove bucket(-file) in minio before db entity deletion

* fix: bucket-file upload handler should be mounted on POST http method

packages/server-storage/package.json

@@ -21,10 +21,11 @@
         "dotenv": "^16.4.4",
         "envix": "^1.3.0",
         "hapic": "^2.5.0",
+        "minio": "^7.1.3",
         "redis-extension": "^1.3.0",
-        "singa": "^1.0.0",
         "routup": "^3.2.0",
-        "typeorm":  "^0.3.20",
+        "singa": "^1.0.0",
+        "typeorm": "^0.3.20",
         "typeorm-extension": "^3.5.0",
         "winston": "^3.11.0"
     },

packages/server-storage/src/config/env/module.ts

@@ -33,6 +33,7 @@ export function useEnv(key?: string) : any {
         env: read('NODE_ENV', 'development') as `${EnvironmentName}`,
         port: readInt('PORT', 3000),
         redisConnectionString: read('REDIS_CONNECTION_STRING', null),
+        minioConnectionString: read('MINIO_CONNECTION_STRING', 'http://admin:[email protected]:9000'),
         vaultConnectionString: read('VAULT_CONNECTION_STRING', 'start123@http://127.0.0.1:8090/v1/'),
 
         authupApiURL: read('AUTHUP_API_URL', 'http://127.0.0.1:3010/'),

packages/server-storage/src/config/env/type.ts

@@ -12,6 +12,7 @@ export interface Environment {
     port: number,
 
     redisConnectionString: string,
+    minioConnectionString: string,
     vaultConnectionString: string,
 
     authupApiURL: string,

packages/server-storage/src/config/module.ts

@@ -5,10 +5,16 @@
  * view the LICENSE file that was distributed with this source code.
  */
 
-import { setupAuthup, setupDatabase, setupRedis } from './services';
+import {
+    setupAuthup,
+    setupDatabase,
+    setupMinio,
+    setupRedis,
+} from './services';
 
 export async function setup() {
     await setupDatabase();
     setupRedis();
+    setupMinio();
     setupAuthup();
 }

packages/server-storage/src/config/services/index.ts

@@ -6,5 +6,6 @@
  */
 
 export * from './authup';
+export * from './minio';
 export * from './database';
 export * from './redis';

packages/server-storage/src/config/services/minio.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import { parseProxyConnectionString } from '@privateaim/core';
+import type { ClientOptions } from 'minio';
+import { Client } from 'minio';
+import { setMinioFactory } from '../../core';
+import { useEnv } from '../env';
+
+export function setupMinio() {
+    const connectionString = useEnv('minioConnectionString');
+    const connectionConfig = parseProxyConnectionString(connectionString);
+
+    setMinioFactory(() => {
+        const options : ClientOptions = {
+            endPoint: connectionConfig.host,
+            port: connectionConfig.port,
+            useSSL: false,
+            accessKey: connectionConfig.auth.username,
+            secretKey: connectionConfig.auth.password,
+        };
+
+        return new Client(options);
+    });
+}

packages/server-storage/src/core/index.ts

@@ -6,4 +6,6 @@
  */
 
 export * from './authup';
+export * from './minio';
 export * from './redis';
+export * from './utils';

packages/server-storage/src/core/minio/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './module';

packages/server-storage/src/core/minio/module.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import type { Client } from 'minio';
+import type { Factory } from 'singa';
+import { singa } from 'singa';
+
+const singleton = singa<Client>({
+    name: 'redis',
+});
+
+export function useMinio() {
+    return singleton.use();
+}
+
+export function hasMinio() {
+    return singleton.has() || singleton.hasFactory();
+}
+
+export function setMinioFactory(factory: Factory<Client>) {
+    return singleton.setFactory(factory);
+}

packages/server-storage/src/core/utils/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './stream-to-buffer';

packages/server-storage/src/core/utils/stream-to-buffer.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2022-2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import type { Stream } from 'node:stream';
+
+export async function streamToBuffer(stream: Stream): Promise<Buffer> {
+    return new Promise <Buffer>((resolve, reject) => {
+        const parts : Buffer[] = [];
+
+        stream.on('data', (chunk) => parts.push(chunk));
+        stream.on('end', () => resolve(Buffer.concat(parts)));
+        stream.on('error', (err) => reject(err));
+    });
+}

packages/server-storage/src/domains/actor/constants.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export enum ActorType {
+    USER = 'user',
+    ROBOT = 'robot',
+}

packages/server-storage/src/domains/actor/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+export * from './constants';
+export * from './type';
+export * from './utils';

packages/server-storage/src/domains/actor/type.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import type { ActorType } from './constants';
+
+export type Actor = {
+    id: string,
+    type: `${ActorType}`
+};

packages/server-storage/src/domains/actor/utils.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2024.
+ * Author Peter Placzek (tada5hi)
+ * For the full copyright and license information,
+ * view the LICENSE file that was distributed with this source code.
+ */
+
+import type { Request } from 'routup';
+import { useRequestEnv } from '../../http/request';
+import { ActorType } from './constants';
+import type { Actor } from './type';
+
+export function getActorFromRequest(req: Request) : Actor | undefined {
+    const userId = useRequestEnv(req, 'userId');
+    if (userId) {
+        return { type: ActorType.USER, id: userId };
+    }
+
+    const robotId = useRequestEnv(req, 'robotId');
+    if (robotId) {
+        return { type: ActorType.ROBOT, id: robotId };
+    }
+
+    return undefined;
+}

packages/server-storage/src/domains/bucket-file/entity.ts

@@ -42,14 +42,16 @@ export class BucketFileEntity {
 
     // ------------------------------------------------------------------
 
-    @Column({ type: 'uuid', nullable: true })
-        user_id: User['id'] | null;
+    @Column({ type: 'varchar', length: 64 })
+        actor_type: string;
 
-    @Column({ type: 'uuid', nullable: true })
-        realm_id: Realm['id'] | null;
+    @Column({ type: 'uuid' })
+        actor_id: string;
+
+    // ------------------------------------------------------------------
 
     @Column({ type: 'uuid', nullable: true })
-        robot_id: Robot['id'] | null;
+        realm_id: Realm['id'] | null;
 
     // ------------------------------------------------------------------
 

packages/server-storage/src/domains/bucket/entity.ts

@@ -41,12 +41,14 @@ export class BucketEntity {
 
     // ------------------------------------------------------------------
 
-    @Column({ type: 'uuid', nullable: true })
-        user_id: User['id'] | null;
+    @Column({ type: 'uuid' })
+        actor_id: string;
 
-    @Column({ type: 'uuid', nullable: true })
-        realm_id: Realm['id'] | null;
+    @Column({ type: 'varchar', length: 64 })
+        actor_type: string;
+
+    // ------------------------------------------------------------------
 
     @Column({ type: 'uuid', nullable: true })
-        robot_id: Robot['id'] | null;
+        realm_id: Realm['id'] | null;
 }

packages/server-storage/src/domains/index.ts

@@ -5,5 +5,6 @@
  * view the LICENSE file that was distributed with this source code.
  */
 
+export * from './actor';
 export * from './bucket';
 export * from './bucket-file';

packages/server-storage/src/http/controllers/bucket-file/handlers/delete.ts

@@ -11,6 +11,7 @@ import { isRealmResourceWritable } from '@authup/core';
 import type { Request, Response } from 'routup';
 import { sendAccepted, useRequestParam } from 'routup';
 import { useDataSource } from 'typeorm-extension';
+import { useMinio } from '../../../../core';
 import { BucketFileEntity } from '../../../../domains';
 import { useRequestEnv } from '../../../request';
 
@@ -24,7 +25,12 @@ export async function executeBucketFileRouteDeleteHandler(req: Request, res: Res
 
     const dataSource = await useDataSource();
     const repository = dataSource.getRepository(BucketFileEntity);
-    const entity = await repository.findOneBy({ id });
+    const entity = await repository.findOne({
+        where: {
+            id,
+        },
+        relations: ['bucket'],
+    });
 
     if (!entity) {
         throw new NotFoundError();
@@ -34,6 +40,9 @@ export async function executeBucketFileRouteDeleteHandler(req: Request, res: Res
         throw new ForbiddenError();
     }
 
+    const minio = useMinio();
+    await minio.removeObject(entity.bucket.name, entity.hash);
+
     const { id: entityId } = entity;
 
     await repository.remove(entity);

packages/server-storage/src/http/controllers/bucket-file/handlers/read.ts

@@ -35,8 +35,8 @@ export async function executeBucketFileRouteGetOneHandler(req: Request, res: Res
                 'created_at',
                 'updated_at',
                 'realm_id',
-                'robot_id',
-                'user_id',
+                'actor_type',
+                'actor_id',
             ],
         },
         relations: {
@@ -71,15 +71,15 @@ export async function executeBucketFileRouteGetManyHandler(req: Request, res: Re
                 'created_at',
                 'updated_at',
                 'realm_id',
-                'robot_id',
-                'user_id',
+                'actor_type',
+                'actor_id',
             ],
         },
         relations: {
             allowed: ['bucket'],
         },
         filters: {
-            allowed: ['id', 'name', 'directory', 'realm_id', 'user_id', 'robot_id'],
+            allowed: ['id', 'name', 'directory', 'realm_id', 'actor_type', 'actor_id'],
         },
         pagination: {
             maxLimit: 50,

packages/server-storage/src/http/controllers/bucket/handlers/create.ts

@@ -10,8 +10,9 @@ import { ForbiddenError } from '@ebec/http';
 import type { Request, Response } from 'routup';
 import { sendCreated } from 'routup';
 import { useDataSource } from 'typeorm-extension';
+import { useMinio } from '../../../../core';
 import { useRequestEnv } from '../../../request';
-import { BucketEntity } from '../../../../domains';
+import { BucketEntity, getActorFromRequest } from '../../../../domains';
 import { runProjectValidation } from '../utils/validation';
 
 export async function executeBucketRouteCreateHandler(req: Request, res: Response) : Promise<any> {
@@ -20,17 +21,25 @@ export async function executeBucketRouteCreateHandler(req: Request, res: Respons
         throw new ForbiddenError();
     }
 
+    const actor = getActorFromRequest(req);
+    if (!actor) {
+        throw new ForbiddenError('Only users and robots are permitted to create a bucket.');
+    }
+
     const result = await runProjectValidation(req, 'create');
 
     const dataSource = await useDataSource();
     const repository = dataSource.getRepository(BucketEntity);
     const entity = repository.create({
-        user_id: useRequestEnv(req, 'userId'),
-        robot_id: useRequestEnv(req, 'robotId'),
+        actor_id: actor.id,
+        actor_type: actor.type,
         ...result.data,
     });
 
     await repository.save(entity);
 
+    const minio = useMinio();
+    await minio.makeBucket(entity.name, entity.region);
+
     return sendCreated(res, entity);
 }