Skip to content

Commit

Permalink
feat: minio client management (env, singleton, ...) (#115)
Browse files Browse the repository at this point in the history
* feat: minio client management (env, singleton, ...)

* feat: bucket file upload handler + crud operations for minio bucket

* feat: restrict bucket(-file) creation to robot/user

* feat: remove bucket(-file) in minio before db entity deletion

* fix: bucket-file upload handler should be mounted on POST http method
  • Loading branch information
tada5hi authored Feb 26, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 89ea187 commit f9689b8
Showing 28 changed files with 358 additions and 29 deletions.
1 change: 1 addition & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions packages/server-storage/package.json
Original file line number Diff line number Diff line change
@@ -21,10 +21,11 @@
"dotenv": "^16.4.4",
"envix": "^1.3.0",
"hapic": "^2.5.0",
"minio": "^7.1.3",
"redis-extension": "^1.3.0",
"singa": "^1.0.0",
"routup": "^3.2.0",
"typeorm": "^0.3.20",
"singa": "^1.0.0",
"typeorm": "^0.3.20",
"typeorm-extension": "^3.5.0",
"winston": "^3.11.0"
},
1 change: 1 addition & 0 deletions packages/server-storage/src/config/env/module.ts
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ export function useEnv(key?: string) : any {
env: read('NODE_ENV', 'development') as `${EnvironmentName}`,
port: readInt('PORT', 3000),
redisConnectionString: read('REDIS_CONNECTION_STRING', null),
minioConnectionString: read('MINIO_CONNECTION_STRING', 'http://admin:[email protected]:9000'),
vaultConnectionString: read('VAULT_CONNECTION_STRING', 'start123@http://127.0.0.1:8090/v1/'),

authupApiURL: read('AUTHUP_API_URL', 'http://127.0.0.1:3010/'),
1 change: 1 addition & 0 deletions packages/server-storage/src/config/env/type.ts
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ export interface Environment {
port: number,

redisConnectionString: string,
minioConnectionString: string,
vaultConnectionString: string,

authupApiURL: string,
8 changes: 7 additions & 1 deletion packages/server-storage/src/config/module.ts
Original file line number Diff line number Diff line change
@@ -5,10 +5,16 @@
* view the LICENSE file that was distributed with this source code.
*/

import { setupAuthup, setupDatabase, setupRedis } from './services';
import {
setupAuthup,
setupDatabase,
setupMinio,
setupRedis,
} from './services';

export async function setup() {
await setupDatabase();
setupRedis();
setupMinio();
setupAuthup();
}
1 change: 1 addition & 0 deletions packages/server-storage/src/config/services/index.ts
Original file line number Diff line number Diff line change
@@ -6,5 +6,6 @@
*/

export * from './authup';
export * from './minio';
export * from './database';
export * from './redis';
29 changes: 29 additions & 0 deletions packages/server-storage/src/config/services/minio.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

import { parseProxyConnectionString } from '@privateaim/core';
import type { ClientOptions } from 'minio';
import { Client } from 'minio';
import { setMinioFactory } from '../../core';
import { useEnv } from '../env';

export function setupMinio() {
const connectionString = useEnv('minioConnectionString');
const connectionConfig = parseProxyConnectionString(connectionString);

setMinioFactory(() => {
const options : ClientOptions = {
endPoint: connectionConfig.host,
port: connectionConfig.port,
useSSL: false,
accessKey: connectionConfig.auth.username,
secretKey: connectionConfig.auth.password,
};

return new Client(options);
});
}
2 changes: 2 additions & 0 deletions packages/server-storage/src/core/index.ts
Original file line number Diff line number Diff line change
@@ -6,4 +6,6 @@
*/

export * from './authup';
export * from './minio';
export * from './redis';
export * from './utils';
8 changes: 8 additions & 0 deletions packages/server-storage/src/core/minio/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

export * from './module';
26 changes: 26 additions & 0 deletions packages/server-storage/src/core/minio/module.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

import type { Client } from 'minio';
import type { Factory } from 'singa';
import { singa } from 'singa';

const singleton = singa<Client>({
name: 'redis',
});

export function useMinio() {
return singleton.use();
}

export function hasMinio() {
return singleton.has() || singleton.hasFactory();
}

export function setMinioFactory(factory: Factory<Client>) {
return singleton.setFactory(factory);
}
8 changes: 8 additions & 0 deletions packages/server-storage/src/core/utils/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

export * from './stream-to-buffer';
18 changes: 18 additions & 0 deletions packages/server-storage/src/core/utils/stream-to-buffer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* Copyright (c) 2022-2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

import type { Stream } from 'node:stream';

export async function streamToBuffer(stream: Stream): Promise<Buffer> {
return new Promise <Buffer>((resolve, reject) => {
const parts : Buffer[] = [];

stream.on('data', (chunk) => parts.push(chunk));
stream.on('end', () => resolve(Buffer.concat(parts)));
stream.on('error', (err) => reject(err));
});
}
11 changes: 11 additions & 0 deletions packages/server-storage/src/domains/actor/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

export enum ActorType {
USER = 'user',
ROBOT = 'robot',
}
10 changes: 10 additions & 0 deletions packages/server-storage/src/domains/actor/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

export * from './constants';
export * from './type';
export * from './utils';
13 changes: 13 additions & 0 deletions packages/server-storage/src/domains/actor/type.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

import type { ActorType } from './constants';

export type Actor = {
id: string,
type: `${ActorType}`
};
25 changes: 25 additions & 0 deletions packages/server-storage/src/domains/actor/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright (c) 2024.
* Author Peter Placzek (tada5hi)
* For the full copyright and license information,
* view the LICENSE file that was distributed with this source code.
*/

import type { Request } from 'routup';
import { useRequestEnv } from '../../http/request';
import { ActorType } from './constants';
import type { Actor } from './type';

export function getActorFromRequest(req: Request) : Actor | undefined {
const userId = useRequestEnv(req, 'userId');
if (userId) {
return { type: ActorType.USER, id: userId };
}

const robotId = useRequestEnv(req, 'robotId');
if (robotId) {
return { type: ActorType.ROBOT, id: robotId };
}

return undefined;
}
12 changes: 7 additions & 5 deletions packages/server-storage/src/domains/bucket-file/entity.ts
Original file line number Diff line number Diff line change
@@ -42,14 +42,16 @@ export class BucketFileEntity {

// ------------------------------------------------------------------

@Column({ type: 'uuid', nullable: true })
user_id: User['id'] | null;
@Column({ type: 'varchar', length: 64 })
actor_type: string;

@Column({ type: 'uuid', nullable: true })
realm_id: Realm['id'] | null;
@Column({ type: 'uuid' })
actor_id: string;

// ------------------------------------------------------------------

@Column({ type: 'uuid', nullable: true })
robot_id: Robot['id'] | null;
realm_id: Realm['id'] | null;

// ------------------------------------------------------------------

12 changes: 7 additions & 5 deletions packages/server-storage/src/domains/bucket/entity.ts
Original file line number Diff line number Diff line change
@@ -41,12 +41,14 @@ export class BucketEntity {

// ------------------------------------------------------------------

@Column({ type: 'uuid', nullable: true })
user_id: User['id'] | null;
@Column({ type: 'uuid' })
actor_id: string;

@Column({ type: 'uuid', nullable: true })
realm_id: Realm['id'] | null;
@Column({ type: 'varchar', length: 64 })
actor_type: string;

// ------------------------------------------------------------------

@Column({ type: 'uuid', nullable: true })
robot_id: Robot['id'] | null;
realm_id: Realm['id'] | null;
}
1 change: 1 addition & 0 deletions packages/server-storage/src/domains/index.ts
Original file line number Diff line number Diff line change
@@ -5,5 +5,6 @@
* view the LICENSE file that was distributed with this source code.
*/

export * from './actor';
export * from './bucket';
export * from './bucket-file';
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ import { isRealmResourceWritable } from '@authup/core';
import type { Request, Response } from 'routup';
import { sendAccepted, useRequestParam } from 'routup';
import { useDataSource } from 'typeorm-extension';
import { useMinio } from '../../../../core';
import { BucketFileEntity } from '../../../../domains';
import { useRequestEnv } from '../../../request';

@@ -24,7 +25,12 @@ export async function executeBucketFileRouteDeleteHandler(req: Request, res: Res

const dataSource = await useDataSource();
const repository = dataSource.getRepository(BucketFileEntity);
const entity = await repository.findOneBy({ id });
const entity = await repository.findOne({
where: {
id,
},
relations: ['bucket'],
});

if (!entity) {
throw new NotFoundError();
@@ -34,6 +40,9 @@ export async function executeBucketFileRouteDeleteHandler(req: Request, res: Res
throw new ForbiddenError();
}

const minio = useMinio();
await minio.removeObject(entity.bucket.name, entity.hash);

const { id: entityId } = entity;

await repository.remove(entity);
Original file line number Diff line number Diff line change
@@ -35,8 +35,8 @@ export async function executeBucketFileRouteGetOneHandler(req: Request, res: Res
'created_at',
'updated_at',
'realm_id',
'robot_id',
'user_id',
'actor_type',
'actor_id',
],
},
relations: {
@@ -71,15 +71,15 @@ export async function executeBucketFileRouteGetManyHandler(req: Request, res: Re
'created_at',
'updated_at',
'realm_id',
'robot_id',
'user_id',
'actor_type',
'actor_id',
],
},
relations: {
allowed: ['bucket'],
},
filters: {
allowed: ['id', 'name', 'directory', 'realm_id', 'user_id', 'robot_id'],
allowed: ['id', 'name', 'directory', 'realm_id', 'actor_type', 'actor_id'],
},
pagination: {
maxLimit: 50,
Original file line number Diff line number Diff line change
@@ -10,8 +10,9 @@ import { ForbiddenError } from '@ebec/http';
import type { Request, Response } from 'routup';
import { sendCreated } from 'routup';
import { useDataSource } from 'typeorm-extension';
import { useMinio } from '../../../../core';
import { useRequestEnv } from '../../../request';
import { BucketEntity } from '../../../../domains';
import { BucketEntity, getActorFromRequest } from '../../../../domains';
import { runProjectValidation } from '../utils/validation';

export async function executeBucketRouteCreateHandler(req: Request, res: Response) : Promise<any> {
@@ -20,17 +21,25 @@ export async function executeBucketRouteCreateHandler(req: Request, res: Respons
throw new ForbiddenError();
}

const actor = getActorFromRequest(req);
if (!actor) {
throw new ForbiddenError('Only users and robots are permitted to create a bucket.');
}

const result = await runProjectValidation(req, 'create');

const dataSource = await useDataSource();
const repository = dataSource.getRepository(BucketEntity);
const entity = repository.create({
user_id: useRequestEnv(req, 'userId'),
robot_id: useRequestEnv(req, 'robotId'),
actor_id: actor.id,
actor_type: actor.type,
...result.data,
});

await repository.save(entity);

const minio = useMinio();
await minio.makeBucket(entity.name, entity.region);

return sendCreated(res, entity);
}
Loading

0 comments on commit f9689b8

Please sign in to comment.