backend: Improve S3 structure adding a parent directory

backend/src/config.ts

@@ -33,8 +33,12 @@ export const LIVEKIT_URL_PRIVATE = process.env.LIVEKIT_URL_PRIVATE || LIVEKIT_UR
 export const LIVEKIT_API_KEY = process.env.LIVEKIT_API_KEY || 'devkey';
 export const LIVEKIT_API_SECRET = process.env.LIVEKIT_API_SECRET || 'secret';
 
-// S3 configuration
+/* S3 configuration */
 export const CALL_S3_BUCKET = process.env.CALL_S3_BUCKET || 'openvidu';
+// Parent directory inside the bucket
+export const CALL_S3_PARENT_DIRECTORY = process.env.CALL_S3_PARENT_DIRECTORY || 'openvidu-call';
+// Recording directory inside the parent directory
+export const CALL_S3_RECORDING_DIRECTORY = process.env.CALL_S3_RECORDING_FOLDER || 'recordings';
 export const CALL_S3_SERVICE_ENDPOINT = process.env.CALL_S3_SERVICE_ENDPOINT || undefined;
 export const CALL_S3_ACCESS_KEY = process.env.CALL_S3_ACCESS_KEY || undefined;
 export const CALL_S3_SECRET_KEY = process.env.CALL_S3_SECRET_KEY || undefined;

backend/src/server.ts

@@ -22,7 +22,9 @@ import {
 	CALL_AWS_REGION,
 	CALL_LOG_LEVEL,
 	CALL_NAME_ID,
-	SERVER_CORS_ORIGIN
+	SERVER_CORS_ORIGIN,
+	CALL_S3_PARENT_DIRECTORY,
+	CALL_S3_RECORDING_DIRECTORY
 } from './config.js';
 
 const createApp = () => {
@@ -85,6 +87,8 @@ const logEnvVars = () => {
 	console.log('S3 Configuration');
 	console.log('---------------------------------------------------------');
 	console.log('CALL S3 BUCKET:', text(CALL_S3_BUCKET));
+	console.log('CALL S3 DIRECTORY:', text(CALL_S3_PARENT_DIRECTORY));
+	console.log('CALL S3 RECORDING DIRECTORY:', text(`${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}`));
 
 	// S3 configuration
 	if (CALL_S3_SERVICE_ENDPOINT) {
@@ -121,17 +125,17 @@ const startServer = (app: express.Application) => {
  * @returns {boolean} True if this module is the main entry point, false otherwise.
  */
 const isMainModule = (): boolean => {
-    const importMetaUrl = import.meta.url;
-    let processArgv1 = process.argv[1];
-
-    if (process.platform === "win32") {
-        processArgv1 = processArgv1.replace(/\\/g, "/");
-        processArgv1 = `file:///${processArgv1}`;
-    } else {
-        processArgv1 = `file://${processArgv1}`;
-    }
+	const importMetaUrl = import.meta.url;
+	let processArgv1 = process.argv[1];
+
+	if (process.platform === 'win32') {
+		processArgv1 = processArgv1.replace(/\\/g, '/');
+		processArgv1 = `file:///${processArgv1}`;
+	} else {
+		processArgv1 = `file://${processArgv1}`;
+	}
 
-    return importMetaUrl === processArgv1;
+	return importMetaUrl === processArgv1;
 };
 
 if (isMainModule()) {

backend/src/services/recording.service.ts

@@ -19,7 +19,7 @@ import { DataTopic } from '../models/signal.model.js';
 import { LoggerService } from './logger.service.js';
 import { RecordingInfo, RecordingStatus } from '../models/recording.model.js';
 import { RecordingHelper } from '../helpers/recording.helper.js';
-import { CALL_S3_BUCKET } from '../config.js';
+import { CALL_S3_BUCKET, CALL_S3_PARENT_DIRECTORY, CALL_S3_RECORDING_DIRECTORY } from '../config.js';
 import { RoomService } from './room.service.js';
 
 export class RecordingService {
@@ -104,7 +104,8 @@ export class RecordingService {
 	async deleteRecording(egressId: string, isRequestedByAdmin: boolean): Promise<RecordingInfo> {
 		try {
 			// Get the recording object from the S3 bucket
-			const metadataObject = await this.s3Service.listObjects('.metadata', `.*${egressId}.*.json`);
+			const directory = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/.metadata`;
+			const metadataObject = await this.s3Service.listObjects(directory, `.*${egressId}.*.json`);
 
 			if (!metadataObject.Contents || metadataObject.Contents.length === 0) {
 				throw errorRecordingNotFound(egressId);
@@ -117,14 +118,16 @@ export class RecordingService {
 				throw errorRecordingNotStopped(egressId);
 			}
 
-			const recordingPath = RecordingHelper.extractFilename(recordingInfo);
+			const recordingFilename = RecordingHelper.extractFilename(recordingInfo);
 
-			if (!recordingPath) throw internalError(`Error extracting path from recording ${egressId}`);
+			if (!recordingFilename) throw internalError(`Error extracting path from recording ${egressId}`);
 
-			this.logger.info(`Deleting recording from S3 ${recordingPath}`);
+			const recordingPath = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/${recordingFilename}`;
 
 			await Promise.all([this.s3Service.deleteObject(metadataPath!), this.s3Service.deleteObject(recordingPath)]);
 
+			this.logger.info(`Recording ${egressId} deleted successfully`);
+
 			if (!isRequestedByAdmin) {
 				const signalOptions: SendDataOptions = {
 					destinationSids: [],
@@ -146,7 +149,8 @@ export class RecordingService {
 	 */
 	async getAllRecordings(): Promise<{ recordingInfo: RecordingInfo[]; continuationToken?: string }> {
 		try {
-			const allEgress = await this.s3Service.listObjects('.metadata', '.json');
+			const directory = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/.metadata`;
+			const allEgress = await this.s3Service.listObjects(directory, '.json');
 			const promises: Promise<RecordingInfo>[] = [];
 
 			allEgress.Contents?.forEach((item) => {
@@ -177,7 +181,8 @@ export class RecordingService {
 			const roomIdSanitized = this.sanitizeRegExp(roomId);
 			// Match the room name and room ID in any order
 			const regexPattern = `${roomNameSanitized}.*${roomIdSanitized}|${roomIdSanitized}.*${roomNameSanitized}\\.json`;
-			const metadatagObject = await this.s3Service.listObjects('.metadata', regexPattern);
+			const directory = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/.metadata`;
+			const metadatagObject = await this.s3Service.listObjects(directory, regexPattern);
 
 			if (!metadatagObject.Contents || metadatagObject.Contents.length === 0) {
 				this.logger.verbose(`No recordings found for room ${roomName}. Returning an empty array.`);
@@ -199,7 +204,8 @@ export class RecordingService {
 	private async getRecording(egressId: string): Promise<RecordingInfo> {
 		const egressIdSanitized = this.sanitizeRegExp(egressId);
 		const regexPattern = `.*${egressIdSanitized}.*\\.json`;
-		const metadataObject = await this.s3Service.listObjects('.metadata', regexPattern);
+		const directory = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/.metadata`;
+		const metadataObject = await this.s3Service.listObjects(directory, regexPattern);
 
 		if (!metadataObject.Contents || metadataObject.Contents.length === 0) {
 			throw errorRecordingNotFound(egressId);
@@ -216,10 +222,11 @@ export class RecordingService {
 	): Promise<{ fileSize: number | undefined; fileStream: Readable; start?: number; end?: number }> {
 		const RECORDING_FILE_PORTION_SIZE = 5 * 1024 * 1024; // 5MB
 		const recordingInfo: RecordingInfo = await this.getRecording(recordingId);
-		const recordingPath = RecordingHelper.extractFilename(recordingInfo);
+		const recordingFilename = RecordingHelper.extractFilename(recordingInfo);
 
-		if (!recordingPath) throw new Error(`Error extracting path from recording ${recordingId}`);
+		if (!recordingFilename) throw new Error(`Error extracting path from recording ${recordingId}`);
 
+		const recordingPath = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/${recordingFilename}`;
 		const data = await this.s3Service.getHeaderObject(recordingPath);
 		const fileSize = data.ContentLength;
 
@@ -257,7 +264,7 @@ export class RecordingService {
 	 */
 	private generateFileOutputFromRequest(recordingId: string): EncodedFileOutput {
 		// Added unique identifier to the file path for avoiding overwriting
-		const filepath = `${recordingId}/${recordingId}-${Date.now()}`;
+		const filepath = `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/${recordingId}/${recordingId}-${Date.now()}`;
 
 		return new EncodedFileOutput({
 			fileType: EncodedFileType.DEFAULT_FILETYPE,

backend/src/services/s3.service.ts

@@ -123,7 +123,14 @@ export class S3Service {
 	 */
 	async deleteObject(name: string, bucket: string = CALL_S3_BUCKET): Promise<DeleteObjectCommandOutput> {
 		try {
-			this.logger.info(`Deleting object in S3: ${name}`);
+			const exists = await this.exists(name, bucket);
+
+			if (!exists) {
+				// Force the error to be thrown when the object does not exist
+				throw new Error(`Object '${name}' does not exist in S3`);
+			}
+
+			this.logger.verbose(`Deleting object in S3: ${name}`);
 			const command = new DeleteObjectCommand({ Bucket: bucket, Key: name });
 			return await this.run(command);
 		} catch (error) {
@@ -135,20 +142,20 @@ export class S3Service {
 	/**
 	 * Lists all objects in an S3 bucket with optional subbucket and search pattern filtering.
 	 *
-	 * @param {string} [subbucket=''] - The subbucket within the main bucket to list objects from.
+	 * @param {string} [directory=''] - The subbucket within the main bucket to list objects from.
 	 * @param {string} [searchPattern=''] - A regex pattern to filter the objects by their keys.
 	 * @param {string} [bucket=CALL_S3_BUCKET] - The name of the S3 bucket. Defaults to CALL_S3_BUCKET.
 	 * @param {number} [maxObjects=1000] - The maximum number of objects to retrieve in one request. Defaults to 1000.
 	 * @returns {Promise<ListObjectsV2CommandOutput>} - A promise that resolves to the output of the ListObjectsV2Command.
 	 * @throws {Error} - Throws an error if there is an issue listing the objects.
 	 */
 	async listObjects(
-		subbucket = '',
+		directory = '',
 		searchPattern = '',
 		bucket: string = CALL_S3_BUCKET,
 		maxObjects = 1000
 	): Promise<ListObjectsV2CommandOutput> {
-		const prefix = subbucket ? `${subbucket}/` : '';
+		const prefix = directory ? `${directory}/` : '';
 		let allContents: _Object[] = [];
 		let continuationToken: string | undefined = undefined;
 		let isTruncated = true;

backend/src/services/webhook.service.ts

@@ -5,7 +5,7 @@ import { DataTopic } from '../models/signal.model.js';
 import { LiveKitService } from './livekit.service.js';
 import { BroadcastingInfo, BroadcastingStatus } from '../models/broadcasting.model.js';
 import { RecordingInfo, RecordingStatus } from '../models/recording.model.js';
-import { LIVEKIT_API_KEY, LIVEKIT_API_SECRET } from '../config.js';
+import { CALL_S3_PARENT_DIRECTORY, CALL_S3_RECORDING_DIRECTORY, LIVEKIT_API_KEY, LIVEKIT_API_SECRET } from '../config.js';
 import { LoggerService } from './logger.service.js';
 import { RoomService } from './room.service.js';
 import { S3Service } from './s3.service.js';
@@ -180,6 +180,6 @@ export class WebhookService {
 		const metadataFilename = `${payload.roomName}-${payload.roomId}`;
 		const recordingFilename = payload.filename?.split('.')[0];
 		const egressId = payload.id;
-		return `.metadata/${metadataFilename}/${recordingFilename}_${egressId}.json`;
+		return `${CALL_S3_PARENT_DIRECTORY}/${CALL_S3_RECORDING_DIRECTORY}/.metadata/${metadataFilename}/${recordingFilename}_${egressId}.json`;
 	}
 }