feat(AWS): Upgrade AWS SDK to v3 (LLC-1464) (#863)

BREAKING CHANGE: Upgraded AWS SDK to v3 and changed app config shape

Author: lucuilearningpool

@types/string-to-stream/index.d.ts

@@ -39,12 +39,13 @@
     "npm": ">3.0.0"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.100.0",
+    "@aws-sdk/lib-storage": "^3.100.0",
     "@azure/storage-blob": "^10.3.0",
     "@google-cloud/storage": "^5.8.1",
     "@learninglocker/xapi-validation": "^2.1.10",
     "accept-language-parser": "^1.5.0",
     "atob": "^2.0.3",
-    "aws-sdk": "^2.205.0",
     "bluebird": "3.7.2",
     "boolean": "^0.2.0",
     "btoa": "^1.1.2",
@@ -55,7 +56,7 @@
     "http-status-codes": "^2.0.0",
     "install": "^0.13.0",
     "ioredis": "^4.14.0",
-    "jscommons": "^3.0.2",
+    "jscommons": "^4.0.0",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.4",
     "mime-types": "2.1.35",
@@ -70,7 +71,6 @@
     "sha1": "^1.1.1",
     "source-map-support": "^0.5.0",
     "stream-to-string": "^1.2.0",
-    "string-to-stream": "^2.0.0",
     "uuid": "^8.0.0"
   },
   "devDependencies": {
@@ -89,7 +89,7 @@
     "@types/mime-types": "2.1.1",
     "@types/mocha": "8.2.3",
     "@types/mongodb": "3.6.18",
-    "@types/node": "9.6.61",
+    "@types/node": "^14.18.18",
     "@types/node-fetch": "2.5.8",
     "@types/object-hash": "1.3.4",
     "@types/redis": "2.8.32",

package.json

@@ -1,4 +1,4 @@
-import { S3 } from 'aws-sdk';
+import { S3ClientConfig } from '@aws-sdk/client-s3';
 import { Redis } from 'ioredis';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
@@ -34,7 +34,7 @@ export default interface AppConfig {
       readonly storageDir: string;
     };
     readonly s3: {
-      readonly awsConfig: S3.ClientConfiguration;
+      readonly awsConfig: S3ClientConfig;
       readonly bucketName: string;
     };
     readonly mongo: {

src/apps/AppConfig.ts

@@ -1,4 +1,4 @@
-import S3 from 'aws-sdk/clients/s3';
+import { S3ClientConfig } from '@aws-sdk/client-s3';
 import Tracker from 'jscommons/dist/tracker/Tracker';
 import { Db } from 'mongodb';
 import { LoggerInstance } from 'winston';
@@ -33,7 +33,7 @@ export default interface AppConfig {
       readonly storageDir: string;
     };
     readonly s3: {
-      readonly awsConfig: S3.ClientConfiguration;
+      readonly awsConfig: S3ClientConfig;
       readonly bucketName: string;
     };
     readonly mongo: {

src/apps/activities/AppConfig.ts

@@ -2,7 +2,7 @@ import { Request, Response } from 'express';
 import { get, mapKeys } from 'lodash';
 import { parse as parseQueryString } from 'query-string';
 import streamToString from 'stream-to-string';
-import stringToStream from 'string-to-stream';
+import { stringToStream } from '../../../../utils/stringToStream';
 import InvalidMethod from '../../errors/InvalidMethod';
 import Config from '../Config';
 import deleteProfileWithService from './deleteProfileWithService';

src/apps/activities/expressPresenter/utils/alternateProfileRequest.ts

@@ -63,7 +63,7 @@ describe('getClient from mongo client', () => {
     await db.collection('lrs').insertOne(TEST_STORE);
     await db.collection('client').insertOne(TEST_CLIENT);
     const result = await authRepo.getClient({ authToken: TEST_TOKEN });
-    assert.equal(result.client._id, TEST_CLIENT._id);
+    assert.strictEqual(result.client._id, TEST_CLIENT._id.toString());
   });
 
   it('should error when getting a untrusted client', async () => {
@@ -130,7 +130,7 @@ describe('getClient from mongo client', () => {
     await db.collection('client').insertOne(TEST_CLIENT);
     await db.collection('oAuthTokens').insertOne(TEST_OAUTH_TOKEN);
     const result = await authRepo.getClient({ authToken: `Bearer ${TEST_ACCESS_TOKEN}` });
-    assert.equal(result.client._id, TEST_CLIENT._id);
+    assert.strictEqual(result.client._id, TEST_CLIENT._id.toString());
   });
 
   it('should error when access_token is not found in collection', async () => {

src/apps/activities/mongoAuthRepo/tests/getClient.test.ts

@@ -1,4 +1,4 @@
-import { S3 } from 'aws-sdk';
+import { S3ClientConfig } from '@aws-sdk/client-s3';
 
 export default interface FactoryConfig {
   readonly factoryName: string;
@@ -18,7 +18,7 @@ export default interface FactoryConfig {
     readonly storageDir: string;
   };
   readonly s3: {
-    readonly awsConfig: S3.ClientConfiguration;
+    readonly awsConfig: S3ClientConfig;
     readonly bucketName: string;
     readonly subFolder: string;
   };

src/apps/activities/repo/storage/FactoryConfig.ts

@@ -1,6 +1,6 @@
 import { ContainerURL, ServiceURL, SharedKeyCredential, StorageURL } from '@azure/storage-blob';
 import { Storage } from '@google-cloud/storage';
-import S3 from 'aws-sdk/clients/s3';
+import { S3Client } from '@aws-sdk/client-s3';
 import azureStorageRepo from '../../azureStorageRepo';
 import googleStorageRepo from '../../googleStorageRepo';
 import localStorageRepo from '../../localStorageRepo';
@@ -13,7 +13,7 @@ export default (factoryConfig: FactoryConfig): Repo => {
     case 's3':
       return s3StorageRepo({
         bucketName: factoryConfig.s3.bucketName,
-        client: new S3(factoryConfig.s3.awsConfig) as any,
+        client: new S3Client(factoryConfig.s3.awsConfig),
         subFolder: factoryConfig.s3.subFolder,
       });
     case 'google':

src/apps/activities/repo/storage/factory.ts

@@ -1,6 +1,6 @@
 import { ContainerURL, ServiceURL, SharedKeyCredential, StorageURL } from '@azure/storage-blob';
 import { Storage } from '@google-cloud/storage';
-import S3 from 'aws-sdk/clients/s3';
+import { S3Client } from '@aws-sdk/client-s3';
 import connectToDb from 'jscommons/dist/mongoRepo/utils/connectToDb';
 import config from '../../../config';
 import logger from '../../../logger';
@@ -54,7 +54,7 @@ const getStorageRepo = (): StorageRepo => {
     case 's3':
       return s3StorageRepo({
         bucketName: config.s3StorageRepo.bucketName,
-        client: new S3(config.s3StorageRepo.awsConfig) as any,
+        client: new S3Client(config.s3StorageRepo.awsConfig),
         subFolder: config.storageSubFolders.activities,
       });
     case 'google':

src/apps/activities/repoFactory/index.ts

@@ -1,3 +1,4 @@
+import { DeleteObjectCommand } from '@aws-sdk/client-s3';
 import DeleteProfileContentOptions from '../repoFactory/options/DeleteProfileContentOptions';
 import getStorageDir from '../utils/getStorageDir';
 import Config from './Config';
@@ -6,11 +7,13 @@ export default (config: Config) => {
   return async (opts: DeleteProfileContentOptions): Promise<void> => {
     const profileDir = getStorageDir({ subfolder: config.subFolder, lrs_id: opts.lrs_id });
     const filePath = `${profileDir}/${opts.key}`;
-    await config.client
-      .deleteObject({
-        Bucket: config.bucketName,
-        Key: filePath,
-      })
-      .promise();
+
+    const deletionCommand = new DeleteObjectCommand({
+      Bucket: config.bucketName,
+      Key: filePath,
+    });
+    await config.client.send(deletionCommand);
+
+    return;
   };
 };