Add file upload api

.env.example

@@ -32,3 +32,15 @@ HSES_DATA_PASSWORD=password
 # In production, Sequelize instance is created with a postgres URI.
 # This URI is automatically dropped into the cloud.gov environment as the env variable DATABASE_URL
 DATABASE_URL=secret
+# Local development variables to support s3 endpoint development and testing
+LOCAL_DEV=true
+# Comment this out if you are not using docker
+S3_ENDPOINT=http://minio:9000
+# Uncomment the following if you aren't using docker for development. Note: This requires minio to be running locally on port 9000
+## and manually creating the bucket specified in $S3_BUCKET
+# S3_ENDPOINT=http://localhost:9000
+S3_BUCKET=ttadp-test
+AWS_SECRET_ACCESS_KEY=EXAMPLESECRETKEY
+AWS_ACCESS_KEY_ID=EXAMPLEID
+MINIO_ROOT_PASSWORD=EXAMPLESECRETKEY
+MINIO_ROOT_USER=EXAMPLEID

README.md

@@ -19,7 +19,7 @@ For the latest on our product mission, goals, initiatives, and KPIs, see the [Pr
 7. Optionally, set `CURRENT_USER` to your current user's uid:gid. This will cause files created by docker compose to be owned by your user instead of root.
 8. Run `yarn docker:db:migrate` to run DB migrations
 9. Run `yarn docker:db:seed` to seed the database with test data.
-10. Run `yarn docker:start` to start the application. The frontend will be available on `localhost:3000` and the backend will run on `localhost:8080`.
+10. Run `yarn docker:start` to start the application. The frontend will be available on `localhost:3000` and the backend will run on `localhost:8080` and minio will run on `localhost:9000`.
 11. Run `yarn docker:stop` to stop the servers and remove the docker containers.
 
 The frontend [proxies requests](https://create-react-app.dev/docs/proxying-api-requests-in-development/) to paths it doesn't recognize to the backend.
@@ -30,9 +30,11 @@ Api documentation uses [Redoc](https://github.com/Redocly/redoc) to serve docume
 
 You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`.
 
+You must also install and run minio locally to use the file upload functionality. Please comment out `S3_ENDPOINT=http://minio:9000` and uncomment `S3_ENDPOINT=http://localhost:9000`
+
 ### Running Tests
 
-Run `yarn docker:deps` to install dependencies. Run `yarn docker:db:migrate` and `yarn docker:test` to run all tests for the frontend and backend.
+Run `yarn docker:deps` to install dependencies. Run `yarn docker:db:migrate`, `yarn docker:db:seed` and `yarn docker:test` to run all tests for the frontend and backend.
 
 ### Docker on Windows
 

docker-compose.yml

@@ -42,5 +42,20 @@ services:
       - "6543:5432"
     volumes:
       - dbdata:/var/lib/postgresql/data
+  minio:
+    image: minio/minio
+    env_file: .env
+    ports:
+      - "9000:9000"
+    volumes:
+      - miniodata:/data
+    command: server /data
+  aws-cli:
+    image: amazon/aws-cli
+    env_file: .env
+    command: ["--endpoint-url", "$S3_ENDPOINT", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"]
+
+
 volumes:
   dbdata: {}
+  miniodata: {}

docs/openapi/index.yaml

@@ -123,6 +123,29 @@ components:
           type: number
         name:
           type: string
+    fileUpload:
+      type: object
+      properties:
+        attachmentType:
+          type: string
+          description: "Type of attachment. Acceptable values are ATTACHMENT or RESOURCE"
+        reportId: 
+          type: number
+          description: "id of the Activity report the file is associated with"
+        File:
+          type: string
+          format: binary
+          description: "File to be uploaded"
+      required:
+      - attachmentType
+      - reportId
+      - File
+    fileResponse:
+      type: object
+      description: ID of file created
+      properties:
+        id:
+          type: number
     permission:
       type: object
       description: A combination of region, scope and user allowing that user to perform an action in a region

docs/openapi/paths/files.yaml

@@ -0,0 +1,20 @@
+post:
+  tags:
+    - files
+  operationId: postFile
+  description: Upload a file to s3
+  requestBody:
+    description: upload a file for an Activity Report
+    required: true
+    content:
+      multipart/form-data:
+        schema:
+          $ref: '../index.yaml#/components/schemas/fileUpload'
+  responses:
+    200:
+      description: One user by an id
+      content:
+        application/json:
+          schema:
+            type: object
+            $ref: '../index.yaml#/components/schemas/fileResponse'

docs/openapi/paths/index.yaml

@@ -20,3 +20,5 @@
   $ref: './activity-reports/activity-reports-id.yaml'
 '/activity-reports/{activityReportId}/submit':
   $ref: './activity-reports/submit.yaml'
+'/files':
+  $ref: './files.yaml'

package.json

@@ -136,6 +136,7 @@
     "@babel/runtime": "^7.12.1",
     "@cucumber/cucumber": "^7.0.0-rc.0",
     "adm-zip": "^0.5.1",
+    "aws-sdk": "^2.826.0",
     "axios": "^0.21.1",
     "chromedriver": "^87.0.0",
     "client-oauth2": "^4.3.3",
@@ -147,9 +148,11 @@
     "express": "^4.17.1",
     "express-unless": "^0.5.0",
     "express-winston": "^4.0.5",
+    "file-type": "^16.1.0",
     "helmet": "^4.1.0",
     "http-codes": "^1.0.0",
     "lodash": "^4.17.20",
+    "multiparty": "^4.2.2",
     "moment": "^2.29.1",
     "mz": "^2.7.0",
     "newrelic": "^7.0.1",
@@ -159,6 +162,7 @@
     "sequelize": "^5.22.3",
     "sequelize-cli": "^6.2.0",
     "url-join": "^4.0.1",
+    "uuid": "^8.3.2",
     "winston": "^3.3.3",
     "xml2json": "^0.12.0",
     "yargs": "^16.1.1"

src/lib/s3Uploader.js

@@ -0,0 +1,45 @@
+import { S3 } from 'aws-sdk';
+
+export const s3 = new S3({
+  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+  endpoint: process.env.S3_ENDPOINT,
+  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  signatureVersion: 'v4',
+  s3ForcePathStyle: true,
+});
+
+export const verifyVersioning = async (bucket = process.env.bucket, s3Client = s3) => {
+  const versioningConfiguration = {
+    MFADelete: 'Disabled',
+    Status: 'Enabled',
+  };
+  let params = {
+    Bucket: bucket,
+  };
+  const data = await s3Client.getBucketVersioning(params);
+  if (!(data) || data.Status !== 'Enabled') {
+    params = {
+      Bucket: bucket,
+      VersioningConfiguration: versioningConfiguration,
+    };
+    return s3Client.putBucketVersioning(params);
+  }
+  return new Promise((resolve) => resolve(data));
+};
+
+const s3Uploader = async (buffer, name, type, s3Client = s3) => {
+  const params = {
+    Body: buffer,
+    Bucket: process.env.S3_BUCKET,
+    ContentType: type.mime,
+    Key: name,
+  };
+  // Only check for versioning if not using Minio
+  if (process.env.LOCAL_DEV !== 'true') {
+    await verifyVersioning();
+  }
+
+  const upload = await s3Client.upload(params);
+  return upload.promise();
+};
+export default s3Uploader;

src/lib/s3Uploader.test.js

@@ -0,0 +1,72 @@
+import { v4 as uuidv4 } from 'uuid';
+import s3Uploader, { verifyVersioning, s3 } from './s3Uploader';
+
+const mockData = {
+  MFADelete: 'Disabled',
+  Status: 'Enabled',
+};
+
+// make sure we save to original value so we can restore it
+const oldEndpoint = process.env.LOCAL_DEV;
+
+describe('s3Uploader.verifyVersioning', () => {
+  let mockGet = jest.spyOn(s3, 'getBucketVersioning').mockImplementation(async () => mockData);
+  const mockPut = jest.spyOn(s3, 'putBucketVersioning').mockImplementation(async (params) => new Promise((res) => res(params)));
+  beforeEach(() => {
+    mockGet.mockClear();
+    mockPut.mockClear();
+  });
+  it('Doesn\'t change things if versioning is enabled', async () => {
+    const got = await verifyVersioning();
+    expect(mockGet.mock.calls.length).toBe(1);
+    expect(mockPut.mock.calls.length).toBe(0);
+    expect(got).toBe(mockData);
+  });
+  it('Enables versioning if it is disabled', async () => {
+    mockGet = jest.spyOn(s3, 'getBucketVersioning').mockImplementationOnce(async () => {});
+    const got = await verifyVersioning();
+    expect(mockGet.mock.calls.length).toBe(1);
+    expect(mockPut.mock.calls.length).toBe(1);
+    expect(got.Bucket).toBe(process.env.S3_BUCKET);
+    expect(got.VersioningConfiguration.MFADelete).toBe(mockData.MFADelete);
+    expect(got.VersioningConfiguration.Status).toBe(mockData.Status);
+  });
+});
+
+describe('s3Uploader', () => {
+  const goodType = { ext: 'pdf', mime: 'application/pdf' };
+  const buf = Buffer.from('Testing, Testing', 'UTF-8');
+  const name = `${uuidv4()}.${goodType.ext}`;
+  const response = {
+    ETag: '"8b03d1d48774bfafdb26691256fc7b2b"',
+    Location: `${process.env.S3_ENDPOINT}/${process.env.S3_BUCKET}/${name}`,
+    key: `${name}`,
+    Key: `${name}`,
+    Bucket: `${process.env.S3_BUCKET}`,
+  };
+  const promise = {
+    promise: () => new Promise((resolve) => resolve(response)),
+  };
+  const mockUpload = jest.spyOn(s3, 'upload').mockImplementation(async () => promise);
+  const mockGet = jest.spyOn(s3, 'getBucketVersioning').mockImplementation(async () => mockData);
+  beforeEach(() => {
+    mockUpload.mockClear();
+    mockGet.mockClear();
+  });
+  afterAll(() => {
+    process.env.LOCAL_DEV = oldEndpoint;
+  });
+
+  it('Correctly Uploads the file', async () => {
+    process.env.LOCAL_DEV = 'true';
+    const got = await s3Uploader(buf, name, goodType);
+    expect(mockGet.mock.calls.length).toBe(0);
+    await expect(got).toBe(response);
+  });
+  it('Correctly Uploads the file and checks versioning', async () => {
+    process.env.LOCAL_DEV = 'false';
+    const got = await s3Uploader(buf, name, goodType);
+    expect(mockGet.mock.calls.length).toBe(1);
+    await expect(got).toBe(response);
+  });
+});

src/migrations/20210114193123-create-file.js

@@ -0,0 +1,44 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.createTable('Files', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER,
+      },
+      activityReportId: {
+        type: Sequelize.INTEGER,
+        allowNull: false,
+        references: {
+          model: {
+            tableName: 'ActivityReports',
+          },
+        },
+      },
+      originalFileName: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      key: {
+        type: Sequelize.STRING,
+        allowNull: false,
+      },
+      status: {
+        type: Sequelize.ENUM('UPLOADING', 'UPLOADED', 'UPLOAD_FAILED', 'SCANNING', 'APPROVED', 'REJECTED'),
+        allowNull: false,
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE,
+      },
+    });
+  },
+  down: async (queryInterface) => {
+    await queryInterface.dropTable('Files');
+  },
+};

src/migrations/20210121192644-add-attachment-type-to-files.js

@@ -0,0 +1,14 @@
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('Files',
+      'attachmentType',
+      {
+        type: Sequelize.ENUM('ATTACHMENT', 'RESOURCE'),
+        allowNull: false,
+      });
+  },
+
+  down: async (queryInterface) => {
+    await queryInterface.removeColumn('Files', 'attachmentType');
+  },
+};

src/models/activityReport.js

@@ -16,6 +16,7 @@ export default (sequelize, DataTypes) => {
       ActivityReport.belongsTo(models.User, { foreignKey: 'lastUpdatedById', as: 'lastUpdatedBy' });
       ActivityReport.belongsTo(models.User, { foreignKey: 'approvingManagerId', as: 'approvingManager' });
       ActivityReport.hasMany(models.ActivityRecipient, { foreignKey: 'activityReportId', as: 'activityRecipients' });
+      ActivityReport.hasMany(models.File, { foreignKey: 'activityReportId', as: 'activityFiles' });
     }
   }
   ActivityReport.init({

src/models/file.js

@@ -0,0 +1,40 @@
+const { Model } = require('sequelize');
+
+module.exports = (sequelize, DataTypes) => {
+  class File extends Model {
+    /**
+     * Helper method for defining associations.
+     * This method is not a part of Sequelize lifecycle.
+     * The `models/index` file will call this method automatically.
+     */
+    static associate(models) {
+      File.belongsTo(models.ActivityReport, { foreignKey: 'activityReportId' });
+    }
+  }
+  File.init({
+    activityReportId: {
+      allowNull: false,
+      type: DataTypes.INTEGER,
+    },
+    originalFileName: {
+      allowNull: false,
+      type: DataTypes.STRING,
+    },
+    key: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    status: {
+      type: DataTypes.ENUM('UPLOADING', 'UPLOADED', 'UPLOAD_FAILED', 'SCANNING', 'APPROVED', 'REJECTED'),
+      allowNull: false,
+    },
+    attachmentType: {
+      type: DataTypes.ENUM('ATTACHMENT', 'RESOURCE'),
+      allowNull: false,
+    },
+  }, {
+    sequelize,
+    modelName: 'File',
+  });
+  return File;
+};

src/routes/apiDirectory.js

@@ -5,6 +5,7 @@ import join from 'url-join';
 import authMiddleware, { login } from '../middleware/authMiddleware';
 import handleErrors from '../lib/apiErrorHandler';
 import adminRouter from './user';
+import filesRouter from './files';
 import activityReportsRouter from './activityReports';
 import { userById } from '../services/users';
 import { auditLogger } from '../logger';
@@ -19,7 +20,7 @@ router.use(authMiddleware.unless({ path: [join('/api', loginPath)] }));
 
 router.use('/admin/users', adminRouter);
 router.use('/activity-reports', activityReportsRouter);
-
+router.use('/files', filesRouter);
 router.use('/hello', (req, res) => {
   res.send('Hello from ttadp');
 });