Merge branch 'main' into feat/forms-1331-event-stream

.github/workflows/.deploy.yaml

@@ -85,7 +85,7 @@ jobs:
           pr_number: ${{ github.event.inputs.pr-number }}
 
   deploy:
-    name: Deploys to selected environment
+    name: Deploy
     environment:
       name: pr
       url: ${{ needs.set-vars.outputs.URL }}

.github/workflows/on_push.yaml

@@ -21,7 +21,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build & Push
         uses: ./.github/actions/build-push-container
         with:
@@ -38,10 +38,12 @@ jobs:
       url: https://${{ env.ACRONYM }}-dev.apps.silver.devops.gov.bc.ca/app
     runs-on: ubuntu-latest
     needs: build
+    outputs:
+      url: https://${{ env.ACRONYM }}-dev.apps.silver.devops.gov.bc.ca/app
     timeout-minutes: 12
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Deploy to Dev
         uses: ./.github/actions/deploy-to-environment
         with:
@@ -57,6 +59,13 @@ jobs:
           route_path: /app
           route_prefix: ${{ vars.ROUTE_PREFIX }}
 
+  scan-dev:
+    name: Scan Dev
+    needs: deploy-dev
+    uses: ./.github/workflows/reusable-owasp-zap.yaml
+    with:
+      url: ${{ needs.deploy-dev.outputs.url }}
+
   deploy-test:
     name: Deploy to Test
     environment:
@@ -69,7 +78,7 @@ jobs:
     timeout-minutes: 12
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Deploy to Test
         uses: ./.github/actions/deploy-to-environment
         with:
@@ -98,7 +107,7 @@ jobs:
     timeout-minutes: 12
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Deploy to Prod
         uses: ./.github/actions/deploy-to-environment
         with:

app/app.js

@@ -39,6 +39,8 @@ app.use(express.urlencoded({ extended: true }));
 // See https://express-rate-limit.github.io/ERR_ERL_UNEXPECTED_X_FORWARDED_FOR
 app.set('trust proxy', 1);
 
+app.set('x-powered-by', false);
+
 // Skip if running tests
 if (process.env.NODE_ENV !== 'test') {
   // Initialize connections and exit if unsuccessful

app/frontend/src/components/designer/FormViewer.vue

@@ -8,14 +8,17 @@ import BaseDialog from '~/components/base/BaseDialog.vue';
 import FormViewerActions from '~/components/designer/FormViewerActions.vue';
 import FormViewerMultiUpload from '~/components/designer/FormViewerMultiUpload.vue';
 import templateExtensions from '~/plugins/templateExtensions';
-import { formService, rbacService } from '~/services';
+import { fileService, formService, rbacService } from '~/services';
 import { useAppStore } from '~/store/app';
 import { useAuthStore } from '~/store/auth';
 import { useFormStore } from '~/store/form';
 import { useNotificationStore } from '~/store/notification';
 
 import { isFormPublic } from '~/utils/permissionUtils';
-import { attachAttributesToLinks } from '~/utils/transformUtils';
+import {
+  attachAttributesToLinks,
+  getDisposition,
+} from '~/utils/transformUtils';
 import { FormPermissions, NotificationTypes } from '~/utils/constants';
 
 export default {
@@ -77,6 +80,7 @@ export default {
       bulkFile: false,
       confirmSubmit: false,
       currentForm: {},
+      downloadTimeout: null,
       doYouWantToSaveTheDraft: false,
       forceNewTabLinks: true,
       form: {},
@@ -121,7 +125,7 @@ export default {
       'tokenParsed',
       'user',
     ]),
-    ...mapState(useFormStore, ['isRTL']),
+    ...mapState(useFormStore, ['downloadedFile', 'isRTL']),
 
     formScheduleExpireMessage() {
       return this.$t('trans.formViewer.formScheduleExpireMessage');
@@ -148,6 +152,9 @@ export default {
           simplefile: {
             config: this.config,
             chefsToken: this.getCurrentAuthHeader,
+            deleteFile: this.deleteFile,
+            getFile: this.getFile,
+            uploadFile: this.uploadFile,
           },
         },
         evalContext: {
@@ -190,13 +197,15 @@ export default {
   },
   beforeUnmount() {
     window.removeEventListener('beforeunload', this.beforeWindowUnload);
+    clearTimeout(this.downloadTimeout);
   },
   beforeUpdate() {
     if (this.forceNewTabLinks) {
       attachAttributesToLinks(this.formSchema.components);
     }
   },
   methods: {
+    ...mapActions(useFormStore, ['downloadFile']),
     ...mapActions(useNotificationStore, ['addNotification']),
     isFormPublic: isFormPublic,
     getCurrentAuthHeader() {
@@ -1079,6 +1088,50 @@ export default {
         e.returnValue = '';
       }
     },
+    async deleteFile(file) {
+      return fileService.deleteFile(file.id);
+    },
+    async getFile(fileId, options = {}) {
+      await this.downloadFile(fileId, options);
+      if (this.downloadedFile && this.downloadedFile.headers) {
+        let data;
+
+        if (
+          this.downloadedFile.headers['content-type'].includes(
+            'application/json'
+          )
+        ) {
+          data = JSON.stringify(this.downloadedFile.data);
+        } else {
+          data = this.downloadedFile.data;
+        }
+
+        if (typeof data === 'string') {
+          data = new Blob([data], {
+            type: this.downloadedFile.headers['content-type'],
+          });
+        }
+
+        // don't need to blob because it's already a blob
+        const url = window.URL.createObjectURL(data);
+        const a = document.createElement('a');
+        a.href = url;
+        a.download = getDisposition(
+          this.downloadedFile.headers['content-disposition']
+        );
+        a.style.display = 'none';
+        a.classList.add('hiddenDownloadTextElement');
+        document.body.appendChild(a);
+        a.click();
+        this.downloadTimeout = setTimeout(() => {
+          document.body.removeChild(a);
+          URL.revokeObjectURL(a.href);
+        });
+      }
+    },
+    async uploadFile(file, config = {}) {
+      return fileService.uploadFile(file, config);
+    },
   },
 };
 </script>

app/frontend/src/services/fileService.js

@@ -2,7 +2,13 @@ import { appAxios } from '~/services/interceptors';
 import { ApiRoutes } from '~/utils/constants';
 
 export default {
-  async getFile(fileId) {
-    return appAxios().get(`${ApiRoutes.FILES}/${fileId}`);
+  async deleteFile(fileId) {
+    return appAxios().delete(`${ApiRoutes.FILES}/${fileId}`);
+  },
+  async getFile(fileId, options = {}) {
+    return appAxios().get(`${ApiRoutes.FILES}/${fileId}`, options);
+  },
+  async uploadFile(file, config = {}) {
+    return appAxios().post(`${ApiRoutes.FILES}`, file, config);
   },
 };

app/frontend/src/store/form.js

@@ -826,10 +826,10 @@ export const useFormStore = defineStore('form', {
       if (!this.form || this.form.isDirty === isDirty) return; // don't do anything if not changing the val (or if form is blank for some reason)
       this.form.isDirty = isDirty;
     },
-    async downloadFile(fileId) {
+    async downloadFile(fileId, options = {}) {
       try {
         this.downloadedFile = {};
-        const response = await fileService.getFile(fileId);
+        const response = await fileService.getFile(fileId, options);
         this.downloadedFile.data = response.data;
         this.downloadedFile.headers = response.headers;
       } catch (error) {

app/src/forms/file/middleware/upload.js

@@ -5,13 +5,15 @@ const os = require('os');
 
 const Problem = require('api-problem');
 
-let uploader = undefined;
-let storage = undefined;
+let fileUploadsDir = os.tmpdir();
 let maxFileSize = bytes.parse('25MB');
 let maxFileCount = 1;
 
+let storage;
+let uploader;
+
 const fileSetup = (options) => {
-  const fileUploadsDir = (options && options.dir) || process.env.FILE_UPLOADS_DIR || fs.realpathSync(os.tmpdir());
+  fileUploadsDir = (options && options.dir) || process.env.FILE_UPLOADS_DIR || fs.realpathSync(os.tmpdir());
   try {
     fs.ensureDirSync(fileUploadsDir);
   } catch (error) {
@@ -61,6 +63,15 @@ module.exports.fileUpload = {
     }
   },
 
+  /**
+   * Gets the directory where the files are uploaded to.
+   *
+   * @returns the file uploads directory.
+   */
+  getFileUploadsDir() {
+    return fileUploadsDir;
+  },
+
   async upload(req, res, next) {
     try {
       if (!uploader) {

app/src/forms/file/storage/objectStorageService.js

@@ -9,6 +9,8 @@ const StorageTypes = require('../../common/constants').StorageTypes;
 const errorToProblem = require('../../../components/errorToProblem');
 const log = require('../../../components/log')(module.filename);
 
+const fileUpload = require('../middleware/upload').fileUpload;
+
 const SERVICE = 'ObjectStorage';
 const TEMP_DIR = 'uploads';
 const Delimiter = '/';
@@ -58,9 +60,31 @@ class ObjectStorageService {
     return '';
   }
 
+  /**
+   * Gets the contents of a file from the local filesystem. Will error if the
+   * requested file is not in the allowed file uploads directory.
+   *
+   * @param {string} filename the filename of the file to be read.
+   * @returns a Buffer containing the file contents.
+   * @throws an Error if the filename is not within the allowed directory.
+   */
+  _readLocalFile(filename) {
+    let fileUploadsDir = fileUpload.getFileUploadsDir();
+    if (!fileUploadsDir.endsWith('/')) {
+      fileUploadsDir += '/';
+    }
+
+    const resolvedFilename = fs.realpathSync(path.resolve(fileUploadsDir, filename));
+    if (!resolvedFilename.startsWith(fileUploadsDir)) {
+      throw new Error(`Invalid path '${filename}'`);
+    }
+
+    return fs.readFileSync(resolvedFilename);
+  }
+
   async uploadFile(fileStorage) {
     try {
-      const fileContent = fs.readFileSync(fileStorage.path);
+      const fileContent = this._readLocalFile(fileStorage.path);
 
       // uploads can go to a 'holding' area, we can shuffle it later if we want to.
       const key = this._join(this._key, TEMP_DIR, fileStorage.id);

app/src/forms/form/exportService.js

@@ -114,14 +114,7 @@ const service = {
 
     if (fields) {
       return formSchemaheaders.filter((header) => {
-        // In the 'fields' sent from the caller we'll have something like
-        // 'datagrid.input', but in the actual submission data in the
-        // 'formSchemaheaders' we'll have things like 'datagrid.0.input',
-        // 'datagrid.1.input', etc. Remove the '.0' array index to get
-        // 'datagrid.input' and then do the comparison.
-        const flattenedHeader = header.replace(/\.\d+\./gi, '.');
-
-        if (Array.isArray(fields) && fields.includes(flattenedHeader)) {
+        if (Array.isArray(fields) && fields.includes(header)) {
           return header;
         }
       });

app/tests/fixtures/submission/kitchen_sink_submission_data_export_datagrid_fields_selection.json

@@ -10,10 +10,13 @@
   "email",
   "forWhichBcLakeRegionAreYouCompletingTheseQuestions",
   "didYouFishAnyBcLakesThisYear",
-  "oneRowPerLake.lakeName",
-  "oneRowPerLake.closestTown",
-  "oneRowPerLake.numberOfDays",
-  "oneRowPerLake.dataGrid.fishType",
-  "oneRowPerLake.dataGrid.numberCaught",
-  "oneRowPerLake.dataGrid.numberKept"
+  "oneRowPerLake.0.lakeName",
+  "oneRowPerLake.0.closestTown",
+  "oneRowPerLake.0.numberOfDays",
+  "oneRowPerLake.0.dataGrid.0.fishType",
+  "oneRowPerLake.0.dataGrid.1.fishType",
+  "oneRowPerLake.0.dataGrid.0.numberCaught",
+  "oneRowPerLake.0.dataGrid.1.numberCaught",
+  "oneRowPerLake.0.dataGrid.0.numberKept",
+  "oneRowPerLake.0.dataGrid.1.numberKept"
 ]

app/tests/unit/forms/file/middleware/upload.spec.js

@@ -236,6 +236,20 @@ describe('fileUpload.init', () => {
   });
 });
 
+describe('fileUpload.getFileUploadsDir', () => {
+  const mockOs = '/mock_os_tmpdir';
+
+  test('uses os.tmpdir when there is no config or environment variable', async () => {
+    fs.realpathSync.mockReturnValueOnce(mockOs);
+    os.tmpdir.mockReturnValueOnce(mockOs);
+    fileUpload.init();
+
+    const result = fileUpload.getFileUploadsDir();
+
+    expect(result).toBe(mockOs);
+  });
+});
+
 describe('fileUpload.upload', () => {
   // These are for the sake of completeness but there isn't much value here.
   describe('400 response when', () => {

app/tests/unit/forms/form/exportService.spec.js

@@ -126,7 +126,8 @@ describe('export', () => {
           'form.username',
           'form.email',
           'dataGrid',
-          'dataGrid.simpletextfield',
+          'dataGrid.0.simpletextfield',
+          'dataGrid.1.simpletextfield',
         ],
         template: 'singleRowCSVExport',
       };
@@ -515,7 +516,7 @@ describe('_buildCsvHeaders', () => {
     // get result columns if we need to filter out the columns
     const result = await exportService._buildCsvHeaders(form, submissionsExport, 1, fields, true);
 
-    expect(result).toHaveLength(29);
+    expect(result).toHaveLength(20);
     expect(result).toEqual(
       expect.arrayContaining([
         'form.confirmationId',