Merge branch 'master' into 230-transcription-ui

README.md

@@ -219,15 +219,29 @@ Node version is set in node version manager
 
 Use node v8 in Functions directory.
 
-Developing is a lot easier if you have your **local emulator** set up.
+Developing is a lot easier if you have your **local emulator** set up. 
 
-1. Follow the instructions
-   [here](https://firebase.google.com/docs/functions/local-2.
-   emulator#set_up_admin_credentials_optional) to get the admin credentials.
-2. You need to save this as `gcp-credentials.json` and keep it in your
+### Running Firebase functions locally:
+
+1. Set up admin credentials for emulated functions via the [service account](https://console.cloud.google.com/iam-admin/serviceaccounts/details/102625058144632397517/keys?authuser=1&folder=&organizationId=&project=newslabs-dev-aa20&supportedpurview=project )
+2. Make sure you are in the `newslab-dev` project and click `Add key`
+3. You need to save this as `gcp-credentials.json` and keep it in your
    `digital-paper-edit-firebase/functions` folder.
-3. Run `./start_firebase_shell` in functions folder.
-   <!-- TODO: Setup eslint in express server -->
+4. cd to your `digital-paper-edit-firebase/functions` folder.
+5. run the following commands
+```
+export GOOGLE_APPLICATION_CREDENTIALS="gcp-credentials.json"
+firebase functions:config:get > .runtimeconfig.json
+```
+3. Then to start the emulator in your terminal window, run 
+```
+firebase functions:shell
+```
+4. BEWARE!!! Running functions locally will really affect files in the Firestore, so tread carefully!
+5. Call the function you would like to test e.g. run `dpeCronExpiredMediaChecker()`
+
+See [here](https://firebase.google.com/docs/functions/local-emulator) for more info on running functions locally.
+
 
 ### Function deployment
 

functions/dpeCronExpiredMediaChecker/index.js

@@ -0,0 +1,79 @@
+const { updateTranscription } = require('../sttChecker');
+const functions = require("firebase-functions");
+
+const daysUntilExpiry = 60;
+
+const isDeletableContentType = (contentType) => {
+  return contentType ? contentType.includes('video') || contentType.includes('audio') : false;
+};
+
+const deleteExpiredFile = async (file, admin) => {
+  await file.delete();
+  const transcriptId = file.metadata.metadata.id;
+  if (transcriptId) {
+    await updateTranscription(admin, transcriptId, file.metadata.metadata.projectId, {
+      status: 'expired',
+      message: `Media older than ${ daysUntilExpiry } days`,
+    });
+  }
+};
+
+const checkForExpiredContent = async (allFiles, admin) => {
+  let filePath = '';
+  let transcriptFilePath = '';
+
+  const expiredMediaFiles = allFiles.map(async (file) => {
+    try {
+      const dateCreated = new Date(file.metadata.timeCreated);
+      const expiryDate = new Date(dateCreated);
+      expiryDate.setDate(dateCreated.getDate() + daysUntilExpiry);
+      filePath = file.metadata.name;
+      transcriptFilePath = file.metadata.metadata ? `projects/${ file.metadata.metadata.projectId }/transcripts/${ file.metadata.metadata.id }` : 'no transcript';
+
+      if (Date.now() >= expiryDate && isDeletableContentType(file.metadata.contentType)) {
+
+        functions.logger.log(`[IN PROGRESS] Expired media is being deleted:
+        filePath: ${ filePath }
+        transcriptFilePath: ${ transcriptFilePath }`);
+        return deleteExpiredFile(file, admin);
+      }
+      return Promise.resolve();
+    } catch (error) {
+      functions.logger.log(`[ERROR]: Unable to delete file ${ filePath } : `, error);
+      return Promise.reject(error);
+    }
+  });
+
+  return expiredMediaFiles;
+};
+
+const dpeCronExpiredMediaChecker = async (bucket, admin) => {
+  functions.logger.log('[START] Checking for expired media files...');
+  let data = [];
+  let allFiles = [];
+
+  try {
+    data = await bucket.getFiles();
+  } catch (error) {
+    functions.logger.log(`[ERROR] data not found`, error);
+    return error;
+  }
+  allFiles = data.length > 0 ? data[0] : [];
+
+  try {
+    const expiredContent = checkForExpiredContent(allFiles, admin);
+    if (expiredContent.length > 0) {
+      return await Promise.all(expiredContent);
+    }
+    functions.logger.log('[COMPLETE] Deleted expired media content ✅');
+    return Promise.resolve();
+  }
+  catch (error) {
+    functions.logger.log(`[ERROR] Files could not be deleted`, error );
+    return Promise.reject(error);
+  }
+};
+
+exports.createHandler = async (bucket, admin) => {
+  await dpeCronExpiredMediaChecker(bucket, admin);
+};

functions/index.js

@@ -6,6 +6,7 @@ const inventoryChecker = require("./inventoryChecker");
 const audioStripper = require("./audioStripper");
 const awsUploader = require("./awsUploader");
 const sttChecker = require("./sttChecker");
+const dpeCronExpiredMediaChecker = require("./dpeCronExpiredMediaChecker");
 
 // Was run as a migration task
 // const compressData = require("./compressData")
@@ -52,6 +53,13 @@ exports.dpeCronSTTJobChecker = functions
     sttChecker.createHandler(admin, config.aws.api.transcriber, context)
   );
 
+exports.dpeCronExpiredMediaChecker = functions
+  .runWith(maxRuntimeOpts)
+  .pubsub.schedule("every 24 hours")
+  .onRun(() =>
+    dpeCronExpiredMediaChecker.createHandler(bucket, admin)
+  );
+
 // For migration of DB
 
 //  exports.compressToGrouped = functions

functions/sttChecker/index.js

@@ -217,3 +217,5 @@ const sttCheckRunner = async (admin, config, execTimestamp) => {
 exports.createHandler = async (admin, config, context) => {
   await sttCheckRunner(admin, config, context.timestamp);
 };
+
+exports.updateTranscription = updateTranscription;