script and function for cleaning roots

.vscode/launch.json

@@ -14,6 +14,15 @@
       "console": "integratedTerminal",
       "program": "${workspaceFolder}/node_modules/.bin/jest",
       "args": ["${fileBasenameNoExtension}"]
+    },
+    {
+      "name": "Debug functions-v2 test",
+      "request": "launch",
+      "type": "node",
+      "console": "integratedTerminal",
+      "program": "${workspaceFolder}/functions-v2/node_modules/.bin/jest",
+      "args": ["${fileBasenameNoExtension}"],
+      "cwd": "${workspaceFolder}/functions-v2"
     }
   ]
 }

functions-v2/jest.config.js

@@ -2,6 +2,19 @@ module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
   testPathIgnorePatterns: ['lib/', 'node_modules/'],
+  moduleNameMapper: {
+    // These are necessary so code imported from ../shared/ will use the same version of
+    // firebase-admin that the local code does.
+    // The explicit `^` and `$` are needed so this only matches what we are importing.
+    // Otherwise it breaks the internal firebase admin code's imports
+    "^firebase-admin$": "<rootDir>/node_modules/firebase-admin",
+    "^firebase-admin/firestore$": "<rootDir>/node_modules/firebase-admin/lib/firestore",
+    "^firebase-admin/app$": "<rootDir>/node_modules/firebase-admin/lib/app",
+    "^firebase-admin/database$": "<rootDir>/node_modules/firebase-admin/lib/database",
+  },
+  // The tests can't be run in parallel because they are using a shared Firestore and
+  // Realtime database.
+  maxWorkers: 1,
 };
 
 // This is configured here because the clearFirebaseData function from
@@ -10,3 +23,4 @@ module.exports = {
 // The port here should match the port that is set in the emulators
 // section of firebase.json
 process.env["FIRESTORE_EMULATOR_HOST"]="127.0.0.1:8088";
+process.env["FIREBASE_DATABASE_EMULATOR_HOST"]="127.0.0.1:9000";

functions-v2/package.json

@@ -5,8 +5,9 @@
     "build": "tsc",
     "build:watch": "tsc --watch",
     "emulator": "firebase emulators:start --project demo-test",
+    "emulator:online": "firebase emulators:start",
     "serve": "npm run build && firebase emulators:start --only functions",
-    "shell": "npm run build && firebase functions:shell",
+    "shell": "npm run build && firebase functions:shell --project demo-test",
     "start": "npm run shell",
     "test": "jest",
     "test:emulator": "firebase emulators:start --project demo-test --only firestore,database",
@@ -16,10 +17,10 @@
   "engines": {
     "node": "20"
   },
-  "main": "lib/src/index.js",
+  "main": "lib/functions-v2/src/index.js",
   "dependencies": {
     "firebase-admin": "^12.1.0",
-    "firebase-functions": "^5.0.0"
+    "firebase-functions": "^5.1.1"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

functions-v2/src/at-midnight.ts

@@ -0,0 +1,28 @@
+import {onSchedule} from "firebase-functions/v2/scheduler";
+import * as logger from "firebase-functions/logger";
+
+// NOTE: in order for this import from shared to work it is necessary
+// to alias "firebase-admin" in tsconfig.json. Otherwise Typescript will
+// read the types from the parent node_modules. The parent directory
+// has a different version of the firebase dependencies, which cause
+// type errors.
+import {cleanFirebaseRoots} from "../../shared/clean-firebase-roots";
+
+export const atMidnight = onSchedule("0 7 * * *", runAtMidnight);
+
+// This function is split out so it can be tested by Jest. The
+// firebase-functions-test library doesn't support wrapping onSchedule.
+export async function runAtMidnight() {
+  await cleanFirebaseRoots({
+    appMode: "qa",
+    hoursAgo: 24,
+    logger,
+    dryRun: false,
+  });
+
+  // When cleanFirebaseRoots is called from a NodeJS script it is
+  // necessary to call Firebase's deleteApp so no threads are left running.
+  // Inside of a firebase function according to
+  // https://stackoverflow.com/a/72933644/3195497
+  // it isn't necessary to call deleteApp when the function is done.
+}

functions-v2/src/index.ts

@@ -1,4 +1,5 @@
 import * as admin from "firebase-admin";
 export {onUserDocWritten} from "./on-user-doc-written";
+export {atMidnight} from "./at-midnight";
 
 admin.initializeApp();

functions-v2/test/at-midnight.test.ts

@@ -0,0 +1,94 @@
+import {
+  clearFirestoreData,
+} from "firebase-functions-test/lib/providers/firestore";
+import {getFirestore, Timestamp} from "firebase-admin/firestore";
+import {getDatabase} from "firebase-admin/database";
+import * as logger from "firebase-functions/logger";
+import {initialize, projectConfig} from "./initialize";
+import {runAtMidnight} from "../src/at-midnight";
+
+jest.mock("firebase-functions/logger");
+
+const {cleanup} = initialize();
+
+const HOUR = 1000 * 60 * 60;
+
+async function writeFirestoreRoot(lastLaunchMillis = 0) {
+  const newRoot = getFirestore()
+    .collection("qa")
+    .doc();
+
+  await newRoot.set({
+    lastLaunchTime: Timestamp.fromMillis(lastLaunchMillis),
+  });
+
+  // Add some sub docs to make sure they are deleted
+  await newRoot.collection("users").doc().set({
+    uid: "test-user",
+  });
+
+  return newRoot;
+}
+
+async function writeDatabaseRoot(rootId: string) {
+  getDatabase().ref("qa").child(rootId).set({someField: "firebase realtime database"});
+}
+
+// In other tests we use firebase-functions-test to wrap the function.
+// In this case it would look like:
+//   const wrapped = fft.wrap(atMidnight);
+// However the wrapper doesn't support onSchedule:
+// - The Typescript types don't allow it
+// - at run time it doesn't pass the right event:
+//   https://github.com/firebase/firebase-functions-test/issues/210
+// So instead the code is separated from the onSchedule and called directly.
+
+describe("atMidnight", () => {
+  beforeEach(async () => {
+    await clearFirestoreData(projectConfig);
+    await getDatabase().ref().set(null);
+  });
+
+  test("clean up firestore roots with no database roots", async () => {
+    await writeFirestoreRoot();
+    await runAtMidnight();
+
+    const roots = await getFirestore().collection("qa").get();
+    expect(roots.size).toBe(0);
+    expect(logger.info)
+      .toHaveBeenCalledWith("Found 1 roots to delete");
+  });
+
+  test("clean up firestore root and database root", async () => {
+    const firestoreRoot = await writeFirestoreRoot();
+    await writeDatabaseRoot(firestoreRoot.id);
+
+    await runAtMidnight();
+
+    const fsRoots = await getFirestore().collection("qa").get();
+    expect(fsRoots.size).toBe(0);
+    const dbRoots = await getDatabase().ref("qa").get();
+    expect(dbRoots.val()).toEqual(null);
+    expect(logger.info)
+      .toHaveBeenCalledWith("Found 1 roots to delete");
+  });
+
+  test("only clean up firestore roots older than 24 hours", async () => {
+    await writeFirestoreRoot(Date.now() - HOUR);
+    await writeFirestoreRoot(Date.now() - 12*HOUR);
+    await writeFirestoreRoot(Date.now() - 23*HOUR);
+    await writeFirestoreRoot(Date.now() - 25*HOUR);
+    await writeFirestoreRoot(Date.now() - 48*HOUR);
+
+    await runAtMidnight();
+
+    const roots = await getFirestore().collection("qa").get();
+    expect(roots.size).toBe(3);
+    expect(logger.info)
+      .toHaveBeenCalledWith("Found 2 roots to delete");
+  });
+
+  afterAll(async () => {
+    await cleanup();
+  });
+});

functions-v2/test/initialize.ts

@@ -0,0 +1,30 @@
+import {deleteApp, initializeApp} from "firebase-admin/app";
+import initializeFFT from "firebase-functions-test";
+
+export const projectConfig = {
+  projectId: "demo-test",
+  // This URL doesn't have to be valid, it just has to a non empty string
+  // The actual database host will be picked up from
+  //   FIREBASE_DATABASE_EMULATOR_HOST
+  // This is defined in jest.config.js
+  databaseURL: "https://not-a-project.firebaseio.com",
+};
+
+export function initialize() {
+  const fft = initializeFFT(projectConfig);
+
+  // When the function is running in the cloud initializeApp is called by index.ts
+  // In our tests we import the function's module directly so we can call
+  // initializeApp ourselves. This is beneficial since initializeApp needs to
+  // be called after initializeFFT above.
+  const fbApp = initializeApp();
+
+  const cleanup = async () => {
+    fft.cleanup();
+    // Deleting the Firebase app is necessary for the Jest tests to exit when they
+    // are complete. FFT creates a testApp which it deletes in cleanup(), but
+    // we are not using this testApp.
+    await deleteApp(fbApp);
+  };
+  return {fft, fbApp, cleanup};
+}

functions-v2/test/on-user-doc-written.test.ts

@@ -1,22 +1,14 @@
-import initializeFFT from "firebase-functions-test";
 import {
   clearFirestoreData,
 } from "firebase-functions-test/lib/providers/firestore";
 import * as logger from "firebase-functions/logger";
 import * as admin from "firebase-admin";
+import {initialize, projectConfig} from "./initialize";
 import {onUserDocWritten} from "../src/on-user-doc-written";
 
 jest.mock("firebase-functions/logger");
 
-process.env["FIRESTORE_EMULATOR_HOST"]="127.0.0.1:8088";
-const projectConfig = {projectId: "demo-test"};
-const fft = initializeFFT(projectConfig);
-
-// When the function is running in the cloud initializeApp is called by index.ts
-// Here we are importing the function's module directly so we can call
-// initializeApp ourselves. This is beneficial since initializeApp needs to
-// be called after initializeFFT above.
-admin.initializeApp();
+const {fft, cleanup} = initialize();
 
 type CollectionRef = admin.firestore.CollectionReference<
   admin.firestore.DocumentData, admin.firestore.DocumentData
@@ -288,7 +280,7 @@ describe("functions", () => {
     });
   });
 
-  afterAll(() => {
-    fft.cleanup();
+  afterAll(async () => {
+    await cleanup();
   });
 });

functions-v2/tsconfig.json

@@ -11,6 +11,17 @@
     // This prevents typescript from trying to include @types from the parent folders.
     // The types in the parent folders conflict so they break the build.
     "typeRoots": ["./node_modules/@types"],
+    "paths": {
+      // These are necessary so code imported from ../shared/ will use the same version of
+      // firebase-admin that the local code does. Technically only "firebase-admin/firestore"
+      // seems to be currently required, but it seems safer to alias all of the admin
+      // libraries the shared code might be using.
+      "firebase-admin": ["./node_modules/firebase-admin/lib"],
+      "firebase-admin/firestore": ["./node_modules/firebase-admin/lib/firestore"],
+      "firebase-admin/app": ["./node_modules/firebase-admin/lib/app"],
+      "firebase-admin/database": ["./node_modules/firebase-admin/lib/database"],
+    },
+
   },
   "compileOnSave": true,
   "include": [

scripts/clean-firebase-roots.ts

@@ -0,0 +1,33 @@
+#!/usr/bin/node
+
+// This script cleans the roots out of the QA or Dev sections of
+// the Firebase Realtime database and Firestore.
+
+// to run this script type the following in the terminal
+// cf. https://stackoverflow.com/a/66626333/16328462
+// $ cd scripts
+// $ npx tsx clean-firebase-roots.ts
+
+import admin from "firebase-admin";
+import { deleteApp } from "firebase-admin/app";
+import {cleanFirebaseRoots} from "../shared/clean-firebase-roots.js";
+import { getScriptRootFilePath } from "./lib/script-utils.js";
+
+const databaseURL = "https://collaborative-learning-ec215.firebaseio.com";
+
+const serviceAccountFile = getScriptRootFilePath("serviceAccountKey.json");
+const credential = admin.credential.cert(serviceAccountFile);
+// Initialize the app with a service account, granting admin privileges
+const fbApp = admin.initializeApp({
+  credential,
+  databaseURL
+});
+
+await cleanFirebaseRoots({
+  appMode: "qa",
+  hoursAgo: 90.7,
+  logger: console,
+  dryRun: true
+});
+
+await deleteApp(fbApp);

shared/clean-firebase-roots.ts

@@ -0,0 +1,50 @@
+// This requires the modern firebase-admin, so it can't be used by functions-v1
+import {Timestamp, getFirestore} from "firebase-admin/firestore";
+import {getDatabase} from "firebase-admin/database";
+
+const HOUR = 1000 * 60 * 60;
+
+interface Logger {
+  info(...args: any[]): void;
+}
+
+interface Params {
+  appMode: "qa" | "dev";
+  hoursAgo: number;
+  logger: Logger;
+  dryRun?: boolean;
+}
+
+export async function cleanFirebaseRoots(
+  { appMode, hoursAgo, logger, dryRun }: Params
+) {
+
+  // Be extra careful so we don't delete production data
+  if (!["qa", "dev"].includes(appMode)) {
+    throw new Error(`Invalid appMode ${appMode}`);
+  }
+
+  // Clean up Firestore and Realtime database roots that haven't been launched in `hoursAgo` hours
+  const cutOffMillis = Date.now() - hoursAgo*HOUR;
+  const qaRootsResult = await getFirestore()
+    .collection(appMode)
+    .where("lastLaunchTime", "<", Timestamp.fromMillis(cutOffMillis))
+    .get();
+
+  logger.info(`Found ${qaRootsResult.size} roots to delete`);
+
+  // Need to be careful to clean up the root in the realtime database
+  // first. The record in Firestore is our only way to figure out which
+  // roots in the realtime database need to be deleted.
+  for (const root of qaRootsResult.docs) {
+    // The Realtime database root is deleted first incase it fails.
+    // This way the root in firestore will remain so we can find it
+    // and try again later.
+    const databasePath = `/${appMode}/${root.id}`;
+    logger.info(`Deleting Realtime Database root: ${databasePath} ...`);
+    if (!dryRun) await getDatabase().ref(`/${appMode}/${root.id}`).remove();
+    logger.info(`Deleting Firestore root: ${root.ref.path} ...`);
+    if (!dryRun) await getFirestore().recursiveDelete(root.ref);
+  }
+
+}