Skip to content

Commit

Permalink
Merge pull request #101 from jsasitorn/debug_logging
Browse files Browse the repository at this point in the history
Make debug logging of typesense imports expicit and configurable
jasonbosco authored Jan 21, 2025

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents 4eb7848 + e73c416 commit 3d66da6
Showing 11 changed files with 553 additions and 97 deletions.
2 changes: 1 addition & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ module.exports = {
jest: true,
},
parserOptions: {
ecmaVersion: 2020,
ecmaVersion: 2022,
},
extends: ["eslint:recommended", "google"],
rules: {
12 changes: 12 additions & 0 deletions extension.yaml
Original file line number Diff line number Diff line change
@@ -108,6 +108,18 @@ params:
value: true
default: false
required: false
- param: LOG_TYPESENSE_INSERTS
label: Log Typesense Inserts for Debugging
description: >-
Should data inserted into Typesense be logged in Cloud Logging? This can be useful for debugging, but should not be enabled in production.
type: select
options:
- label: No
value: false
- label: Yes
value: true
default: false
required: false
- param: LOCATION
label: Cloud Functions location
description: >-
6 changes: 5 additions & 1 deletion functions/src/backfill.js
Original file line number Diff line number Diff line change
@@ -68,7 +68,11 @@ module.exports = functions.firestore.document(config.typesenseBackfillTriggerDoc
const pathParams = utils.pathMatchesSelector(docPath, config.firestoreCollectionPath);

if (!isGroupQuery || (isGroupQuery && pathParams !== null)) {
return await utils.typesenseDocumentFromSnapshot(doc, pathParams);
const typesenseDocument = await utils.typesenseDocumentFromSnapshot(doc, pathParams);
if (config.shouldLogTypesenseInserts) {
functions.logger.debug(`Backfilling document ${JSON.stringify(typesenseDocument)}`);
}
return typesenseDocument;
} else {
return null;
}
1 change: 1 addition & 0 deletions functions/src/config.js
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ module.exports = {
.map((f) => f.trim())
.filter((f) => f),
shouldFlattenNestedDocuments: process.env.FLATTEN_NESTED_DOCUMENTS === "true",
shouldLogTypesenseInserts: process.env.LOG_TYPESENSE_INSERTS === "true",
typesenseHosts:
(process.env.TYPESENSE_HOSTS || "").split(",").map((e) => e.trim()),
typesensePort: process.env.TYPESENSE_PORT || 443,
6 changes: 5 additions & 1 deletion functions/src/indexOnWrite.js
Original file line number Diff line number Diff line change
@@ -22,7 +22,11 @@ module.exports = functions.firestore.document(config.firestoreCollectionPath)
const latestSnapshot = await snapshot.after.ref.get();
const typesenseDocument = await utils.typesenseDocumentFromSnapshot(latestSnapshot, context.params);

functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
if (config.shouldLogTypesenseInserts) {
functions.logger.debug(`Upserting document ${JSON.stringify(typesenseDocument)}`);
} else {
functions.logger.debug(`Upserting document ${typesenseDocument.id}`);
}
return await typesense
.collections(encodeURIComponent(config.typesenseCollectionName))
.documents()
2 changes: 0 additions & 2 deletions functions/src/utils.js
Original file line number Diff line number Diff line change
@@ -145,8 +145,6 @@ exports.typesenseDocumentFromSnapshot = async (firestoreDocumentSnapshot, contex
// using flat to flatten nested objects for older versions of Typesense that did not support nested fields
// https://typesense.org/docs/0.22.2/api/collections.html#indexing-nested-fields
const typesenseDocument = config.shouldFlattenNestedDocuments ? flattenDocument(mappedDocument) : mappedDocument;
console.log("typesenseDocument", typesenseDocument);

typesenseDocument.id = firestoreDocumentSnapshot.id;

if (contextParams && Object.entries(contextParams).length) {
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -5,9 +5,9 @@
"emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
"export": "firebase emulators:export emulator_data",
"test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3",
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\" --testPathIgnorePatterns=\"Subcollection\"'",
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'",
"test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
"test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'",
"test-part-3": "jest --testRegex=\"writeLogging\" --testRegex=\"Subcollection\" --detectOpenHandles",
"typesenseServer": "docker compose up",
"lint:fix": "eslint . --fix",
"lint": "eslint ."
111 changes: 58 additions & 53 deletions test/backfillSubcollection.spec.js
Original file line number Diff line number Diff line change
@@ -1,59 +1,56 @@
const firebase = require("firebase-admin");
const config = require("../functions/src/config.js");
const typesense = require("../functions/src/createTypesenseClient.js")();

const app = firebase.initializeApp({
// Use a special URL to talk to the Realtime Database emulator
databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
projectId: process.env.GCLOUD_PROJECT,
});
const firestore = app.firestore();
const {TestEnvironment} = require("./support/testEnvironment");

// test case configs
const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users";
const TEST_FIRESTORE_CHILD_FIELD_NAME = "books";

describe("backfillSubcollection", () => {
const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH;
let testEnvironment;

const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH;
const unrelatedCollectionPath = "unrelatedCollectionToNotBackfill";
const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME;
const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME;
let parentIdField = null;

beforeAll(() => {
const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g);
expect(matches).toBeDefined();
expect(matches.length).toBe(1);
let config = null;
let firestore = null;
let typesense = null;

beforeAll((done) => {
testEnvironment = new TestEnvironment({
dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env",
outputAllEmulatorLogs: true,
});
testEnvironment.setupTestEnvironment((err) => {
const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g);
expect(matches).toBeDefined();
expect(matches.length).toBe(1);

parentIdField = matches[0].replace(/{|}/g, "");
expect(parentIdField).toBeDefined();

config = testEnvironment.config;
firestore = testEnvironment.firestore;
typesense = testEnvironment.typesense;
done();
});
});

parentIdField = matches[0].replace(/{|}/g, "");
expect(parentIdField).toBeDefined();
afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
// Clear the database between tests
// For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions
await firestore.recursiveDelete(firestore.collection(parentCollectionPath));
await firestore.recursiveDelete(firestore.collection(unrelatedCollectionPath));

// Clear any previously created collections
try {
await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
} catch (e) {
console.info(`${config.typesenseCollectionName} collection not found, proceeding...`);
}

// Create a new Typesense collection
return typesense.collections().create({
name: config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});
});

afterAll(async () => {
// clean up the firebase app after all tests have run
await app.delete();
await testEnvironment.clearAllData();
});

describe("when firestore_collections is not specified", () => {
it("backfills existing Firestore data in all collections to Typesense" +
" when `trigger: true` is set " +
` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
" when `trigger: true` is set on trigger document", async () => {
const parentDocData = {
nested_field: {
field1: "value1",
@@ -67,7 +64,7 @@ describe("backfillSubcollection", () => {
};

// create parent document in Firestore
const parentDocRef = await firestore.collection(parentCollectionPath).add(parentDocData);
const parentDocRef = await testEnvironment.firestore.collection(parentCollectionPath).add(parentDocData);

// create a subcollection with document under the parent document
const subCollectionRef = parentDocRef.collection(childFieldName);
@@ -78,24 +75,24 @@ describe("backfillSubcollection", () => {

// The above will automatically add the document to Typesense,
// so delete it so we can test backfill
await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
await typesense.collections().create({
name: config.typesenseCollectionName,
await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
await testEnvironment.typesense.collections().create({
name: testEnvironment.config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});

await firestore
.collection(config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
await testEnvironment.firestore
.collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
.doc("backfill")
.set({trigger: true});
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// Check that the data was backfilled
const typesenseDocsStr = await typesense
.collections(encodeURIComponent(config.typesenseCollectionName))
const typesenseDocsStr = await testEnvironment.typesense
.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
@@ -113,8 +110,7 @@ describe("backfillSubcollection", () => {
describe("when firestore_collections is specified", () => {
describe("when firestore_collections includes this collection", () => {
it("backfills existing Firestore data in this particular collection to Typesense" +
" when `trigger: true` is set " +
` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
" when `trigger: true` is set on trigger document", async () => {
const parentDocData = {
nested_field: {
field1: "value1",
@@ -163,6 +159,7 @@ describe("backfillSubcollection", () => {
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
console.log(typesenseDocs);
expect(typesenseDocs.length).toBe(1);
expect(typesenseDocs[0]).toStrictEqual({
id: subDocRef.id,
@@ -175,8 +172,7 @@ describe("backfillSubcollection", () => {

describe("when firestore_collections does not include this collection", () => {
it("does not backfill existing Firestore data in this particular collection to Typesense" +
" when `trigger: true` is set " +
` in ${config.typesenseBackfillTriggerDocumentInFirestore}`, async () => {
" when `trigger: true` is set on trigger document", async () => {
const parentDocData = {
nested_field: {
field1: "value1",
@@ -194,7 +190,7 @@ describe("backfillSubcollection", () => {

// create a subcollection with document under the parent document
const subCollectionRef = parentDocRef.collection(childFieldName);
await subCollectionRef.add(subDocData);
const subDocRef = await subCollectionRef.add(subDocData);
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

@@ -224,6 +220,15 @@ describe("backfillSubcollection", () => {
.documents()
.export();
expect(typesenseDocsStr).toEqual("");

// Check that the error was logged
testEnvironment.resetCapturedEmulatorLogs();
subDocRef.delete();
await new Promise((r) => setTimeout(r, 5000));

expect(testEnvironment.capturedEmulatorLogs).toContain(
`Could not find a document with id: ${subDocRef.id}`,
);
});
});
});
74 changes: 37 additions & 37 deletions test/indexOnWriteSubcollection.spec.js
Original file line number Diff line number Diff line change
@@ -1,50 +1,50 @@
const firebase = require("firebase-admin");
const config = require("../functions/src/config.js");
const typesense = require("../functions/src/createTypesenseClient.js")();

const app = firebase.initializeApp({
// Use a special URL to talk to the Realtime Database emulator
databaseURL: `${process.env.FIREBASE_DATABASE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
projectId: process.env.GCLOUD_PROJECT,
});
const firestore = app.firestore();
const {TestEnvironment} = require("./support/testEnvironment");

// test case configs
const TEST_FIRESTORE_PARENT_COLLECTION_PATH = "users";
const TEST_FIRESTORE_CHILD_FIELD_NAME = "books";


describe("indexOnWriteSubcollection", () => {
const parentCollectionPath = process.env.TEST_FIRESTORE_PARENT_COLLECTION_PATH;
const childFieldName = process.env.TEST_FIRESTORE_CHILD_FIELD_NAME;
let testEnvironment;

const parentCollectionPath = TEST_FIRESTORE_PARENT_COLLECTION_PATH;
const childFieldName = TEST_FIRESTORE_CHILD_FIELD_NAME;
let parentIdField = null;

beforeAll(async () => {
const matches = config.firestoreCollectionPath.match(/{([^}]+)}/g);
expect(matches).toBeDefined();
expect(matches.length).toBe(1);
let config = null;
let firestore = null;
let typesense = null;

parentIdField = matches[0].replace(/{|}/g, "");
expect(parentIdField).toBeDefined();
beforeAll((done) => {
testEnvironment = new TestEnvironment({
dotenvPath: "extensions/test-params-subcategory-flatten-nested-false.local.env",
outputAllEmulatorLogs: true,
});
testEnvironment.setupTestEnvironment((err) => {
const matches = testEnvironment.config.firestoreCollectionPath.match(/{([^}]+)}/g);
expect(matches).toBeDefined();
expect(matches.length).toBe(1);

parentIdField = matches[0].replace(/{|}/g, "");
expect(parentIdField).toBeDefined();

config = testEnvironment.config;
firestore = testEnvironment.firestore;
typesense = testEnvironment.typesense;
done();
});
});

afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
// delete the Firestore collection
// For subcollections, need special handling to clear parent collection. Deleting here triggers firebase functions
await firestore.recursiveDelete(firestore.collection(parentCollectionPath));

// Clear any previously created collections
try {
await typesense.collections(encodeURIComponent(config.typesenseCollectionName)).delete();
} catch (e) {
console.info(`${config.typesenseCollectionName} collection not found, proceeding...`);
}

// recreate the Typesense collection
await typesense.collections().create({
name: config.typesenseCollectionName,
fields: [{name: ".*", type: "auto"}],
enable_nested_fields: true,
});
});

afterAll(async () => {
// clean up the whole firebase app
await app.delete();
await testEnvironment.clearAllData();
});

describe("Backfill from dynamic subcollections", () => {
239 changes: 239 additions & 0 deletions test/support/testEnvironment.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
const {execSync, spawn} = require("child_process");
const firebase = require("firebase-admin");
const path = require("path");
const fs = require("fs");

const directConsole = require("console");

/**
* Load the default Firebase project ID from .firebaserc and set it to process.env.GCLOUD_PROJECT.
* Also set FIRESTORE_EMULATOR_HOST to the Firestore emulator host.
* @param {string} projectRootPath the path to the root of the project
* @throws {Error} - If the .firebaserc file is missing or the default project is not set.
*/
function loadFirebaseEnvironment(projectRootPath) {
try {
const firebaseRc = JSON.parse(fs.readFileSync(path.resolve(projectRootPath, ".firebaserc"), "utf8"));
const defaultProjectId = firebaseRc.projects?.default;

if (!defaultProjectId) {
throw new Error("No default project found in .firebaserc");
}

process.env.GCLOUD_PROJECT = defaultProjectId;
process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; // explcitly set emulator host, used for firebase-admin
} catch (error) {
console.error("Error loading .firebaserc:", error.message);
throw error;
}
}

/**
* Class that sets up a test environment for the Firestore Typesense Search extension.
* <p>
* Creates a new Firebase project, sets up the Firestore emulator, and
* configures the extension to use the Firestore emulator as the backend.
* <p>
* Also provides a Firestore client for tests to use, and a Typesense client
* that is configured to connect to the Typesense server running on localhost.
* <p>
* Additionally, provides a method to capture the emulator logs and write them
* to a file.
*/
class TestEnvironment {
// Global ouput all emulator logs
projectRootPath = path.resolve(__dirname, "../../");
firebaseEnvPath = "extensions/firestore-typesense-search.env.local";
shouldOutputAllEmulatorLogs = false;
dotenvPath = null;
dotenvConfig = null;

// Emulator vars
emulator = null;
capturedEmulatorLogs = "";
shouldLogEmulator = false;

// Test client case vars
config = null;
firebaseApp = null;
firestore = null;
typesense = null;

/**
* Initialize a test environment with a specific dotenv config and a flag for logging all emulator logs.
* @param {{dotenvConfig: string, debugLog: boolean}} config
* @param {string} config.dotenvConfig - path to the env file to use for the firebase emulator and test
* @param {boolean} config.debugLog - whether to log all emulator logs to console
*/
constructor({
dotenvPath,
dotenvConfig,
outputAllEmulatorLogs = false,
} = {}) {
this.dotenvPath = dotenvPath;
this.dotenvConfig = dotenvConfig;
this.shouldOutputAllEmulatorLogs = outputAllEmulatorLogs;

if (dotenvPath && dotenvConfig) {
throw new Error("Provide either 'dotenvPath' or 'dotenvConfig', not both.");
}
}

/**
* Set up a test environment for the Firestore Typesense Search extension.
* <p>
* Creates a new Firebase project, sets up the Firestore emulator, and
* configures the extension to use the Firestore emulator as the backend.
* <p>
* Also provides a Firestore client for tests to use, and a Typesense client
* that is configured to connect to the Typesense server running on localhost.
* <p>
* @param {function} done - callback to call when the test environment is set up
*/
setupTestEnvironment(done) {
directConsole.log("Setting Up Firebase emulator...");
if (this.dotenvPath) {
directConsole.log(`Copying ${this.dotenvPath} to ${this.firebaseEnvPath}...`);
execSync(
`cp -f ${this.dotenvPath} ${this.firebaseEnvPath}`,
);
} else if (this.dotenvConfig) {
fs.writeFileSync(this.firebaseEnvPath, this.dotenvConfig);
}

this.emulator = spawn(
"firebase",
[
"emulators:start",
"--only",
"functions,firestore,extensions",
],
{
stdio: ["pipe", "pipe", "pipe"],
env: {
...process.env,
NODE_OPTIONS: "--experimental-vm-modules",
FORCE_COLOR: "1",
},
},
);

// Listen for logs from the emulator
this.emulator.stdout.on("data", (data) => {
let logMessage = data.toString().trim();

if (this.shouldLogEmulator) {
try {
// eslint-disable-next-line no-control-regex
const stripAnsi = (str) => str.replace(/\x1b\[[0-9;]*m/g, "");
let flatLogMessage = stripAnsi(logMessage);

if (flatLogMessage.startsWith(">")) {
flatLogMessage = flatLogMessage.replace(/^>\s*/, ""); // Removes "> "
}

const parsedLog = JSON.parse(flatLogMessage);
logMessage = parsedLog.message;
} catch (e) {
// Not a JSON log, keep the original logMessage
}
this.capturedEmulatorLogs += logMessage + "\n";
}

if (this.shouldOutputAllEmulatorLogs) {
directConsole.log(logMessage);
}

if (logMessage.includes("All emulators ready")) { // Adjust to the actual readiness log message
directConsole.log("Emulator is ready");

try {
directConsole.log("Loading testing firebase config and modules...");
const dotenvResult = require("dotenv").config({path: path.resolve(this.projectRootPath, this.firebaseEnvPath)}); // load same .env as emulator
if (dotenvResult.error) {
throw dotenvResult.error;
}

this.config = require(path.resolve(this.projectRootPath, "functions/src/config.js"));
this.typesense = require(path.resolve(this.projectRootPath, "functions/src/createTypesenseClient.js"))();

loadFirebaseEnvironment(this.projectRootPath);

directConsole.log("Initializing Firebase Admin Client...");
this.firebaseApp = firebase.initializeApp({
databaseURL: `${process.env.FIRESTORE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
projectId: process.env.GCLOUD_PROJECT,
});
this.firestore = this.firebaseApp.firestore();

directConsole.log("Test environment initialization complete");
this.shouldLogEmulator = true;
done();
} catch (e) {
directConsole.error("Error loading environment variables:", e);
done(e);
}
}
});

this.emulator.stderr.on("data", (data) => {
directConsole.error(data.toString());
});

this.emulator.on("close", (code) => {
if (code !== 0) {
done(new Error("Emulator exited unexpectedly."));
}
});
}

/**
* Reset the captured emulator logs.
* This will clear the logs collected from the Firebase Emulator.
*/
resetCapturedEmulatorLogs() {
this.capturedEmulatorLogs = "";
}

/**
* Clean up the test environment after all tests have run.
* This will shut down the Firebase Emulator and clean up any data that was created during the tests.
*/
async teardownTestEnvironment() {
this.shouldLogEmulator = false;
if (this.emulator) {
this.emulator.kill("SIGINT");
await new Promise((resolve) => this.emulator.on("exit", resolve));
}

await this.firebaseApp.delete();
}

/**
* Clears all data in Firestore and Typesense before a test is run.
* This will delete the Firestore collection and any data in it, and
* delete the Typesense collection and any data in it.
*/
async clearAllData() {
try {
await this.firestore.recursiveDelete(this.firestore.collection(this.config.firestoreCollectionPath));
} catch (e) {
directConsole.info(`${this.config.firestoreCollectionPath} collection not found, proceeding...`);
}

try {
await this.typesense.collections(encodeURIComponent(this.config.typesenseCollectionName)).delete();
} catch (e) {
directConsole.info(`${this.config.typesenseCollectionName} collection not found, proceeding...`);
}
await this.typesense.collections().create({
name: this.config.typesenseCollectionName,
fields: [{name: ".*", type: "auto"}],
enable_nested_fields: true,
});
}
}

module.exports = {
TestEnvironment,
};
193 changes: 193 additions & 0 deletions test/writeLogging.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
const {TestEnvironment} = require("./support/testEnvironment");

describe("indexOnWriteLogging - when shouldLogTypesenseInserts is false", () => {
let testEnvironment;

beforeAll((done) => {
try {
testEnvironment = new TestEnvironment({
dotenvPath: "extensions/test-params-flatten-nested-true.local.env",
});
testEnvironment.setupTestEnvironment(done);
} catch (e) {
console.error(e);
done(e);
}
});

afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
await testEnvironment.clearAllData();
});

describe("testing onWrite logging", () => {
it("logs only itemId", async () => {
const docData = {
author: "value1",
title: "value2",
};
testEnvironment.resetCapturedEmulatorLogs();
const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData);

await new Promise((r) => setTimeout(r, 5000));
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Upserting document ${docRef.id}`,
);
});
});

describe("testing backfill logging", () => {
it("backfills existing Firestore data in all collections to Typesense", async () => {
const book = {
author: "Author A",
title: "Title X",
country: "USA",
};
const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book);
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// The above will automatically add the document to Typesense,
// so delete it so we can test backfill
await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
await testEnvironment.typesense.collections().create({
name: testEnvironment.config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});

await testEnvironment.firestore
.collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
.doc("backfill")
.set({trigger: true});
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// Check that the data was backfilled
const typesenseDocsStr = await testEnvironment.typesense
.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
expect(typesenseDocs.length).toBe(1);
expect(typesenseDocs[0]).toStrictEqual({
id: firestoreDoc.id,
author: book.author,
title: book.title,
});

// Check that the backfill log was written
expect(testEnvironment.capturedEmulatorLogs).not.toContain(
"Backfilling document",
);

expect(testEnvironment.capturedEmulatorLogs).toContain(
"Imported 1 documents into Typesense",
);
});
});
});

describe("indexOnWriteLogging - when shouldLogTypesenseInserts is true", () => {
let testEnvironment;

beforeAll((done) => {
testEnvironment = new TestEnvironment({
dotenvConfig: `
LOCATION=us-central1
FIRESTORE_COLLECTION_PATH=books
FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref
FLATTEN_NESTED_DOCUMENTS=true
LOG_TYPESENSE_INSERTS=true
TYPESENSE_HOSTS=localhost
TYPESENSE_PORT=8108
TYPESENSE_PROTOCOL=http
TYPESENSE_COLLECTION_NAME=books_firestore/1
TYPESENSE_API_KEY=xyz
`,
});
testEnvironment.setupTestEnvironment(done);
});

afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
await testEnvironment.clearAllData();
});

describe("testing basic onWrite logging", () => {
it("logs detailed inserts", async () => {
const docData = {
author: "value1",
title: "value2",
};

testEnvironment.resetCapturedEmulatorLogs();
const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData);

await new Promise((r) => setTimeout(r, 5000));
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Upserting document ${JSON.stringify({...docData, id: docRef.id})}`,
);
});
});

describe("testing backfill logging", () => {
it("backfills existing Firestore data in all collections to Typesense", async () => {
const book = {
author: "Author A",
title: "Title X",
country: "USA",
};
const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book);
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// The above will automatically add the document to Typesense,
// so delete it so we can test backfill
await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
await testEnvironment.typesense.collections().create({
name: testEnvironment.config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});

await testEnvironment.firestore
.collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
.doc("backfill")
.set({trigger: true});
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// Check that the data was backfilled
const typesenseDocsStr = await testEnvironment.typesense
.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
expect(typesenseDocs.length).toBe(1);
const expectedResult = {
author: book.author,
title: book.title,
id: firestoreDoc.id,
};
expect(typesenseDocs[0]).toStrictEqual(expectedResult);

// Check that the backfill log was written
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Backfilling document ${JSON.stringify(expectedResult)}`,
);

expect(testEnvironment.capturedEmulatorLogs).toContain(
"Imported 1 documents into Typesense",
);
});
});
});

0 comments on commit 3d66da6

Please sign in to comment.