Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make debug logging of typesense imports expicit and configurable #101

Merged
merged 4 commits into from
Jan 21, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add test cases for logging tests using TestEnvironment class
jsasitorn committed Dec 30, 2024
commit 919cd5372046a85978091a0e615b62d198723f29
2 changes: 1 addition & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ module.exports = {
jest: true,
},
parserOptions: {
ecmaVersion: 2020,
ecmaVersion: 2022,
},
extends: ["eslint:recommended", "google"],
rules: {
5 changes: 3 additions & 2 deletions package.json
tharropoulos marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -4,10 +4,11 @@
"scripts": {
"emulator": "cross-env DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:start --import=emulator_data",
"export": "firebase emulators:export emulator_data",
"test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3",
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testPathIgnorePatterns=\"WithoutFlattening\" --testPathIgnorePatterns=\"Subcollection\"'",
"test": "npm run test-part-1 && npm run test-part-2 && npm run test-part-3 && npm run test-part-4",
"test-part-1": "cp -f extensions/test-params-flatten-nested-true.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-true.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithFlattening\" --testRegex=\"backfill.spec\"'",
"test-part-2": "cp -f extensions/test-params-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"WithoutFlattening\"'",
"test-part-3": "cp -f extensions/test-params-subcategory-flatten-nested-false.local.env extensions/firestore-typesense-search.env.local && cross-env NODE_OPTIONS=--experimental-vm-modules DOTENV_CONFIG=extensions/test-params-subcategory-flatten-nested-false.local.env firebase emulators:exec --only functions,firestore,extensions 'jest --testRegex=\"Subcollection\"'",
"test-part-4": "jest --testRegex=\"writeLogging\" --detectOpenHandles",
"typesenseServer": "docker compose up",
"lint:fix": "eslint . --fix",
"lint": "eslint ."
239 changes: 239 additions & 0 deletions test/support/testEnvironment.js
tharropoulos marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
const {execSync, spawn} = require("child_process");
const firebase = require("firebase-admin");
const path = require("path");
const fs = require("fs");

const directConsole = require("console");

/**
* Load the default Firebase project ID from .firebaserc and set it to process.env.GCLOUD_PROJECT.
* Also set FIRESTORE_EMULATOR_HOST to the Firestore emulator host.
* @param {string} projectRootPath the path to the root of the project
* @throws {Error} - If the .firebaserc file is missing or the default project is not set.
*/
function loadFirebaseEnvironment(projectRootPath) {
try {
const firebaseRc = JSON.parse(fs.readFileSync(path.resolve(projectRootPath, ".firebaserc"), "utf8"));
const defaultProjectId = firebaseRc.projects?.default;

if (!defaultProjectId) {
throw new Error("No default project found in .firebaserc");
}

process.env.GCLOUD_PROJECT = defaultProjectId;
process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; // explcitly set emulator host, used for firebase-admin
} catch (error) {
console.error("Error loading .firebaserc:", error.message);
throw error;
}
}

/**
* Class that sets up a test environment for the Firestore Typesense Search extension.
* <p>
* Creates a new Firebase project, sets up the Firestore emulator, and
* configures the extension to use the Firestore emulator as the backend.
* <p>
* Also provides a Firestore client for tests to use, and a Typesense client
* that is configured to connect to the Typesense server running on localhost.
* <p>
* Additionally, provides a method to capture the emulator logs and write them
* to a file.
*/
class TestEnvironment {
// Global ouput all emulator logs
projectRootPath = path.resolve(__dirname, "../../");
firebaseEnvPath = "extensions/firestore-typesense-search.env.local";
shouldOutputAllEmulatorLogs = false;
dotenvPath = null;
dotenvConfig = null;

// Emulator vars
emulator = null;
capturedEmulatorLogs = "";
shouldLogEmulator = false;

// Test client case vars
config = null;
firebaseApp = null;
firestore = null;
typesense = null;

/**
* Initialize a test environment with a specific dotenv config and a flag for logging all emulator logs.
* @param {{dotenvConfig: string, debugLog: boolean}} config
* @param {string} config.dotenvConfig - path to the env file to use for the firebase emulator and test
* @param {boolean} config.debugLog - whether to log all emulator logs to console
*/
constructor({
dotenvPath,
dotenvConfig,
outputAllEmulatorLogs = false,
} = {}) {
this.dotenvPath = dotenvPath;
this.dotenvConfig = dotenvConfig;
this.shouldOutputAllEmulatorLogs = outputAllEmulatorLogs;

if (dotenvPath && dotenvConfig) {
throw new Error("Provide either 'dotenvPath' or 'dotenvConfig', not both.");
}
}

/**
* Set up a test environment for the Firestore Typesense Search extension.
* <p>
* Creates a new Firebase project, sets up the Firestore emulator, and
* configures the extension to use the Firestore emulator as the backend.
* <p>
* Also provides a Firestore client for tests to use, and a Typesense client
* that is configured to connect to the Typesense server running on localhost.
* <p>
* @param {function} done - callback to call when the test environment is set up
*/
setupTestEnvironment(done) {
directConsole.log("Setting Up Firebase emulator...");
if (this.dotenvPath) {
directConsole.log(`Copying ${this.dotenvPath} to ${this.firebaseEnvPath}...`);
execSync(
`cp -f ${this.dotenvPath} ${this.firebaseEnvPath}`,
);
} else if (this.dotenvConfig) {
fs.writeFileSync(this.firebaseEnvPath, this.dotenvConfig);
}

this.emulator = spawn(
"firebase",
[
"emulators:start",
"--only",
"functions,firestore,extensions",
],
{
stdio: ["pipe", "pipe", "pipe"],
env: {
...process.env,
NODE_OPTIONS: "--experimental-vm-modules",
FORCE_COLOR: "1",
},
},
);

// Listen for logs from the emulator
this.emulator.stdout.on("data", (data) => {
let logMessage = data.toString().trim();

if (this.shouldLogEmulator) {
try {
// eslint-disable-next-line no-control-regex
const stripAnsi = (str) => str.replace(/\x1b\[[0-9;]*m/g, "");
let flatLogMessage = stripAnsi(logMessage);

if (flatLogMessage.startsWith(">")) {
flatLogMessage = flatLogMessage.replace(/^>\s*/, ""); // Removes "> "
}

const parsedLog = JSON.parse(flatLogMessage);
logMessage = parsedLog.message;
} catch (e) {
// Not a JSON log, keep the original logMessage
}
this.capturedEmulatorLogs += logMessage + "\n";
}

if (this.shouldOutputAllEmulatorLogs) {
directConsole.log(logMessage);
}

if (logMessage.includes("All emulators ready")) { // Adjust to the actual readiness log message
directConsole.log("Emulator is ready");

try {
directConsole.log("Loading testing firebase config and modules...");
const dotenvResult = require("dotenv").config({path: path.resolve(this.projectRootPath, this.firebaseEnvPath)}); // load same .env as emulator
if (dotenvResult.error) {
throw dotenvResult.error;
}

this.config = require(path.resolve(this.projectRootPath, "functions/src/config.js"));
this.typesense = require(path.resolve(this.projectRootPath, "functions/src/createTypesenseClient.js"))();

loadFirebaseEnvironment(this.projectRootPath);

directConsole.log("Initializing Firebase Admin Client...");
this.firebaseApp = firebase.initializeApp({
databaseURL: `${process.env.FIRESTORE_EMULATOR_HOST}?ns=${process.env.GCLOUD_PROJECT}`,
projectId: process.env.GCLOUD_PROJECT,
});
this.firestore = this.firebaseApp.firestore();

directConsole.log("Test environment initialization complete");
this.shouldLogEmulator = true;
done();
} catch (e) {
directConsole.error("Error loading environment variables:", e);
done(e);
}
}
});

this.emulator.stderr.on("data", (data) => {
directConsole.error(data.toString());
});

this.emulator.on("close", (code) => {
if (code !== 0) {
done(new Error("Emulator exited unexpectedly."));
}
});
}

/**
* Reset the captured emulator logs.
* This will clear the logs collected from the Firebase Emulator.
*/
resetCapturedEmulatorLogs() {
this.capturedEmulatorLogs = "";
}

/**
* Clean up the test environment after all tests have run.
* This will shut down the Firebase Emulator and clean up any data that was created during the tests.
*/
async teardownTestEnvironment() {
this.shouldLogEmulator = false;
if (this.emulator) {
this.emulator.kill("SIGINT");
await new Promise((resolve) => this.emulator.on("exit", resolve));
}

await this.firebaseApp.delete();
}

/**
* Clears all data in Firestore and Typesense before a test is run.
* This will delete the Firestore collection and any data in it, and
* delete the Typesense collection and any data in it.
*/
async clearAllData() {
try {
await this.firestore.recursiveDelete(this.firestore.collection(this.config.firestoreCollectionPath));
} catch (e) {
directConsole.info(`${this.config.firestoreCollectionPath} collection not found, proceeding...`);
}

try {
await this.typesense.collections(encodeURIComponent(this.config.typesenseCollectionName)).delete();
} catch (e) {
directConsole.info(`${this.config.typesenseCollectionName} collection not found, proceeding...`);
}
await this.typesense.collections().create({
name: this.config.typesenseCollectionName,
fields: [{name: ".*", type: "auto"}],
enable_nested_fields: true,
});
}
}

module.exports = {
TestEnvironment,
};
193 changes: 193 additions & 0 deletions test/writeLogging.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
const {TestEnvironment} = require("./support/testEnvironment");

describe("indexOnWriteLogging - when shouldLogTypesenseInserts is false", () => {
let testEnvironment;

beforeAll((done) => {
try {
testEnvironment = new TestEnvironment({
dotenvPath: "extensions/test-params-flatten-nested-true.local.env",
});
testEnvironment.setupTestEnvironment(done);
} catch (e) {
console.error(e);
done(e);
}
});

afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
await testEnvironment.clearAllData();
});

describe("testing onWrite logging", () => {
it("logs only itemId", async () => {
const docData = {
author: "value1",
title: "value2",
};
testEnvironment.resetCapturedEmulatorLogs();
const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData);

await new Promise((r) => setTimeout(r, 5000));
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Upserting document ${docRef.id}`,
);
});
});

describe("testing backfill logging", () => {
it("backfills existing Firestore data in all collections to Typesense", async () => {
const book = {
author: "Author A",
title: "Title X",
country: "USA",
};
const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book);
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// The above will automatically add the document to Typesense,
// so delete it so we can test backfill
await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
await testEnvironment.typesense.collections().create({
name: testEnvironment.config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});

await testEnvironment.firestore
.collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
.doc("backfill")
.set({trigger: true});
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// Check that the data was backfilled
const typesenseDocsStr = await testEnvironment.typesense
.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
expect(typesenseDocs.length).toBe(1);
expect(typesenseDocs[0]).toStrictEqual({
id: firestoreDoc.id,
author: book.author,
title: book.title,
});

// Check that the backfill log was written
expect(testEnvironment.capturedEmulatorLogs).not.toContain(
"Backfilling document",
);

expect(testEnvironment.capturedEmulatorLogs).toContain(
"Imported 1 documents into Typesense",
);
});
});
});

describe("indexOnWriteLogging - when shouldLogTypesenseInserts is true", () => {
let testEnvironment;

beforeAll((done) => {
testEnvironment = new TestEnvironment({
dotenvConfig: `
LOCATION=us-central1
FIRESTORE_COLLECTION_PATH=books
FIRESTORE_COLLECTION_FIELDS=author,title,rating,isAvailable,location,createdAt,nested_field,tags,nullField,ref
FLATTEN_NESTED_DOCUMENTS=true
LOG_TYPESENSE_INSERTS=true
TYPESENSE_HOSTS=localhost
TYPESENSE_PORT=8108
TYPESENSE_PROTOCOL=http
TYPESENSE_COLLECTION_NAME=books_firestore/1
TYPESENSE_API_KEY=xyz
`,
});
testEnvironment.setupTestEnvironment(done);
});

afterAll(async () => {
await testEnvironment.teardownTestEnvironment();
});

beforeEach(async () => {
await testEnvironment.clearAllData();
});

describe("testing basic onWrite logging", () => {
it("logs detailed inserts", async () => {
const docData = {
author: "value1",
title: "value2",
};

testEnvironment.resetCapturedEmulatorLogs();
const docRef = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(docData);

await new Promise((r) => setTimeout(r, 5000));
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Upserting document ${JSON.stringify({...docData, id: docRef.id})}`,
);
});
});

describe("testing backfill logging", () => {
it("backfills existing Firestore data in all collections to Typesense", async () => {
const book = {
author: "Author A",
title: "Title X",
country: "USA",
};
const firestoreDoc = await testEnvironment.firestore.collection(testEnvironment.config.firestoreCollectionPath).add(book);
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// The above will automatically add the document to Typesense,
// so delete it so we can test backfill
await testEnvironment.typesense.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName)).delete();
await testEnvironment.typesense.collections().create({
name: testEnvironment.config.typesenseCollectionName,
fields: [
{name: ".*", type: "auto"},
],
});

await testEnvironment.firestore
.collection(testEnvironment.config.typesenseBackfillTriggerDocumentInFirestore.split("/")[0])
.doc("backfill")
.set({trigger: true});
// Wait for firestore cloud function to write to Typesense
await new Promise((r) => setTimeout(r, 2000));

// Check that the data was backfilled
const typesenseDocsStr = await testEnvironment.typesense
.collections(encodeURIComponent(testEnvironment.config.typesenseCollectionName))
.documents()
.export();
const typesenseDocs = typesenseDocsStr.split("\n").map((s) => JSON.parse(s));
expect(typesenseDocs.length).toBe(1);
const expectedResult = {
author: book.author,
title: book.title,
id: firestoreDoc.id,
};
expect(typesenseDocs[0]).toStrictEqual(expectedResult);

// Check that the backfill log was written
expect(testEnvironment.capturedEmulatorLogs).toContain(
`Backfilling document ${JSON.stringify(expectedResult)}`,
);

expect(testEnvironment.capturedEmulatorLogs).toContain(
"Imported 1 documents into Typesense",
);
});
});
});