feat: add simple integration test

.env.example

@@ -0,0 +1,5 @@
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=pgrag
+DB_USER=pgrag
+DB_PASSWORD=password

.gitignore

@@ -8,3 +8,7 @@ node_modules
 
 # OS X
 .DS_Store
+
+.env.dev
+.env.test	
+.env

README.md

@@ -27,6 +27,15 @@ PG Rag is not meant to provide a full fledged advanced RAG for any document and
 
 pg-rag uses the (Officeparser)[https://github.com/harshankur/officeParser] library to parse files. Lookup this library to know what kind of files are supported.
 
+## Development & running tests
+
+Pre-requisites:
+- PostgreSQL with the pgvector extension available/installed
+- Ollama running locally with `mistral` model installed
+
+Copy and adapt the `.env.example` file to `.env.dev` and adapt it.
+Ensure your PostgreSQL database user is able to install the pgvector extension or pre-install it.
+
 ## Improvements/TODO
 
 - Summarize documents and vectorize that

docs/examples/ollama.ts

@@ -20,7 +20,7 @@ import { fileURLToPath } from 'url';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 const embeddings = new OllamaEmbeddings({
-  model: "mistral",
+  model: "llama3",
   baseUrl: "http://127.0.0.1:11434",
 });
 
@@ -36,13 +36,13 @@ async function run() {
   const pdf = fs.readFileSync(path.join(__dirname, './example.pdf'))
 
   const pgRag = await PgRag.init({dbPool: pool, embeddings, resetDB: true})
-  await pgRag.saveDocument({data: pdf, fileName: 'example.pdf'})
+  const jobId = await pgRag.saveDocument({data: pdf, name: 'example.pdf'})
 
-  setTimeout(async () => {
-    const res = await pgRag.search({prompt: 'Tell me about Sparse Vector Representation'})
-    console.log(res)
-    process.exit()
-  }, 5000)
+  await pgRag.waitForDocumentProcessed(jobId!)
+
+  const res = await pgRag.search({prompt: 'Tell me about Sparse Vector Representation'})
+  console.log('Search response', res)
+  process.exit() // workers prevent auto-shutdown
 }
 
 run()

package.json

@@ -6,7 +6,7 @@
   "type": "module",
   "scripts": {
     "lint": "eslint .",
-    "test": "echo \"Error: no test specified\" && exit 0",
+    "test": "find ./src -type f -name '*.test.ts' | xargs node --import=tsx --test",
     "prepare": "husky"
   },
   "repository": {
@@ -25,6 +25,7 @@
     "@commitlint/config-conventional": "^19.2.2",
     "@eslint/js": "^9.2.0",
     "@types/pg": "^8.11.5",
+    "dotenv": "^16.4.5",
     "eslint": "^8.57.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.1.3",
@@ -43,6 +44,7 @@
     "@langchain/openai": "^0.0.28",
     "@nearform/sql": "^1.10.5",
     "@types/pdf-parse": "^1.1.4",
+    "file-type": "^19.0.0",
     "langchain": "^0.1.36",
     "officeparser": "^4.1.1",
     "pdf-parse": "^1.1.1",