Merge pull request #8 from jembi/CU-86c11rv16_automated-tests

Cu 86c11rv16 automated tests

.env.dev

@@ -17,7 +17,7 @@ BODY_SIZE_LIMIT=50mb
 MINIO_ENDPOINT=localhost
 MINIO_PORT=9000
 MINIO_USE_SSL=false
-MINIO_BUCKET=climate-mediator
+MINIO_BUCKETS=climate-mediator
 MINIO_ACCESS_KEY=tCroZpZ3usDUcvPM3QT6
 MINIO_SECRET_KEY=suVjMHUpVIGyWx8fFJHTiZiT88dHhKgVpzvYTOKK
 MINIO_PREFIX=

.github/workflows/tests.yml

@@ -8,7 +8,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: Install modules
         run: npm i
+
+      # Set up Docker Compose
+      - name: Set up Docker Compose
+        uses: docker/setup-buildx-action@v3
+
+      # Start Docker Compose services
+      - name: Start services with Docker Compose
+        run: docker compose up -d
 
       - name: Run tests
         run: npm run test
-
+
+      - name: Tear down services
+        if: always()
+        run: docker compose down

.gitignore

@@ -132,5 +132,4 @@ dist
 # Minio
 minio
 fs
-tmp
-sample
+tmp

README.md

@@ -1,4 +1,5 @@
-# climate-mediator
+# Climate Mediator
+
 Processes climate-related data as an example of unstructured data handling
 
 ## Local Development Setup

docker-compose.yml

@@ -5,5 +5,39 @@ services:
       - 9000:9000
       - 9001:9001
     command: server /data --console-address ":9001"
-    volumes:
-      - ./minio:/data
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:23.8.14.6
+    ports:
+      - 8123:8123
+      - 9002:9000
+    environment:
+      - CLICKHOUSE_PASSWORD=dev_password_only
+
+  openhim-console:
+    image: jembi/openhim-console:v1.18.2
+    ports:
+      - 80:80
+    networks:
+      - openhim
+
+  openhim-core:
+    image: jembi/openhim-core:v8.5.1
+    environment:
+      - mongo_url=mongodb://mongo-db:27017/openhim
+    ports:
+      - 8080:8080
+      - 5000:5000
+      - 5001:5001
+    networks:
+      - openhim
+
+  mongo-db:
+    container_name: mongo-db
+    image: mongo:4.0
+    networks:
+      - openhim
+    restart: unless-stopped
+
+networks:
+  openhim:

package.json

@@ -20,7 +20,7 @@
   "devDependencies": {
     "@types/chai": "^4.3.11",
     "@types/express": "^4.17.21",
-    "@types/mocha": "^10.0.6",
+    "@types/mocha": "^10.0.9",
     "@types/multer": "^1.4.12",
     "@types/nock": "^11.1.0",
     "@types/node": "^22.9.0",

src/config/config.ts

@@ -27,6 +27,7 @@ export const getConfig = () => {
       endPoint: process.env.MINIO_ENDPOINT || 'localhost',
       port: process.env.MINIO_PORT ? parseInt(process.env.MINIO_PORT) : 9000,
       useSSL: process.env.MINIO_USE_SSL === 'true' ? true : false,
+      buckets: process.env.MINIO_BUCKETS || 'climate-mediator',
       bucket: process.env.MINIO_BUCKET || 'climate-mediator',
       bucketRegion: process.env.MINIO_BUCKET_REGION || 'us-east-1',
       accessKey: process.env.MINIO_ACCESS_KEY || 'tCroZpZ3usDUcvPM3QT6',

src/index.ts

@@ -1,99 +1,24 @@
 import express from 'express';
-import * as Minio from 'minio';
 import path from 'path';
 import { getConfig } from './config/config';
 import logger from './logger';
 import routes from './routes/index';
 import { setupMediator } from './openhim/openhim';
-import { validateJsonFile, getCsvHeaders } from './utils/file-validators';
-import { readFile, rm } from 'fs/promises';
-import { createTable, flattenJson, insertFromS3 } from './utils/clickhouse';
+import {
+  createMinioBucketListeners,
+} from './utils/minioClient';
 
 const app = express();
 
 app.use('/', routes);
 
-if (getConfig().runningMode !== 'testing') {
-  app.listen(getConfig().port, () => {
-    logger.info(`Server is running on port - ${getConfig().port}`);
+createMinioBucketListeners();
 
-    if (getConfig().registerMediator) {
-      setupMediator(path.resolve(__dirname, './openhim/mediatorConfig.json'));
-    }
-  });
-}
+app.listen(getConfig().port, () => {
+  logger.info(`Server is running on port - ${getConfig().port}`);
 
-async function setupMinio() {
-  const { bucket, endPoint, port, useSSL, accessKey, secretKey, prefix, suffix } =
-    getConfig().minio;
-
-  const minioClient = new Minio.Client({
-    endPoint,
-    port,
-    useSSL,
-    accessKey,
-    secretKey,
-  });
-
-  try {
-    // Test connection by attempting to list buckets
-    await minioClient.listBuckets();
-    logger.info(`Successfully connected to Minio at ${endPoint}:${port}/${bucket}`);
-  } catch (error) {
-    logger.error(`Failed to connect to Minio: ${error}`);
-    throw error;
+  if (getConfig().runningMode !== 'testing' && getConfig().registerMediator) {
+    setupMediator(path.resolve(__dirname, './openhim/mediatorConfig.json'));
   }
-  const listener = minioClient.listenBucketNotification(bucket, prefix, suffix, [
-    's3:ObjectCreated:*',
-  ]);
-
-  listener.on('notification', async (notification) => {
-    //@ts-ignore
-    const file = notification.s3.object.key;
-
-    //@ts-ignore
-    const tableName = notification.s3.bucket.name;
-
-    //@ts-ignore
-    minioClient.fGetObject(bucket, file, `tmp/${file}`, async (err) => {
-      if (err) {
-        logger.error(err);
-      } else {
-        const fileBuffer = await readFile(`tmp/${file}`);
-
-        //get the file extension
-        const extension = file.split('.').pop();
-        logger.info(`File Downloaded - Type: ${extension}`);
-
-        if (extension === 'json' && validateJsonFile(fileBuffer)) {
-          // flatten the json and pass it to clickhouse
-          //const fields = flattenJson(JSON.parse(fileBuffer.toString()));
-          //await createTable(fields, tableName);
-          logger.warn(`File type not currently supported- ${extension}`);
-        } else if (extension === 'csv' && getCsvHeaders(fileBuffer)) {
-          //get the first line of the csv file
-          const fields = (await readFile(`tmp/${file}`, 'utf8')).split('\n')[0].split(',');
-
-          // Construct the S3-style URL for the file
-          const minioUrl = `http://${endPoint}:${port}/${bucket}/${file}`;
-
-          // First create table
-          await createTable(fields, tableName);
-          logger.info(`Inserting data into ${tableName} from ${minioUrl}`);
-
-          // Insert data into clickhouse
-          await insertFromS3(tableName, minioUrl, {
-            accessKey,
-            secretKey
-          });
-
-        } else {
-          logger.warn(`Unknown file type - ${extension}`);
-        }
-        await rm(`tmp/${file}`);
-      }
-    });
-  });
-}
+});
 
-setupMinio();

src/openhim/mediatorConfig.json

@@ -13,8 +13,7 @@
           "host": "climate-mediator",
           "port": "3000",
           "primary": true,
-          "type": "http",
-          "path": "/climate/upload"
+          "type": "http"
         }
       ],
       "allow": [

src/routes/index.ts

@@ -5,6 +5,7 @@ import { getCsvHeaders } from '../utils/file-validators';
 import logger from '../logger';
 import fs from 'fs/promises';
 import path from 'path';
+import e from 'express';
 import { uploadToMinio } from '../utils/minioClient';
 
 // Constants

src/utils/clickhouse.ts

@@ -5,6 +5,7 @@ import logger from '../logger';
 const { clickhouse } = getConfig();
 const { url, password } = clickhouse;
 
+
 export async function createTable(fields: string[], tableName: string) {
   const client = createClient({
     url,
@@ -26,14 +27,14 @@ export async function createTable(fields: string[], tableName: string) {
   }
 
   try {
-    console.debug(`Creating table ${normalizedTableName} with fields ${fields.join(', ')}`);
+    logger.debug(`Creating table ${normalizedTableName} with fields ${fields.join(', ')}`);
     const result = await client.query({
       query: generateDDL(fields, normalizedTableName),
     });
-    console.log('Table created successfully');
+    logger.info(`Table ${normalizedTableName} created successfully`);
   } catch (error) {
-    console.log('Error checking/creating table');
-    console.error(error);
+    logger.error(`Error checking/creating table ${normalizedTableName}`);
+    logger.debug(JSON.stringify(error));
     return false;
   }
 
@@ -42,13 +43,7 @@ export async function createTable(fields: string[], tableName: string) {
 }
 
 export function generateDDL(fields: string[], tableName: string) {
-  return `CREATE TABLE IF NOT EXISTS ${tableName} (
-    table_id UUID DEFAULT generateUUIDv4(),
-    ${fields.map((field) => `${field} VARCHAR`).join(', ')}
-  )
-  ENGINE = MergeTree
-  ORDER BY (table_id)
-  `;
+  return `CREATE TABLE ${tableName} (table_id UUID DEFAULT generateUUIDv4(),${fields.map((field) => `${field} VARCHAR`).join(', ')}) ENGINE = MergeTree ORDER BY (table_id)`;
 }
 
 export function flattenJson(json: any, prefix = ''): string[] {

src/utils/file-validators.ts

@@ -11,8 +11,10 @@ export function validateJsonFile(file: Buffer) {
 export function getCsvHeaders(file: Buffer) {
   //convert the buffer to a string
   const csv = file.toString();
+  //check if the new line character is \n or \r\n
+  const newLineChar = csv.includes('\r\n') ? '\r\n' : '\n';
   //get the first line of the csv file
-  const firstLine = csv.split('\n')[0];
+  const firstLine = csv.split(newLineChar)[0];
   //split the first line by commas
   const columns = firstLine.split(',');