diff --git a/.gitignore b/.gitignore
index 58b3ce5..c233b6a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ target/
/adapters/hive/hive_data/
/adapters/hive/mariadb_data/
/adapters/hive/beeline.conf/
+/adapters/hive/hadoop_data/
diff --git a/Cargo.lock b/Cargo.lock
index 0df78b9..ba3bf32 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1091,6 +1091,7 @@ dependencies = [
"http",
"indexmap 2.2.6",
"jni",
+ "log",
"mime",
"ndc-models",
"ndc-sdk",
diff --git a/adapters/hive/docker-compose.yaml b/adapters/hive/docker-compose.yaml
index 8027822..5a4c7c9 100644
--- a/adapters/hive/docker-compose.yaml
+++ b/adapters/hive/docker-compose.yaml
@@ -19,24 +19,102 @@ services:
volumes:
- ./hive_data:/opt/hive
- ./hadoop_data:/opt/hadoop
+ - ./hive-site.xml:/opt/hive/conf/hive-site.xml
command: |
bash -c '
set -e
+ export HADOOP_HOME=/opt/hadoop
+ export HIVE_HOME=/opt/hive
+ export PATH=$PATH:/opt/hadoop/bin:/opt/hive/bin
+
+ # Create a script to set environment variables
+ cat << EOF > /tmp/set_env.sh
+ export HADOOP_HOME=/opt/hadoop
+ export HIVE_HOME=/opt/hive
+ export PATH=\$PATH:/opt/hadoop/bin:/opt/hive/bin
+ EOF
+
+ # Source the environment script in .bashrc
+ echo "source /tmp/set_env.sh" >> ~/.bashrc
+
+ # Source the environment script for the current session
+ source /tmp/set_env.sh
+
apt-get update && apt-get install -y wget
- # ... (previous wget and extraction commands remain the same)
+
+ # Download and extract Hive if not already present
+ if [ ! -f "/opt/hive/bin/hive" ]; then
+ for i in {1..3}; do
+ wget -O apache-hive-3.1.3-bin.tar.gz https://archive.apache.org/dist/hive/hive-3.1.3/apache-hive-3.1.3-bin.tar.gz
+ if tar -tzf apache-hive-3.1.3-bin.tar.gz &> /dev/null; then
+ break
+ else
+ echo "Download corrupted, retrying..."
+ rm apache-hive-3.1.3-bin.tar.gz
+ fi
+ done
+ tar -xzf apache-hive-3.1.3-bin.tar.gz
+ # Move everything except the conf directory
+ mv apache-hive-3.1.3-bin/bin apache-hive-3.1.3-bin/lib apache-hive-3.1.3-bin/scripts /opt/hive/
+ # Only copy conf files that dont already exist
+ cp -n apache-hive-3.1.3-bin/conf/* /opt/hive/conf/
+ rm -rf apache-hive-3.1.3-bin apache-hive-3.1.3-bin.tar.gz
+ fi
+
+ # Ensure Hive lib directory exists
+ mkdir -p /opt/hive/lib
+
+ # Download and extract Hadoop if not already present
+ if [ ! -f "$HADOOP_HOME/bin/hadoop" ]; then
+ for i in {1..3}; do
+ wget -O hadoop-3.3.6.tar.gz https://archive.apache.org/dist/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz
+ if tar -tzf hadoop-3.3.6.tar.gz &> /dev/null; then
+ break
+ else
+ echo "Download corrupted, retrying..."
+ rm hadoop-3.3.6.tar.gz
+ fi
+ done
+ tar -xzf hadoop-3.3.6.tar.gz
+ mv hadoop-3.3.6/* $HADOOP_HOME/
+ rm -rf hadoop-3.3.6 hadoop-3.3.6.tar.gz
+ fi
+
+ # Download MySQL connector if not present
+ if [ ! -f "/opt/hive/lib/mysql-connector-java.jar" ]; then
+ wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar -O /opt/hive/lib/mysql-connector-java.jar
+ fi
+
+ # Copy Hadoop JARs to Hive lib
+ find $HADOOP_HOME -name "*.jar" -exec cp {} /opt/hive/lib/ \;
+
+ # Ensure hive-exec-*.jar is present
+ if [ ! -f "/opt/hive/lib/hive-exec-*.jar" ]; then
+ echo "hive-exec-*.jar is missing. Hive installation may be incomplete."
+ exit 1
+ fi
+
echo "Waiting for MariaDB to be fully ready..."
- sleep 15
+ sleep 30 # Increased wait time
echo "Running schematool..."
- /opt/hive/bin/schematool -dbType mysql -initSchema --verbose
+ /opt/hive/bin/schematool -dbType mysql -initSchema --verbose || {
+ echo "Schema initialization failed. Checking for file casing issues..."
+ if [ -f "/var/lib/mysql/metastore/KEY_CONSTRAINTS.frm" ]; then
+ mv /var/lib/mysql/metastore/KEY_CONSTRAINTS.frm /var/lib/mysql/metastore/key_constraints.frm
+ echo "File casing corrected. Retrying schema initialization..."
+ /opt/hive/bin/schematool -dbType mysql -initSchema --verbose
+ else
+ echo "File casing issue not found. Please check the logs for more details."
+ exit 1
+ fi
+ }
echo "Schematool completed. Starting Hive metastore..."
/opt/hive/bin/hive --service metastore
'
environment:
HADOOP_HOME: /opt/hadoop
- HIVE_CONF_javax_jdo_option_ConnectionURL: jdbc:mysql://mariadb:3306/metastore
- HIVE_CONF_javax_jdo_option_ConnectionDriverName: com.mysql.cj.jdbc.Driver
- HIVE_CONF_javax_jdo_option_ConnectionUserName: root
- HIVE_CONF_javax_jdo_option_ConnectionPassword: rootpassword
+ HIVE_HOME: /opt/hive
+ PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hadoop/bin:/opt/hive/bin
depends_on:
mariadb:
condition: service_healthy
@@ -59,6 +137,7 @@ services:
"
environment:
HADOOP_HOME: /opt/hadoop
+ HIVE_HOME: /opt/hive
HIVE_CONF_javax_jdo_option_ConnectionURL: jdbc:mysql://mariadb/metastore
HIVE_CONF_javax_jdo_option_ConnectionDriverName: com.mysql.cj.jdbc.Driver
HIVE_CONF_javax_jdo_option_ConnectionUserName: hive
diff --git a/adapters/hive/hive-site.xml b/adapters/hive/hive-site.xml
new file mode 100644
index 0000000..cc91131
--- /dev/null
+++ b/adapters/hive/hive-site.xml
@@ -0,0 +1,20 @@
+
+
+
+
+ javax.jdo.option.ConnectionURL
+ jdbc:mysql://mariadb:3306/metastore
+
+
+ javax.jdo.option.ConnectionDriverName
+ com.mysql.cj.jdbc.Driver
+
+
+ javax.jdo.option.ConnectionUserName
+ root
+
+
+ javax.jdo.option.ConnectionPassword
+ rootpassword
+
+
\ No newline at end of file
diff --git a/adapters/jdbc/dev.docker-compose.calcite.yaml b/adapters/jdbc/dev.docker-compose.calcite.yaml
index 12f0f90..ce25c16 100644
--- a/adapters/jdbc/dev.docker-compose.calcite.yaml
+++ b/adapters/jdbc/dev.docker-compose.calcite.yaml
@@ -3,7 +3,7 @@ services:
build:
context: .
dockerfile_inline: |-
- FROM jdbc_connector:latest
+ FROM meta_connector:latest
COPY ./ /etc/connector
develop:
watch:
diff --git a/adapters/jdbc/dev.local.configuration.json b/adapters/jdbc/dev.local.configuration.json
index 1c7d30d..a3429c6 100644
--- a/adapters/jdbc/dev.local.configuration.json
+++ b/adapters/jdbc/dev.local.configuration.json
@@ -3,154 +3,242 @@
"$schema": "schema.json",
"model": {
"version": "1.0",
- "defaultSchema": "test",
+ "defaultSchema": "DEFAULT",
"schemas": [
{
"type": "jdbc",
- "name": "test",
- "jdbcUser": "kenstott",
- "jdbcPassword": "rN8qOh6AEMCP",
- "jdbcUrl": "jdbc:postgresql://ep-yellow-salad-961725.us-west-2.aws.neon.tech/crisp-sheepdog-47_db_3216533?sslmode=require",
- "jdbcCatalog": "public"
+ "name": "crm",
+ "jdbcUrl": "jdbc:redshift://default-workgroup.528956693660.us-east-2.redshift-serverless.amazonaws.com:5439/dev?user=redshiftdev&password=#Hasura2024!¤tSchema=crm"
+ },
+ {
+ "type": "jdbc",
+ "name": "emr",
+ "jdbcUrl": "jdbc:redshift://default-workgroup.528956693660.us-east-2.redshift-serverless.amazonaws.com:5439/dev?user=redshiftdev&password=#Hasura2024!¤tSchema=emr"
}
]
},
"model_file_path": "./model.json",
"fixes": true,
"metadata": {
- "Stores": {
- "schema": "test",
- "name": "Stores",
+ "customers": {
+ "schema": "crm",
+ "name": "customers",
"columns": {
- "squareMeters": {
- "name": "squareMeters",
- "scalarType": "DOUBLE",
+ "customer_id": {
+ "name": "customer_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "email": {
+ "name": "email",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "state": {
- "name": "state",
+ "last_name": {
+ "name": "last_name",
"scalarType": "VARCHAR",
"nullable": true
},
- "storeKey": {
- "name": "storeKey",
- "scalarType": "INTEGER",
+ "first_name": {
+ "name": "first_name",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "openDate": {
- "name": "openDate",
- "scalarType": "TIMESTAMP",
+ "registration_date": {
+ "name": "registration_date",
+ "scalarType": "DATE",
"nullable": true
},
- "country": {
- "name": "country",
+ "phone_number": {
+ "name": "phone_number",
"scalarType": "VARCHAR",
"nullable": true
}
},
- "primaryKeys": [],
- "exportedKeys": []
+ "primaryKeys": [
+ "customer_id"
+ ],
+ "exportedKeys": [
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "crm",
+ "pkTableName": "customers",
+ "pkColumnName": "customer_id",
+ "pkName": "customers_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "crm",
+ "fkTableName": "contacts",
+ "fkColumnName": "customer_id",
+ "fkName": "contacts_customer_id_fkey"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "crm",
+ "pkTableName": "customers",
+ "pkColumnName": "customer_id",
+ "pkName": "customers_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "crm",
+ "fkTableName": "orders",
+ "fkColumnName": "customer_id",
+ "fkName": "orders_customer_id_fkey"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "crm",
+ "pkTableName": "customers",
+ "pkColumnName": "customer_id",
+ "pkName": "customers_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "crm",
+ "fkTableName": "tasks",
+ "fkColumnName": "customer_id",
+ "fkName": "tasks_customer_id_fkey"
+ }
+ ]
},
- "DataDictionary": {
- "schema": "test",
- "name": "DataDictionary",
+ "medications": {
+ "schema": "emr",
+ "name": "medications",
"columns": {
- "table": {
- "name": "table",
+ "medication_id": {
+ "name": "medication_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "patient_id": {
+ "name": "patient_id",
+ "scalarType": "INTEGER",
+ "nullable": true
+ },
+ "prescription_date": {
+ "name": "prescription_date",
+ "scalarType": "DATE",
+ "nullable": true
+ },
+ "dosage": {
+ "name": "dosage",
"scalarType": "VARCHAR",
"nullable": true
},
- "field": {
- "name": "field",
+ "notes": {
+ "name": "notes",
"scalarType": "VARCHAR",
"nullable": true
},
- "description": {
- "name": "description",
+ "medication_name": {
+ "name": "medication_name",
"scalarType": "VARCHAR",
"nullable": true
}
},
- "primaryKeys": [],
+ "primaryKeys": [
+ "medication_id"
+ ],
"exportedKeys": []
},
- "Customers": {
- "schema": "test",
- "name": "Customers",
+ "doctors": {
+ "schema": "emr",
+ "name": "doctors",
"columns": {
- "birthday": {
- "name": "birthday",
- "scalarType": "TIMESTAMP",
- "nullable": true
- },
- "city": {
- "name": "city",
+ "phone_number": {
+ "name": "phone_number",
"scalarType": "VARCHAR",
"nullable": true
},
- "name": {
- "name": "name",
+ "specialization": {
+ "name": "specialization",
"scalarType": "VARCHAR",
"nullable": true
},
- "state": {
- "name": "state",
+ "address": {
+ "name": "address",
"scalarType": "VARCHAR",
"nullable": true
},
- "zipCode": {
- "name": "zipCode",
+ "first_name": {
+ "name": "first_name",
"scalarType": "VARCHAR",
"nullable": true
},
- "continent": {
- "name": "continent",
+ "last_name": {
+ "name": "last_name",
"scalarType": "VARCHAR",
"nullable": true
},
- "country": {
- "name": "country",
- "scalarType": "VARCHAR",
- "nullable": true
+ "doctor_id": {
+ "name": "doctor_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ }
+ },
+ "primaryKeys": [
+ "doctor_id"
+ ],
+ "exportedKeys": [
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "doctors",
+ "pkColumnName": "doctor_id",
+ "pkName": "doctors_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "appointments",
+ "fkColumnName": "doctor_id",
+ "fkName": "fk_appointments_doctors"
+ }
+ ]
+ },
+ "orders": {
+ "schema": "crm",
+ "name": "orders",
+ "columns": {
+ "order_id": {
+ "name": "order_id",
+ "scalarType": "INTEGER",
+ "nullable": false
},
- "stateCode": {
- "name": "stateCode",
- "scalarType": "VARCHAR",
+ "order_date": {
+ "name": "order_date",
+ "scalarType": "DATE",
"nullable": true
},
- "gender": {
- "name": "gender",
- "scalarType": "VARCHAR",
+ "customer_id": {
+ "name": "customer_id",
+ "scalarType": "INTEGER",
"nullable": true
},
- "customerKey": {
- "name": "customerKey",
- "scalarType": "INTEGER",
+ "total_amount": {
+ "name": "total_amount",
+ "scalarType": "FLOAT",
"nullable": true
}
},
- "primaryKeys": [],
- "exportedKeys": []
- },
- "z": {
- "schema": "test",
- "name": "z",
- "columns": {
- "z": {
- "name": "z",
- "scalarType": "BOOLEAN",
- "nullable": true
+ "primaryKeys": [
+ "order_id"
+ ],
+ "exportedKeys": [
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "crm",
+ "pkTableName": "orders",
+ "pkColumnName": "order_id",
+ "pkName": "orders_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "crm",
+ "fkTableName": "order_items",
+ "fkColumnName": "order_id",
+ "fkName": "order_items_order_id_fkey"
}
- },
- "primaryKeys": [],
- "exportedKeys": []
+ ]
},
- "Sales": {
- "schema": "test",
- "name": "Sales",
+ "order_items": {
+ "schema": "crm",
+ "name": "order_items",
"columns": {
- "lineItem": {
- "name": "lineItem",
+ "order_id": {
+ "name": "order_id",
"scalarType": "INTEGER",
"nullable": true
},
@@ -159,125 +247,320 @@
"scalarType": "INTEGER",
"nullable": true
},
- "deliveryDate": {
- "name": "deliveryDate",
- "scalarType": "TIMESTAMP",
+ "item_amount": {
+ "name": "item_amount",
+ "scalarType": "FLOAT",
"nullable": true
},
- "productKey": {
- "name": "productKey",
+ "order_item_id": {
+ "name": "order_item_id",
"scalarType": "INTEGER",
- "nullable": true
+ "nullable": false
},
- "orderNumber": {
- "name": "orderNumber",
+ "product_id": {
+ "name": "product_id",
"scalarType": "INTEGER",
"nullable": true
+ }
+ },
+ "primaryKeys": [
+ "order_item_id"
+ ],
+ "exportedKeys": []
+ },
+ "contacts": {
+ "schema": "crm",
+ "name": "contacts",
+ "columns": {
+ "contact_id": {
+ "name": "contact_id",
+ "scalarType": "INTEGER",
+ "nullable": false
},
- "orderDate": {
- "name": "orderDate",
- "scalarType": "TIMESTAMP",
+ "customer_id": {
+ "name": "customer_id",
+ "scalarType": "INTEGER",
"nullable": true
},
- "storeKey": {
- "name": "storeKey",
- "scalarType": "INTEGER",
+ "contact_type": {
+ "name": "contact_type",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "customerKey": {
- "name": "customerKey",
- "scalarType": "INTEGER",
+ "contact_date": {
+ "name": "contact_date",
+ "scalarType": "DATE",
"nullable": true
},
- "currencyCode": {
- "name": "currencyCode",
+ "notes": {
+ "name": "notes",
"scalarType": "VARCHAR",
"nullable": true
}
},
- "primaryKeys": [],
+ "primaryKeys": [
+ "contact_id"
+ ],
"exportedKeys": []
},
- "Products": {
- "schema": "test",
- "name": "Products",
+ "tasks": {
+ "schema": "crm",
+ "name": "tasks",
"columns": {
- "unitPriceUSD": {
- "name": "unitPriceUSD",
- "scalarType": "DOUBLE",
+ "task_status": {
+ "name": "task_status",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "productName": {
- "name": "productName",
+ "customer_id": {
+ "name": "customer_id",
+ "scalarType": "INTEGER",
+ "nullable": true
+ },
+ "task_id": {
+ "name": "task_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "task_description": {
+ "name": "task_description",
"scalarType": "VARCHAR",
"nullable": true
},
- "category": {
- "name": "category",
+ "task_date": {
+ "name": "task_date",
+ "scalarType": "DATE",
+ "nullable": true
+ }
+ },
+ "primaryKeys": [
+ "task_id"
+ ],
+ "exportedKeys": []
+ },
+ "medical_conditions": {
+ "schema": "emr",
+ "name": "medical_conditions",
+ "columns": {
+ "condition_name": {
+ "name": "condition_name",
"scalarType": "VARCHAR",
"nullable": true
},
- "categoryKey": {
- "name": "categoryKey",
+ "notes": {
+ "name": "notes",
+ "scalarType": "VARCHAR",
+ "nullable": true
+ },
+ "patient_id": {
+ "name": "patient_id",
"scalarType": "INTEGER",
"nullable": true
},
- "unitCostUSD": {
- "name": "unitCostUSD",
- "scalarType": "DOUBLE",
+ "condition_id": {
+ "name": "condition_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "diagnosis_date": {
+ "name": "diagnosis_date",
+ "scalarType": "DATE",
+ "nullable": true
+ }
+ },
+ "primaryKeys": [
+ "condition_id"
+ ],
+ "exportedKeys": []
+ },
+ "products": {
+ "schema": "crm",
+ "name": "products",
+ "columns": {
+ "price": {
+ "name": "price",
+ "scalarType": "FLOAT",
"nullable": true
},
- "subcategory": {
- "name": "subcategory",
+ "product_id": {
+ "name": "product_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "description": {
+ "name": "description",
"scalarType": "VARCHAR",
"nullable": true
},
- "brand": {
- "name": "brand",
+ "product_name": {
+ "name": "product_name",
"scalarType": "VARCHAR",
"nullable": true
- },
- "color": {
- "name": "color",
+ }
+ },
+ "primaryKeys": [
+ "product_id"
+ ],
+ "exportedKeys": [
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "crm",
+ "pkTableName": "products",
+ "pkColumnName": "product_id",
+ "pkName": "products_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "crm",
+ "fkTableName": "order_items",
+ "fkColumnName": "product_id",
+ "fkName": "order_items_product_id_fkey"
+ }
+ ]
+ },
+ "appointments": {
+ "schema": "emr",
+ "name": "appointments",
+ "columns": {
+ "notes": {
+ "name": "notes",
"scalarType": "VARCHAR",
"nullable": true
},
- "productKey": {
- "name": "productKey",
+ "appointment_id": {
+ "name": "appointment_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "doctor_id": {
+ "name": "doctor_id",
"scalarType": "INTEGER",
"nullable": true
},
- "subcategoryKey": {
- "name": "subcategoryKey",
+ "patient_id": {
+ "name": "patient_id",
"scalarType": "INTEGER",
"nullable": true
+ },
+ "appointment_date": {
+ "name": "appointment_date",
+ "scalarType": "TIMESTAMP",
+ "nullable": true
+ },
+ "reason": {
+ "name": "reason",
+ "scalarType": "VARCHAR",
+ "nullable": true
}
},
- "primaryKeys": [],
+ "primaryKeys": [
+ "appointment_id"
+ ],
"exportedKeys": []
},
- "ExchangeRates": {
- "schema": "test",
- "name": "ExchangeRates",
+ "patients": {
+ "schema": "emr",
+ "name": "patients",
"columns": {
- "exchange": {
- "name": "exchange",
- "scalarType": "DOUBLE",
+ "first_name": {
+ "name": "first_name",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "date": {
- "name": "date",
- "scalarType": "TIMESTAMP",
+ "date_of_birth": {
+ "name": "date_of_birth",
+ "scalarType": "DATE",
+ "nullable": true
+ },
+ "patient_id": {
+ "name": "patient_id",
+ "scalarType": "INTEGER",
+ "nullable": false
+ },
+ "phone_number": {
+ "name": "phone_number",
+ "scalarType": "VARCHAR",
+ "nullable": true
+ },
+ "address": {
+ "name": "address",
+ "scalarType": "VARCHAR",
"nullable": true
},
- "currency": {
- "name": "currency",
+ "gender": {
+ "name": "gender",
+ "scalarType": "VARCHAR",
+ "nullable": true
+ },
+ "last_name": {
+ "name": "last_name",
"scalarType": "VARCHAR",
"nullable": true
}
},
- "primaryKeys": [],
- "exportedKeys": []
+ "primaryKeys": [
+ "patient_id"
+ ],
+ "exportedKeys": [
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "patients",
+ "pkColumnName": "patient_id",
+ "pkName": "patients_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "appointments",
+ "fkColumnName": "patient_id",
+ "fkName": "fk_appointments_patients"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "patients",
+ "pkColumnName": "patient_id",
+ "pkName": "patients_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "medical_conditions",
+ "fkColumnName": "patient_id",
+ "fkName": "fk_medical_conditions_patients"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "patients",
+ "pkColumnName": "patient_id",
+ "pkName": "patients_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "medical_conditions",
+ "fkColumnName": "patient_id",
+ "fkName": "medical_conditions_patient_id_fkey"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "patients",
+ "pkColumnName": "patient_id",
+ "pkName": "patients_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "medications",
+ "fkColumnName": "patient_id",
+ "fkName": "fk_medications_patients"
+ },
+ {
+ "pkTableCatalog": "dev",
+ "pkTableSchema": "emr",
+ "pkTableName": "patients",
+ "pkColumnName": "patient_id",
+ "pkName": "patients_pkey",
+ "fkTableCatalog": "dev",
+ "fkTableSchema": "emr",
+ "fkTableName": "medications",
+ "fkColumnName": "patient_id",
+ "fkName": "medications_patient_id_fkey"
+ }
+ ]
}
}
}
\ No newline at end of file
diff --git a/adapters/jdbc/docker-compose.calcite.yaml b/adapters/jdbc/docker-compose.calcite.yaml
index 6c23427..1d86fcf 100644
--- a/adapters/jdbc/docker-compose.calcite.yaml
+++ b/adapters/jdbc/docker-compose.calcite.yaml
@@ -3,7 +3,7 @@ services:
build:
context: .
dockerfile_inline: |-
- FROM jdbc_connector:latest
+ FROM meta_connector:latest
COPY ./ /etc/connector
develop:
watch:
@@ -19,5 +19,4 @@ services:
target: 8080
published: "8081"
protocol: tcp
-# volumes:
-# - /Users/kennethstott/Documents/GitHub/ndc-calcite/adapters/file/resources/test/sales:/data
+
diff --git a/adapters/jdbc/model.json b/adapters/jdbc/model.json
index 07ff45e..44053c1 100644
--- a/adapters/jdbc/model.json
+++ b/adapters/jdbc/model.json
@@ -1,14 +1,16 @@
{
"version": "1.0",
- "defaultSchema": "test",
+ "defaultSchema": "DEFAULT",
"schemas": [
{
"type": "jdbc",
- "name": "test",
- "jdbcUser": "kenstott",
- "jdbcPassword": "rN8qOh6AEMCP",
- "jdbcUrl": "jdbc:postgresql://ep-yellow-salad-961725.us-west-2.aws.neon.tech/crisp-sheepdog-47_db_3216533?sslmode=require",
- "jdbcCatalog": "public"
+ "name": "crm",
+ "jdbcUrl": "jdbc:redshift://default-workgroup.528956693660.us-east-2.redshift-serverless.amazonaws.com:5439/dev?user=redshiftdev&password=#Hasura2024!¤tSchema=crm"
+ },
+ {
+ "type": "jdbc",
+ "name": "emr",
+ "jdbcUrl": "jdbc:redshift://default-workgroup.528956693660.us-east-2.redshift-serverless.amazonaws.com:5439/dev?user=redshiftdev&password=#Hasura2024!¤tSchema=emr"
}
]
-}
\ No newline at end of file
+}
diff --git a/connector-definition.tgz b/connector-definition.tgz
index ea4c89f..4ee40d8 100644
Binary files a/connector-definition.tgz and b/connector-definition.tgz differ
diff --git a/crates/connectors/ndc-calcite/Cargo.toml b/crates/connectors/ndc-calcite/Cargo.toml
index dd8accd..cc80c6f 100644
--- a/crates/connectors/ndc-calcite/Cargo.toml
+++ b/crates/connectors/ndc-calcite/Cargo.toml
@@ -53,6 +53,7 @@ url = "2.5.0"
dotenv = "0.15.0"
once_cell = "1.19.0"
anyhow = "1.0.86"
+log = "0.4.22"
[dev-dependencies]
axum-test-helper = "0.3.0"
\ No newline at end of file
diff --git a/crates/connectors/ndc-calcite/src/connector/calcite.rs b/crates/connectors/ndc-calcite/src/connector/calcite.rs
index 8f1b09b..603ca6a 100644
--- a/crates/connectors/ndc-calcite/src/connector/calcite.rs
+++ b/crates/connectors/ndc-calcite/src/connector/calcite.rs
@@ -6,7 +6,7 @@
use std::collections::BTreeMap;
use std::{env, fs};
use std::path::Path;
-
+use log::{info, error};
use async_trait::async_trait;
use dotenv;
use jni::objects::GlobalRef;
@@ -266,6 +266,7 @@ impl Connector for Calcite {
state: &Self::State,
request: models::QueryRequest,
) -> Result, QueryError> {
+ // println!("{:?}", serde_json::to_string_pretty(&request));
let variable_sets = request.variables.unwrap_or(vec![BTreeMap::new()]);
let mut row_sets = vec![];
let input_map: BTreeMap = request.arguments.clone();
@@ -278,7 +279,7 @@ impl Connector for Calcite {
)
.collect();
for variables in &variable_sets {
- let row_set = execute_query_with_variables(
+ let row_set = match execute_query_with_variables(
configuration,
&request.collection,
&relationship_arguments,
@@ -287,9 +288,22 @@ impl Connector for Calcite {
variables,
&state,
&false
- )?;
+ ) {
+ Ok(row_set) => {
+ info!("execute_query_with_variables was successful");
+ row_set
+ },
+ Err(e) => {
+ error!("Error executing query: {:?}", e);
+ return Err(e.into());
+ },
+ };
+ // println!("Get row set");
+ // println!("{:?}", serde_json::to_string_pretty(&row_set));
row_sets.push(row_set);
+ // println!("Pushed row set");
}
+ // println!("Returning row sets");
Ok(models::QueryResponse(row_sets).into())
}
}
diff --git a/crates/connectors/ndc-calcite/src/sql.rs b/crates/connectors/ndc-calcite/src/sql.rs
index dff211e..5e9aeb0 100644
--- a/crates/connectors/ndc-calcite/src/sql.rs
+++ b/crates/connectors/ndc-calcite/src/sql.rs
@@ -63,7 +63,7 @@ fn select(
let field_statement = format!("'{}', {}.\"{}\"", key, table, column, );
field_statements.push(field_statement);
} else {
- let field_statement = format!("{}.\"{}\"", table, column, );
+ let field_statement = format!("{}.\"{}\" AS \"{}\"", table, column, key);
field_statements.push(field_statement);
}
// TODO: use nested fields???