From 0d467c4aafbf84464225a978489464db8670c4ee Mon Sep 17 00:00:00 2001 From: Croft Date: Wed, 21 Feb 2024 15:00:44 +0100 Subject: [PATCH 01/34] Added some diagnostics for testing purposes --- src/main.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main.rs b/src/main.rs index 7792425..c2ef83e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -307,8 +307,12 @@ async fn run_exporter_query( } fn replace_cql_library(mut query: CqlQuery) -> Result { + info!("replace_cql_library: entered"); + let old_data_value = &query.lib["content"][0]["data"]; + info!("replace_cql_library: look for Field .content[0].data in old data"); + let old_data_string = old_data_value .as_str() .ok_or(FocusError::ParsingError(format!( @@ -316,11 +320,17 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { query.lib )))?; + info!("replace_cql_library: do a base 64 decode on old data"); + let decoded_cql = util::base64_decode(old_data_string)?; + info!("replace_cql_library: convert to UTF8"); + let decoded_string = str::from_utf8(&decoded_cql) .map_err(|_| FocusError::ParsingError("CQL query was invalid".into()))?; + info!("replace_cql_library: check for unexpected defines"); + match is_cql_tampered_with(decoded_string) { false => debug!("CQL not tampered with"), true => { @@ -331,11 +341,15 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { } }; + info!("replace_cql_library: multiple operations"); + let replaced_cql_str = util::replace_cql(decoded_string); let replaced_cql_str_base64 = BASE64.encode(replaced_cql_str); let new_data_value = serde_json::to_value(replaced_cql_str_base64) .expect("unable to turn base64 string into json value - this should not happen"); + info!("replace_cql_library: Update the CqlQuery with the new data value"); + let a = &mut query.lib["content"][0]["data"]; *a = new_data_value; From dcebc22b6c2139992427a4e1d349afdfd6ba8717 Mon Sep 17 00:00:00 2001 From: Croft Date: Thu, 22 Feb 2024 15:53:10 +0100 Subject: [PATCH 02/34] Initial commit for the EHDS2 project The changes are really just to test out how Focus is working. --- DockerfileWithBuild | 26 ++++++++++++++++++++++++++ build.sh | 1 + src/main.rs | 1 + 3 files changed, 28 insertions(+) create mode 100644 DockerfileWithBuild create mode 100644 build.sh diff --git a/DockerfileWithBuild b/DockerfileWithBuild new file mode 100644 index 0000000..cab5aed --- /dev/null +++ b/DockerfileWithBuild @@ -0,0 +1,26 @@ +# This Dockerfile builds the Rust project and creates the final image. + +# Use the official Rust image as the build environment +FROM rust:latest as builder + +# Set the working directory inside the container +WORKDIR /usr/src/your_project_name + +# Copy the project files into the container +COPY . . + +# Build the Rust project +RUN cargo build --release + +# Create a smaller final image +FROM gcr.io/distroless/cc-debian12 + +# Set the working directory inside the container +WORKDIR /usr/src/your_project_name + +# Copy the built binary from the builder stage +COPY --from=builder /usr/src/your_project_name/target/release/focus /usr/local/bin/ + +# Set the entry point for the final image +ENTRYPOINT ["/usr/local/bin/focus"] + diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..b1f8441 --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +docker build -f DockerfileWithBuild -t samply/focus --no-cache . diff --git a/src/main.rs b/src/main.rs index c2ef83e..75fa601 100644 --- a/src/main.rs +++ b/src/main.rs @@ -330,6 +330,7 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { .map_err(|_| FocusError::ParsingError("CQL query was invalid".into()))?; info!("replace_cql_library: check for unexpected defines"); + info!("replace_cql_library: decoded_string {:?}", decoded_string); match is_cql_tampered_with(decoded_string) { false => debug!("CQL not tampered with"), From eb1e181571117478222dc9482928050460292bf0 Mon Sep 17 00:00:00 2001 From: Croft Date: Thu, 22 Feb 2024 16:49:21 +0100 Subject: [PATCH 03/34] Added info for diagnostics --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 75fa601..3793f75 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,7 +31,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use std::{process::exit, time::Duration}; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, warn}; +use tracing::{debug, error, warn, info}; // result cache type SearchQuery = String; From 434e50b4112b6747b64cf8cb0c30ddd3a0fa0d0d Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 23 Feb 2024 10:51:13 +0100 Subject: [PATCH 04/34] Added more diagnostics --- src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.rs b/src/main.rs index 3793f75..133e8c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -104,6 +104,8 @@ pub async fn main() -> ExitCode { } async fn main_loop() -> ExitCode { + info!("main_loop: entered"); + // TODO: The report cache init should be an fn on the cache let report_cache: ReportCache = ReportCache::new(); @@ -172,6 +174,8 @@ async fn run_cql_query( report_cache: Arc>, project: String, ) -> Result { + info!("run_cql_query: entered"); + let encoded_query = query.lib["content"][0]["data"] .as_str() @@ -244,6 +248,8 @@ async fn run_cql_query( ) }); + info!("run_cql_query: finished"); + Ok(result) } From 01909a80ad2433de79893b8577e0f52429221084 Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 23 Feb 2024 13:47:36 +0100 Subject: [PATCH 05/34] Added first stratifier for the EHDS2 project --- resources/cql/EHDS2_STRAT_GENDER_STRATIFIER | 2 ++ src/main.rs | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 resources/cql/EHDS2_STRAT_GENDER_STRATIFIER diff --git a/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER b/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER new file mode 100644 index 0000000..4145d04 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER @@ -0,0 +1,2 @@ +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender diff --git a/src/main.rs b/src/main.rs index 133e8c3..56f3f49 100644 --- a/src/main.rs +++ b/src/main.rs @@ -360,6 +360,8 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { let a = &mut query.lib["content"][0]["data"]; *a = new_data_value; + info!("replace_cql_library: finished"); + Ok(query) } From ed49a5cd3435d7290e6bbabd650abf62d30b85f3 Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 23 Feb 2024 14:18:25 +0100 Subject: [PATCH 06/34] Adding second EHDS2 stratifier --- resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION | 1 + 1 file changed, 1 insertion(+) create mode 100644 resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION diff --git a/resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION b/resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION new file mode 100644 index 0000000..80876f8 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION @@ -0,0 +1 @@ +define InInitialPopulation: From e006fff9613b21c0ec388fdb8f678664164179fe Mon Sep 17 00:00:00 2001 From: Croft Date: Fri, 23 Feb 2024 15:56:31 +0100 Subject: [PATCH 07/34] Added stratifiers for Observation --- resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER | 6 ++++++ resources/cql/EHDS2_STRAT_DEF_OBSERVATION | 1 + 2 files changed, 7 insertions(+) create mode 100644 resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER create mode 100644 resources/cql/EHDS2_STRAT_DEF_OBSERVATION diff --git a/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER b/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER new file mode 100644 index 0000000..775b943 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER @@ -0,0 +1,6 @@ +define function Antibiotic(observation FHIR.Observation): + case FHIRHelpers.ToCode(observation.value.coding.where(system = 'https://ecdc.amr/antibiotic-codes').first()) + when Code 'MEM' from Antibiotic then 'MEM' + when null then 'Unknown' + else 'Unknown' + end diff --git a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION new file mode 100644 index 0000000..94053c4 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION @@ -0,0 +1 @@ +define Observation: From 28fb37a6c0ce319a505c3c9b416fcf6a609157cb Mon Sep 17 00:00:00 2001 From: Croft Date: Mon, 26 Feb 2024 09:04:40 +0100 Subject: [PATCH 08/34] New diagnostic --- src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.rs b/src/main.rs index 56f3f49..aee8896 100644 --- a/src/main.rs +++ b/src/main.rs @@ -360,6 +360,7 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { let a = &mut query.lib["content"][0]["data"]; *a = new_data_value; + info!("replace_cql_library: new_data_value {:?}", new_data_value); info!("replace_cql_library: finished"); Ok(query) From 880096c1338309f443d547aa25ccdda5a21b11d7 Mon Sep 17 00:00:00 2001 From: Croft Date: Mon, 26 Feb 2024 09:14:02 +0100 Subject: [PATCH 09/34] Correction --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index aee8896..ad8b01c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -355,12 +355,12 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { let new_data_value = serde_json::to_value(replaced_cql_str_base64) .expect("unable to turn base64 string into json value - this should not happen"); + info!("replace_cql_library: new_data_value {:?}", new_data_value); info!("replace_cql_library: Update the CqlQuery with the new data value"); let a = &mut query.lib["content"][0]["data"]; *a = new_data_value; - info!("replace_cql_library: new_data_value {:?}", new_data_value); info!("replace_cql_library: finished"); Ok(query) From cab43f262adb406196df17e0fce71c8b70afb126 Mon Sep 17 00:00:00 2001 From: Croft Date: Mon, 26 Feb 2024 09:33:49 +0100 Subject: [PATCH 10/34] Diagnistic --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index ad8b01c..0f31472 100644 --- a/src/main.rs +++ b/src/main.rs @@ -355,7 +355,7 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { let new_data_value = serde_json::to_value(replaced_cql_str_base64) .expect("unable to turn base64 string into json value - this should not happen"); - info!("replace_cql_library: new_data_value {:?}", new_data_value); + info!("replace_cql_library: replaced_cql_str {:?}", replaced_cql_str); info!("replace_cql_library: Update the CqlQuery with the new data value"); let a = &mut query.lib["content"][0]["data"]; From 7da8db294f31e46e6a2f703c389e5dc5c00dafe5 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Wed, 13 Mar 2024 09:40:08 +0100 Subject: [PATCH 11/34] Multiple issues * Reverted to working version of DockerfileWithBuild * Plaintext build.sh * New CQL for patient age * Corrected Observation CQL * Diagnostics in Rust code --- .dockerignore | 2 -- DockerfileWithBuild | 1 + build.sh | 2 +- resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER | 9 +++++++++ resources/cql/EHDS2_STRAT_DEF_OBSERVATION | 1 + src/main.rs | 17 ++++++++++++++++- 6 files changed, 28 insertions(+), 4 deletions(-) delete mode 100644 .dockerignore create mode 100644 resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 37ad7ed..0000000 --- a/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!artifacts/ diff --git a/DockerfileWithBuild b/DockerfileWithBuild index cab5aed..811bb90 100644 --- a/DockerfileWithBuild +++ b/DockerfileWithBuild @@ -10,6 +10,7 @@ WORKDIR /usr/src/your_project_name COPY . . # Build the Rust project +RUN echo building project RUN cargo build --release # Create a smaller final image diff --git a/build.sh b/build.sh index b1f8441..738c070 100644 --- a/build.sh +++ b/build.sh @@ -1 +1 @@ -docker build -f DockerfileWithBuild -t samply/focus --no-cache . +docker build --progress=plain -f DockerfileWithBuild -t samply/focus --no-cache . diff --git a/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER new file mode 100644 index 0000000..a7331e6 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER @@ -0,0 +1,9 @@ +define BaseAgeInYears: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientAge').value + +define AgeInYears: +ToInteger(if (BaseAgeInYears is null) then '-1' else BaseAgeInYears) + +define AgeClass: +ToString((AgeInYears div 10) * 10) + diff --git a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION index 94053c4..cc94d2b 100644 --- a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION +++ b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION @@ -1 +1,2 @@ define Observation: + if InInitialPopulation then [Observation] else {} as List diff --git a/src/main.rs b/src/main.rs index 0f31472..9f2785b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,6 +85,8 @@ const REPORTCACHE_TTL: Duration = Duration::from_secs(86400); //24h #[tokio::main] pub async fn main() -> ExitCode { + info!("main: entered"); + if let Err(e) = logger::init_logger() { error!("Cannot initalize logger: {}", e); exit(1); @@ -93,6 +95,8 @@ pub async fn main() -> ExitCode { let _ = CONFIG.api_key; // Initialize config + info!("main: going to Tokio"); + tokio::select! { _ = graceful_shutdown::wait_for_signal() => { ExitCode::SUCCESS @@ -140,6 +144,7 @@ async fn main_loop() -> ExitCode { failures = 0; } } + info!("main_loop: continuing"); error!( "Encountered too many errors -- exiting after {} attempts.", CONFIG.retry_count @@ -186,8 +191,12 @@ async fn run_cql_query( let mut key_exists = false; + info!("run_cql_query: obfuscate"); + let obfuscate = CONFIG.obfuscate == config::Obfuscate::Yes && !CONFIG.unobfuscated.contains(&project); + info!("run_cql_query: get report"); + let report_from_cache = match report_cache .lock() .await @@ -205,6 +214,8 @@ async fn run_cql_query( None => None, }; + info!("run_cql_query: match report_from_cache"); + let cql_result_new = match report_from_cache { Some(some_report_from_cache) => some_report_from_cache.to_string(), None => { @@ -239,6 +250,8 @@ async fn run_cql_query( } }; + info!("run_cql_query: beam_result"); + let result = beam_result(task.to_owned(), cql_result_new).unwrap_or_else(|e| { beam::beam_result::perm_failed( CONFIG.beam_app_id_long.clone(), @@ -351,11 +364,13 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { info!("replace_cql_library: multiple operations"); let replaced_cql_str = util::replace_cql(decoded_string); + + info!("replace_cql_library: replaced_cql_str {:?}", replaced_cql_str); + let replaced_cql_str_base64 = BASE64.encode(replaced_cql_str); let new_data_value = serde_json::to_value(replaced_cql_str_base64) .expect("unable to turn base64 string into json value - this should not happen"); - info!("replace_cql_library: replaced_cql_str {:?}", replaced_cql_str); info!("replace_cql_library: Update the CqlQuery with the new data value"); let a = &mut query.lib["content"][0]["data"]; From 39fed29afb9f3b7f247a03f46dbf12f2b874af0c Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Fri, 15 Mar 2024 13:36:38 +0100 Subject: [PATCH 12/34] Added new attributes to EHDS2 search --- resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER | 2 +- resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER | 9 +++++++++ .../cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER | 9 +++++++++ .../cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER | 9 +++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER create mode 100644 resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER create mode 100644 resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER diff --git a/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER index a7331e6..56970a1 100644 --- a/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER +++ b/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER @@ -2,7 +2,7 @@ define BaseAgeInYears: Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientAge').value define AgeInYears: -ToInteger(if (BaseAgeInYears is null) then '-1' else BaseAgeInYears) +ToInteger(if (BaseAgeInYears is null or BaseAgeInYears = '-') then '-1' else BaseAgeInYears) define AgeClass: ToString((AgeInYears div 10) * 10) diff --git a/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER new file mode 100644 index 0000000..9d3d5c9 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER @@ -0,0 +1,9 @@ +define BaseHospitalId: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalId').value + +define HospitalId: +if (BaseHospitalId is null or BaseHospitalId = '-') then 'Unkown' else BaseHospitalId + +define HospitalIdClass: +HospitalId + diff --git a/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER new file mode 100644 index 0000000..2f6d596 --- /dev/null +++ b/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER @@ -0,0 +1,9 @@ +define BaseHospitalUnitType: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalUnitType').value + +define HospitalUnitType: +if (BaseHospitalUnitType is null or BaseHospitalUnitType = '-') then 'Unkown' else BaseHospitalUnitType + +define HospitalUnitTypeClass: +HospitalUnitType + diff --git a/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER new file mode 100644 index 0000000..bf8543e --- /dev/null +++ b/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER @@ -0,0 +1,9 @@ +define BaseLaboratoryCode: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientLaboratoryCode').value + +define LaboratoryCode: +if (BaseLaboratoryCode is null or BaseLaboratoryCode = '-') then 'Unkown' else BaseLaboratoryCode + +define LaboratoryCodeClass: +LaboratoryCode + From 458fa31b4cfb077b6c7712a1d14ba584cead3f1e Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Fri, 22 Mar 2024 10:06:04 +0100 Subject: [PATCH 13/34] First template for ECDC Observation resources With this commit I have managed to get Observation count to work and have also implemented the first Observation-related query (pathogen). Note that I had to change DockerfileWithBuild to fix the Rust image version, because the build was breaking with "latest". --- DockerfileWithBuild | 2 +- resources/cql/EHDS2_STRAT_DEF_OBSERVATION | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/DockerfileWithBuild b/DockerfileWithBuild index 811bb90..30d18b2 100644 --- a/DockerfileWithBuild +++ b/DockerfileWithBuild @@ -1,7 +1,7 @@ # This Dockerfile builds the Rust project and creates the final image. # Use the official Rust image as the build environment -FROM rust:latest as builder +FROM rust:1.76.0-bookworm as builder # Set the working directory inside the container WORKDIR /usr/src/your_project_name diff --git a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION index cc94d2b..37df1e7 100644 --- a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION +++ b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION @@ -1,2 +1,6 @@ -define Observation: +define ObservationList: if InInitialPopulation then [Observation] else {} as List + +define function PathogenCode(observation FHIR.Observation): + (observation.value as CodeableConcept).coding.where(system = 'https://ecdc.amr/pathogen-codes').code.first() + From 6e671dce229846c42a4b1f41ffe00ad1fef823a5 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Tue, 26 Mar 2024 13:56:42 +0100 Subject: [PATCH 14/34] Completed the remaining 4 searches and stratifiers for Objects --- resources/cql/EHDS2_STRAT_DEF_OBSERVATION | 93 ++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION index 37df1e7..aca6691 100644 --- a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION +++ b/resources/cql/EHDS2_STRAT_DEF_OBSERVATION @@ -1,6 +1,97 @@ define ObservationList: if InInitialPopulation then [Observation] else {} as List +// Return the value in the given system for the given Observation +define function ObservationValueCode(observation FHIR.Observation, system_var String): + (observation.value as CodeableConcept).coding.where(system = system_var).code.first() + +// Check to see if there are any Observation resources with the +// given system/value pair. +define function ExistsObservationValueCode(system_var String, code_var String): + exists from [Observation] O + where ObservationValueCode(O, system_var) = code_var + +// Return the value associated with the extension with the given URL for the given Observation +define function BaseObservationExtensionValue(observation FHIR.Observation, url_var String): + observation.extension.where(url = url_var).value + +// Return the value of the Observation extension with the given URL +define function ObservationExtensionValue(observation FHIR.Observation, url_var String): + if (BaseObservationExtensionValue(observation, url_var) is null or BaseObservationExtensionValue(observation, url_var) = '-') then 'Unkown' else BaseObservationExtensionValue(observation, url_var) + +define function ExistsObservationExtensionValue(url_var String, value_var String): + exists from [Observation] O + where ObservationExtensionValue(O, url_var) = value_var + +// Return the pathogen type for the given Observation define function PathogenCode(observation FHIR.Observation): - (observation.value as CodeableConcept).coding.where(system = 'https://ecdc.amr/pathogen-codes').code.first() + ObservationValueCode(observation, 'https://ecdc.amr/pathogen-codes') + +// Check to see if there are any Observation resources with the +// given pathogen type. +define function ExistsPathogenCode(code_var String): + ExistsObservationValueCode('https://ecdc.amr/pathogen-codes', code_var) + +// Return the antibiotic type for the given Observation +define function AntibioticCode(observation FHIR.Observation): + ObservationValueCode(observation, 'https://ecdc.amr/antibiotic-codes') + +// Check to see if there are any Observation resources with the +// given antibiotic type. +define function ExistsAntibioticCode(code_var String): + ExistsObservationValueCode('https://ecdc.amr/antibiotic-codes', code_var) + +// Return the resistance type for the given Observation +define function SirCode(observation FHIR.Observation): + ObservationValueCode(observation, 'https://ecdc.amr/sir-codes') + +// Check to see if there are any Observation resources with the +// given resistance type. +define function ExistsSirCode(code_var String): + ExistsObservationValueCode('https://ecdc.amr/sir-codes', code_var) + +// Return the data source for the given Observation +define function DataSource(observation FHIR.Observation): + ObservationExtensionValue(observation, 'https://ecdc.amr/fhir/StructureDefinition/ObservationDataSource') + +// Check to see if there are any Observation resources with the +// given data source. +define function ExistsDataSource(value_var String): + ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationDataSource', value_var) + +// Return the isolate ID for the given Observation +define function IsolateId(observation FHIR.Observation): + ObservationExtensionValue(observation, 'https://ecdc.amr/fhir/StructureDefinition/ObservationIsolateId') + +// Check to see if there are any Observation resources with the +// given isolate ID. +define function ExistsIsolateId(value_var String): + ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationIsolateId', value_var) + +// Return the patient type for the given Observation +define function PatientType(observation FHIR.Observation): + ObservationExtensionValue(observation, 'https://ecdc.amr/fhir/StructureDefinition/ObservationPatientType') + +// Check to see if there are any Observation resources with the +// given patient type. +define function ExistsPatientType(value_var String): + ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationPatientType', value_var) + +// Return the reference guidelines SIR for the given Observation +define function ReferenceGuidelinesSir(observation FHIR.Observation): + ObservationExtensionValue(observation, 'https://ecdc.amr/fhir/StructureDefinition/ObservationReferenceGuidelinesSIR') + +// Check to see if there are any Observation resources with the +// given reference guidelines SIR. +define function ExistsReferenceGuidelinesSir(value_var String): + ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationReferenceGuidelinesSIR', value_var) + +// Return the reporting country for the given Observation +define function ReportingCountry(observation FHIR.Observation): + ObservationExtensionValue(observation, 'https://ecdc.amr/fhir/StructureDefinition/ObservationReportingCountry') + +// Check to see if there are any Observation resources with the +// given reporting country. +define function ExistsReportingCountry(value_var String): + ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationReportingCountry', value_var) From ba37c99517fb029a97b9fa94b11832681488fae3 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Wed, 27 Mar 2024 09:28:48 +0100 Subject: [PATCH 15/34] Combined patient-related queries into one file, simplified file naming --- ...POPULATION => EHDS2_IN_INITIAL_POPULATION} | 0 ...TRAT_DEF_OBSERVATION => EHDS2_OBSERVATION} | 0 resources/cql/EHDS2_PATIENT | 38 +++++++++++++++++++ .../cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER | 9 ----- .../cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER | 6 --- resources/cql/EHDS2_STRAT_GENDER_STRATIFIER | 2 - .../EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER | 9 ----- ..._STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER | 9 ----- ...DS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER | 9 ----- 9 files changed, 38 insertions(+), 44 deletions(-) rename resources/cql/{EHDS2_STRAT_DEF_IN_INITIAL_POPULATION => EHDS2_IN_INITIAL_POPULATION} (100%) rename resources/cql/{EHDS2_STRAT_DEF_OBSERVATION => EHDS2_OBSERVATION} (100%) create mode 100644 resources/cql/EHDS2_PATIENT delete mode 100644 resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER delete mode 100644 resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER delete mode 100644 resources/cql/EHDS2_STRAT_GENDER_STRATIFIER delete mode 100644 resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER delete mode 100644 resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER delete mode 100644 resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER diff --git a/resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION b/resources/cql/EHDS2_IN_INITIAL_POPULATION similarity index 100% rename from resources/cql/EHDS2_STRAT_DEF_IN_INITIAL_POPULATION rename to resources/cql/EHDS2_IN_INITIAL_POPULATION diff --git a/resources/cql/EHDS2_STRAT_DEF_OBSERVATION b/resources/cql/EHDS2_OBSERVATION similarity index 100% rename from resources/cql/EHDS2_STRAT_DEF_OBSERVATION rename to resources/cql/EHDS2_OBSERVATION diff --git a/resources/cql/EHDS2_PATIENT b/resources/cql/EHDS2_PATIENT new file mode 100644 index 0000000..3fa6ff9 --- /dev/null +++ b/resources/cql/EHDS2_PATIENT @@ -0,0 +1,38 @@ +define Gender: +if (Patient.gender is null) then 'unknown' else Patient.gender +define BaseAgeInYears: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientAge').value + +define AgeInYears: +ToInteger(if (BaseAgeInYears is null or BaseAgeInYears = '-') then '-1' else BaseAgeInYears) + +define AgeClass: +ToString((AgeInYears div 10) * 10) + +define BaseHospitalId: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalId').value + +define HospitalId: +if (BaseHospitalId is null or BaseHospitalId = '-') then 'Unkown' else BaseHospitalId + +define HospitalIdClass: +HospitalId + +define BaseHospitalUnitType: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalUnitType').value + +define HospitalUnitType: +if (BaseHospitalUnitType is null or BaseHospitalUnitType = '-') then 'Unkown' else BaseHospitalUnitType + +define HospitalUnitTypeClass: +HospitalUnitType + +define BaseLaboratoryCode: +Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientLaboratoryCode').value + +define LaboratoryCode: +if (BaseLaboratoryCode is null or BaseLaboratoryCode = '-') then 'Unkown' else BaseLaboratoryCode + +define LaboratoryCodeClass: +LaboratoryCode + diff --git a/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER deleted file mode 100644 index 56970a1..0000000 --- a/resources/cql/EHDS2_STRAT_AGE_CLASS_STRATIFIER +++ /dev/null @@ -1,9 +0,0 @@ -define BaseAgeInYears: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientAge').value - -define AgeInYears: -ToInteger(if (BaseAgeInYears is null or BaseAgeInYears = '-') then '-1' else BaseAgeInYears) - -define AgeClass: -ToString((AgeInYears div 10) * 10) - diff --git a/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER b/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER deleted file mode 100644 index 775b943..0000000 --- a/resources/cql/EHDS2_STRAT_ANTIBIOTIC_STRATIFIER +++ /dev/null @@ -1,6 +0,0 @@ -define function Antibiotic(observation FHIR.Observation): - case FHIRHelpers.ToCode(observation.value.coding.where(system = 'https://ecdc.amr/antibiotic-codes').first()) - when Code 'MEM' from Antibiotic then 'MEM' - when null then 'Unknown' - else 'Unknown' - end diff --git a/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER b/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER deleted file mode 100644 index 4145d04..0000000 --- a/resources/cql/EHDS2_STRAT_GENDER_STRATIFIER +++ /dev/null @@ -1,2 +0,0 @@ -define Gender: -if (Patient.gender is null) then 'unknown' else Patient.gender diff --git a/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER deleted file mode 100644 index 9d3d5c9..0000000 --- a/resources/cql/EHDS2_STRAT_HOSPITAL_ID_CLASS_STRATIFIER +++ /dev/null @@ -1,9 +0,0 @@ -define BaseHospitalId: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalId').value - -define HospitalId: -if (BaseHospitalId is null or BaseHospitalId = '-') then 'Unkown' else BaseHospitalId - -define HospitalIdClass: -HospitalId - diff --git a/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER deleted file mode 100644 index 2f6d596..0000000 --- a/resources/cql/EHDS2_STRAT_HOSPITAL_UNIT_TYPE_CLASS_STRATIFIER +++ /dev/null @@ -1,9 +0,0 @@ -define BaseHospitalUnitType: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalUnitType').value - -define HospitalUnitType: -if (BaseHospitalUnitType is null or BaseHospitalUnitType = '-') then 'Unkown' else BaseHospitalUnitType - -define HospitalUnitTypeClass: -HospitalUnitType - diff --git a/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER b/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER deleted file mode 100644 index bf8543e..0000000 --- a/resources/cql/EHDS2_STRAT_LABORATORY_CODE_CLASS_STRATIFIER +++ /dev/null @@ -1,9 +0,0 @@ -define BaseLaboratoryCode: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientLaboratoryCode').value - -define LaboratoryCode: -if (BaseLaboratoryCode is null or BaseLaboratoryCode = '-') then 'Unkown' else BaseLaboratoryCode - -define LaboratoryCodeClass: -LaboratoryCode - From a9620416bee033f67c95ccd29c47ba55b7a282d1 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Wed, 27 Mar 2024 15:41:52 +0100 Subject: [PATCH 16/34] Standardized the CQL for EHDS2 --- resources/cql/EHDS2_OBSERVATION | 2 +- resources/cql/EHDS2_PATIENT | 44 ++++++++++++++------------------- resources/cql/EHDS2_UTIL | 7 ++++++ 3 files changed, 26 insertions(+), 27 deletions(-) create mode 100644 resources/cql/EHDS2_UTIL diff --git a/resources/cql/EHDS2_OBSERVATION b/resources/cql/EHDS2_OBSERVATION index aca6691..9a93cd3 100644 --- a/resources/cql/EHDS2_OBSERVATION +++ b/resources/cql/EHDS2_OBSERVATION @@ -17,7 +17,7 @@ define function BaseObservationExtensionValue(observation FHIR.Observation, url_ // Return the value of the Observation extension with the given URL define function ObservationExtensionValue(observation FHIR.Observation, url_var String): - if (BaseObservationExtensionValue(observation, url_var) is null or BaseObservationExtensionValue(observation, url_var) = '-') then 'Unkown' else BaseObservationExtensionValue(observation, url_var) + SensibleString(BaseObservationExtensionValue(observation, url_var)) define function ExistsObservationExtensionValue(url_var String, value_var String): exists from [Observation] O diff --git a/resources/cql/EHDS2_PATIENT b/resources/cql/EHDS2_PATIENT index 3fa6ff9..d01ab92 100644 --- a/resources/cql/EHDS2_PATIENT +++ b/resources/cql/EHDS2_PATIENT @@ -1,38 +1,30 @@ +// Return the value of the Patient extension with the given URL. +// Return "Unknown" if not available or empty. +define function PatientExtensionValue(url_var String): + SensibleString(Patient.extension.where(url = url_var).value) + +// Return gender of patient define Gender: -if (Patient.gender is null) then 'unknown' else Patient.gender -define BaseAgeInYears: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientAge').value + SensibleString(Patient.gender) +// Return age of patient, as an integer define AgeInYears: -ToInteger(if (BaseAgeInYears is null or BaseAgeInYears = '-') then '-1' else BaseAgeInYears) + ToInteger(PatientExtensionValue('https://ecdc.amr/fhir/StructureDefinition/PatientAge')) +// Round patient age to the nearest multiple of 10 (i/p for a histogram). +// Return result as a string define AgeClass: -ToString((AgeInYears div 10) * 10) - -define BaseHospitalId: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalId').value + ToString((AgeInYears div 10) * 10) +// Return ID of hospital associated with patient define HospitalId: -if (BaseHospitalId is null or BaseHospitalId = '-') then 'Unkown' else BaseHospitalId - -define HospitalIdClass: -HospitalId - -define BaseHospitalUnitType: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientHospitalUnitType').value + PatientExtensionValue('https://ecdc.amr/fhir/StructureDefinition/PatientHospitalId') +// Return hospital unit type associated with patient define HospitalUnitType: -if (BaseHospitalUnitType is null or BaseHospitalUnitType = '-') then 'Unkown' else BaseHospitalUnitType - -define HospitalUnitTypeClass: -HospitalUnitType - -define BaseLaboratoryCode: -Patient.extension.where(url = 'https://ecdc.amr/fhir/StructureDefinition/PatientLaboratoryCode').value + PatientExtensionValue('https://ecdc.amr/fhir/StructureDefinition/PatientHospitalUnitType') +// Return laboratory code associated with patient define LaboratoryCode: -if (BaseLaboratoryCode is null or BaseLaboratoryCode = '-') then 'Unkown' else BaseLaboratoryCode - -define LaboratoryCodeClass: -LaboratoryCode + PatientExtensionValue('https://ecdc.amr/fhir/StructureDefinition/PatientLaboratoryCode') diff --git a/resources/cql/EHDS2_UTIL b/resources/cql/EHDS2_UTIL new file mode 100644 index 0000000..b47e4c5 --- /dev/null +++ b/resources/cql/EHDS2_UTIL @@ -0,0 +1,7 @@ +// Return "val" if the supplied string value is sensible. +// Return "Unknown" if null or empty. +define function SensibleString(val String): + if (val is null or val = '' or val = '-') + then 'Unkown' + else val + From db56a54ca3d2a2641f9f1764d9454a1db36f9b7a Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Thu, 28 Mar 2024 09:45:39 +0100 Subject: [PATCH 17/34] Added EHDS2/ECDC-specific documentation --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 24bbd2a..7c70d12 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ Creating a sample task using CURL: curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-fefbffffeeff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"ewoJImxhbmciOiAiY3FsIiwKCSJsaWIiOiB7CgkJImNvbnRlbnQiOiBbCgkJCXsKCQkJCSJjb250ZW50VHlwZSI6ICJ0ZXh0L2NxbCIsCgkJCQkiZGF0YSI6ICJiR2xpY21GeWVTQlNaWFJ5YVdWMlpRcDFjMmx1WnlCR1NFbFNJSFpsY25OcGIyNGdKelF1TUM0d0p3cHBibU5zZFdSbElFWklTVkpJWld4d1pYSnpJSFpsY25OcGIyNGdKelF1TUM0d0p3b0tZMjlrWlhONWMzUmxiU0JzYjJsdVl6b2dKMmgwZEhBNkx5OXNiMmx1WXk1dmNtY25DbU52WkdWemVYTjBaVzBnYVdOa01UQTZJQ2RvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNOcFpDOXBZMlF0TVRBbkNtTnZaR1Z6ZVhOMFpXMGdVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxPaUFuYUhSMGNITTZMeTltYUdseUxtSmliWEpwTG1SbEwwTnZaR1ZUZVhOMFpXMHZVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxKd29LQ21OdmJuUmxlSFFnVUdGMGFXVnVkQW9LUWtKTlVrbGZVMVJTUVZSZlIwVk9SRVZTWDFOVVVrRlVTVVpKUlZJS0NrSkNUVkpKWDFOVVVrRlVYMFJGUmw5VFVFVkRTVTFGVGdwcFppQkpia2x1YVhScFlXeFFiM0IxYkdGMGFXOXVJSFJvWlc0Z1cxTndaV05wYldWdVhTQmxiSE5sSUh0OUlHRnpJRXhwYzNROFUzQmxZMmx0Wlc0K0NncENRazFTU1Y5VFZGSkJWRjlUUVUxUVRFVmZWRmxRUlY5VFZGSkJWRWxHU1VWU0NncENRazFTU1Y5VFZGSkJWRjlEVlZOVVQwUkpRVTVmVTFSU1FWUkpSa2xGVWdvS1FrSk5Va2xmVTFSU1FWUmZSRWxCUjA1UFUwbFRYMU5VVWtGVVNVWkpSVklLQ2tKQ1RWSkpYMU5VVWtGVVgwRkhSVjlUVkZKQlZFbEdTVVZTQ2dwQ1FrMVNTVjlUVkZKQlZGOUVSVVpmU1U1ZlNVNUpWRWxCVEY5UVQxQlZURUZVU1U5T0NuUnlkV1U9IgoJCQl9CgkJXSwKCQkicmVzb3VyY2VUeXBlIjogIkxpYnJhcnkiLAoJCSJzdGF0dXMiOiAiYWN0aXZlIiwKCQkidHlwZSI6IHsKCQkJImNvZGluZyI6IFsKCQkJCXsKCQkJCQkiY29kZSI6ICJsb2dpYy1saWJyYXJ5IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbGlicmFyeS10eXBlIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjdmZjUzMmFkLTY5ZTQtNDhlZC1hMmQzLTllZmFmYjYwOWY2MiIKCX0sCgkibWVhc3VyZSI6IHsKCQkiZ3JvdXAiOiBbCgkJCXsKCQkJCSJjb2RlIjogewoJCQkJCSJ0ZXh0IjogInBhdGllbnRzIgoJCQkJfSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkluSW5pdGlhbFBvcHVsYXRpb24iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsLWlkZW50aWZpZXIiCgkJCQkJCX0KCQkJCQl9CgkJCQldLAoJCQkJInN0cmF0aWZpZXIiOiBbCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkdlbmRlciIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiR2VuZGVyIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbCIKCQkJCQkJfQoJCQkJCX0sCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkFnZSIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiQWdlQ2xhc3MiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfSwKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiQ3VzdG9kaWFuIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJDdXN0b2RpYW4iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJkaWFnbm9zaXMiCgkJCQl9LAoJCQkJImV4dGVuc2lvbiI6IFsKCQkJCQl7CgkJCQkJCSJ1cmwiOiAiaHR0cDovL2hsNy5vcmcvZmhpci91cy9jcWZtZWFzdXJlcy9TdHJ1Y3R1cmVEZWZpbml0aW9uL2NxZm0tcG9wdWxhdGlvbkJhc2lzIiwKCQkJCQkJInZhbHVlQ29kZSI6ICJDb25kaXRpb24iCgkJCQkJfQoJCQkJXSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkRpYWdub3NpcyIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiZGlhZ25vc2lzIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJEaWFnbm9zaXNDb2RlIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbC1pZGVudGlmaWVyIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJzcGVjaW1lbiIKCQkJCX0sCgkJCQkiZXh0ZW5zaW9uIjogWwoJCQkJCXsKCQkJCQkJInVybCI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3VzL2NxZm1lYXN1cmVzL1N0cnVjdHVyZURlZmluaXRpb24vY3FmbS1wb3B1bGF0aW9uQmFzaXMiLAoJCQkJCQkidmFsdWVDb2RlIjogIlNwZWNpbWVuIgoJCQkJCX0KCQkJCV0sCgkJCQkicG9wdWxhdGlvbiI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJImNvZGluZyI6IFsKCQkJCQkJCQl7CgkJCQkJCQkJCSJjb2RlIjogImluaXRpYWwtcG9wdWxhdGlvbiIsCgkJCQkJCQkJCSJzeXN0ZW0iOiAiaHR0cDovL3Rlcm1pbm9sb2d5LmhsNy5vcmcvQ29kZVN5c3RlbS9tZWFzdXJlLXBvcHVsYXRpb24iCgkJCQkJCQkJfQoJCQkJCQkJXQoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJTcGVjaW1lbiIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAic2FtcGxlX2tpbmQiCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIlNhbXBsZVR5cGUiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9CgkJXSwKCQkibGlicmFyeSI6ICJ1cm46dXVpZDo3ZmY1MzJhZC02OWU0LTQ4ZWQtYTJkMy05ZWZhZmI2MDlmNjIiLAoJCSJyZXNvdXJjZVR5cGUiOiAiTWVhc3VyZSIsCgkJInNjb3JpbmciOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiY29ob3J0IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbWVhc3VyZS1zY29yaW5nIgoJCQkJfQoJCQldCgkJfSwKCQkic3RhdHVzIjogImFjdGl2ZSIsCgkJInN1YmplY3RDb2RlYWJsZUNvbmNlcHQiOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiUGF0aWVudCIsCgkJCQkJInN5c3RlbSI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3Jlc291cmNlLXR5cGVzIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjVlZThkZTczLTM0N2UtNDdjYS1hMDE0LWYyZTcxNzY3YWRmYyIKCX0KfQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` +## Changes specific to EHDS2/ECDC + +The files ```resources/cql/EHDS2_*``` contain the CQL templates used in the EHDS2 ECDC pilot project. I have chosen to outsource all CQL to focus. EHDS2_UTIL provides utility functions used frequently. EHDS2_PATIENT and EHDS2_OBSERVATION contain the queries relating to Patient and Observation resources respectively. There are small but important differences between Patient and Observation in the way that queries work, which you will notice if you look closely at the definitions in these two files. Finally, EHDS2_IN_INITIAL_POPULATION is used to build the actual query. This file just contains a stub; the full query is built in Lens, comprising of calls to the functions and variable definitions found in EHDS2_PATIENT and EHDS2_OBSERVATION. + ## License This code is licensed under the Apache License 2.0. For details, please see [LICENSE](./LICENSE) From 1c9108bf9fba855c9d910f9d3dc3539c85a2a4a6 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Thu, 6 Jun 2024 13:21:23 +0200 Subject: [PATCH 18/34] Added CQL for dealing with specimens --- resources/cql/EHDS2_SPECIMEN | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 resources/cql/EHDS2_SPECIMEN diff --git a/resources/cql/EHDS2_SPECIMEN b/resources/cql/EHDS2_SPECIMEN new file mode 100644 index 0000000..7bdca37 --- /dev/null +++ b/resources/cql/EHDS2_SPECIMEN @@ -0,0 +1,13 @@ +define SpecimenList: + if InInitialPopulation then [Specimen] else {} as List + +// Return the isolate ID for the given Specimen +define function SpecimenIsolateId(specimen FHIR.Specimen): + SensibleString(specimen.id) + +// Check to see if there are any Specimen resources with the +// given isolate ID. +define function ExistsSpecimenIsolateId(id_var String): + exists from [Specimen] S + where SpecimenIsolateId(S) = id_var + From ec7f0803da86d6c56f9a4b79f2002011e9a9d6ab Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Wed, 12 Jun 2024 13:56:47 +0200 Subject: [PATCH 19/34] Adding searches for laboratories and hospitals --- resources/cql/EHDS2_HOSPITAL | 18 ++++++++++++++++++ resources/cql/EHDS2_LABORATORY | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 resources/cql/EHDS2_HOSPITAL create mode 100644 resources/cql/EHDS2_LABORATORY diff --git a/resources/cql/EHDS2_HOSPITAL b/resources/cql/EHDS2_HOSPITAL new file mode 100644 index 0000000..ddd2d47 --- /dev/null +++ b/resources/cql/EHDS2_HOSPITAL @@ -0,0 +1,18 @@ +// Return a list of all Hospitals. +define HospitalList: + if InInitialPopulation then + [CareTeam] O + where StartsWith(O.name,'Hospital') + else + {} as List + +// Return the ID for the given hospital CareTeam +define function HospitalId(careTeam FHIR.CareTeam): + SensibleString(careTeam.id) + +// Check to see if there are any CareTeam resources with the +// given ID and names that start with "Hospital" +define function ExistsHospitalId(id_var String): + exists from [CareTeam] O + where HospitalId(O) = id_var + and StartsWith(O.name,'Hospital') diff --git a/resources/cql/EHDS2_LABORATORY b/resources/cql/EHDS2_LABORATORY new file mode 100644 index 0000000..f0eb07f --- /dev/null +++ b/resources/cql/EHDS2_LABORATORY @@ -0,0 +1,18 @@ +// Return a list of all Laboratories. +define LaboratoryList: + if InInitialPopulation then + [CareTeam] O + where StartsWith(O.name,'Laboratory') + else + {} as List + +// Return the ID for the given laboratory CareTeam +define function LaboratoryId(careTeam FHIR.CareTeam): + SensibleString(careTeam.id) + +// Check to see if there are any CareTeam resources with the +// given ID and names that start with "Laboratory" +define function ExistsLaboratoryId(id_var String): + exists from [CareTeam] O + where LaboratoryId(O) = id_var + and StartsWith(O.name,'Laboratory') From 8a5c00f909828137e6bbf427a03bfdc840592900 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Fri, 14 Jun 2024 15:29:34 +0200 Subject: [PATCH 20/34] Added CQL for Reference Guide SIR --- resources/cql/EHDS2_REFGUIDE | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 resources/cql/EHDS2_REFGUIDE diff --git a/resources/cql/EHDS2_REFGUIDE b/resources/cql/EHDS2_REFGUIDE new file mode 100644 index 0000000..8bd2fe8 --- /dev/null +++ b/resources/cql/EHDS2_REFGUIDE @@ -0,0 +1,18 @@ +// Return a list of all Refguides. +define RefguideList: + if InInitialPopulation then + [CareTeam] O + where StartsWith(O.name,'Refguide') + else + {} as List + +// Return the ID for the given hospital CareTeam +define function RefguideId(careTeam FHIR.CareTeam): + SensibleString(careTeam.id) + +// Check to see if there are any CareTeam resources with the +// given ID and names that start with "Refguide" +define function ExistsRefguideId(id_var String): + exists from [CareTeam] O + where RefguideId(O) = id_var + and StartsWith(O.name,'Refguide') From fbd43010a6633e41fb227153d6bf83313f803d98 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Thu, 11 Jul 2024 15:13:56 +0200 Subject: [PATCH 21/34] Use stratifiers to populate Labs, Hospitals and Refguides The "CareTeam hack" is no longer needed, so I can remove the corresponding CQL. --- resources/cql/EHDS2_HOSPITAL | 18 ------------------ resources/cql/EHDS2_LABORATORY | 18 ------------------ resources/cql/EHDS2_REFGUIDE | 18 ------------------ 3 files changed, 54 deletions(-) delete mode 100644 resources/cql/EHDS2_HOSPITAL delete mode 100644 resources/cql/EHDS2_LABORATORY delete mode 100644 resources/cql/EHDS2_REFGUIDE diff --git a/resources/cql/EHDS2_HOSPITAL b/resources/cql/EHDS2_HOSPITAL deleted file mode 100644 index ddd2d47..0000000 --- a/resources/cql/EHDS2_HOSPITAL +++ /dev/null @@ -1,18 +0,0 @@ -// Return a list of all Hospitals. -define HospitalList: - if InInitialPopulation then - [CareTeam] O - where StartsWith(O.name,'Hospital') - else - {} as List - -// Return the ID for the given hospital CareTeam -define function HospitalId(careTeam FHIR.CareTeam): - SensibleString(careTeam.id) - -// Check to see if there are any CareTeam resources with the -// given ID and names that start with "Hospital" -define function ExistsHospitalId(id_var String): - exists from [CareTeam] O - where HospitalId(O) = id_var - and StartsWith(O.name,'Hospital') diff --git a/resources/cql/EHDS2_LABORATORY b/resources/cql/EHDS2_LABORATORY deleted file mode 100644 index f0eb07f..0000000 --- a/resources/cql/EHDS2_LABORATORY +++ /dev/null @@ -1,18 +0,0 @@ -// Return a list of all Laboratories. -define LaboratoryList: - if InInitialPopulation then - [CareTeam] O - where StartsWith(O.name,'Laboratory') - else - {} as List - -// Return the ID for the given laboratory CareTeam -define function LaboratoryId(careTeam FHIR.CareTeam): - SensibleString(careTeam.id) - -// Check to see if there are any CareTeam resources with the -// given ID and names that start with "Laboratory" -define function ExistsLaboratoryId(id_var String): - exists from [CareTeam] O - where LaboratoryId(O) = id_var - and StartsWith(O.name,'Laboratory') diff --git a/resources/cql/EHDS2_REFGUIDE b/resources/cql/EHDS2_REFGUIDE deleted file mode 100644 index 8bd2fe8..0000000 --- a/resources/cql/EHDS2_REFGUIDE +++ /dev/null @@ -1,18 +0,0 @@ -// Return a list of all Refguides. -define RefguideList: - if InInitialPopulation then - [CareTeam] O - where StartsWith(O.name,'Refguide') - else - {} as List - -// Return the ID for the given hospital CareTeam -define function RefguideId(careTeam FHIR.CareTeam): - SensibleString(careTeam.id) - -// Check to see if there are any CareTeam resources with the -// given ID and names that start with "Refguide" -define function ExistsRefguideId(id_var String): - exists from [CareTeam] O - where RefguideId(O) = id_var - and StartsWith(O.name,'Refguide') From b4b235264a26b68ab5bed34cc6e5e1350bc8aac3 Mon Sep 17 00:00:00 2001 From: Gerhard Salvini Date: Thu, 18 Jul 2024 12:26:04 +0200 Subject: [PATCH 22/34] Added CQL functions for "date used for statistics" and "date valid from" --- resources/cql/EHDS2_OBSERVATION | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/resources/cql/EHDS2_OBSERVATION b/resources/cql/EHDS2_OBSERVATION index 9a93cd3..a5542ac 100644 --- a/resources/cql/EHDS2_OBSERVATION +++ b/resources/cql/EHDS2_OBSERVATION @@ -95,3 +95,31 @@ define function ReportingCountry(observation FHIR.Observation): define function ExistsReportingCountry(value_var String): ExistsObservationExtensionValue('https://ecdc.amr/fhir/StructureDefinition/ObservationReportingCountry', value_var) +// Return the year from the date used for statistics +define function YearDateUsedForStatistics(observation FHIR.Observation): + year from observation.issued + +// Return the month from the date used for statistics +define function MonthDateUsedForStatistics(observation FHIR.Observation): + month from observation.issued + +// Return the year-month from the date used for statistics +define function YearMonthDateUsedForStatistics(observation FHIR.Observation): + ToString(YearDateUsedForStatistics(observation)) + '-' + ToString(MonthDateUsedForStatistics(observation)) + +// Return the the date valid from +define function DateValidFrom(observation FHIR.Observation): + ToDate(observation.effective as dateTime) + +// Return the year from the date valid from +define function YearDateValidFrom(observation FHIR.Observation): + year from DateValidFrom(observation) + +// Return the month from the date valid from +define function MonthDateValidFrom(observation FHIR.Observation): + month from DateValidFrom(observation) + +// Return the year-month from the date valid from +define function YearMonthDateValidFrom(observation FHIR.Observation): + ToString(YearDateValidFrom(observation)) + '-' + ToString(MonthDateValidFrom(observation)) + From c24f69c5886242369492258b891e711201a349dc Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Mon, 29 Jul 2024 09:52:02 +0200 Subject: [PATCH 23/34] Overwrite slightly changed files with the latest develop versions --- README.md | 54 ++++++----- src/main.rs | 260 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 189 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 33e9ff6..1718f1a 100644 --- a/README.md +++ b/README.md @@ -34,68 +34,72 @@ BEAM_APP_ID_LONG = "app1.broker.example.com" ### Optional variables ```bash -RETRY_COUNT = "32" # The maximum number of retries for beam and blaze healthchecks, default value: 32 -ENDPOINT_TYPE = "blaze" # Type of the endpoint, allowed values: "blaze", "omop", default value: "blaze" +RETRY_COUNT = "32" # The maximum number of retries for beam and blaze healthchecks; default value: 32 +ENDPOINT_TYPE = "blaze" # Type of the endpoint, allowed values: "blaze", "omop", "sql", "blaze-and-sql"; default value: "blaze" EXPORTER_URL = " https://exporter.site/" # The exporter URL -OBFUSCATE = "yes" # Should the results be obfuscated - the "master switch", allowed values: "yes", "no", default value: "yes" -OBFUSCATE_BELOW_10_MODE = "1" # The mode of obfuscating values below 10: 0 - return zero, 1 - return ten, 2 - obfuscate using Laplace distribution and rounding, has no effect if OBFUSCATE = "no", default value: 1 -DELTA_PATIENT = "1." # Sensitivity parameter for obfuscating the counts in the Patient stratifier, has no effect if OBFUSCATE = "no", default value: 1 -DELTA_SPECIMEN = "20." # Sensitivity parameter for obfuscating the counts in the Specimen stratifier, has no effect if OBFUSCATE = "no", default value: 20 -DELTA_DIAGNOSIS = "3." # Sensitivity parameter for obfuscating the counts in the Diagnosis stratifier, has no effect if OBFUSCATE = "no", default value: 3 -DELTA_PROCEDURES = "1.7" # Sensitivity parameter for obfuscating the counts in the Procedures stratifier, has no effect if OBFUSCATE = "no", default value: 1.7 -DELTA_MEDICATION_STATEMENTS = "2.1" # Sensitivity parameter for obfuscating the counts in the Medication Statements stratifier, has no effect if OBFUSCATE = "no", default value: 2.1 -DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Histo stratifier, has no effect if OBFUSCATE = "no", default value: 20 -EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1 -ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10 -PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ;, default value: "exliquid;dktk_supervisors;exporter;ehds2" -QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached -PROVIDER = "name" #OMOP provider name -PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" #Base64 encoded OMOP provider icon +OBFUSCATE = "yes" # Should the results be obfuscated - the "master switch", allowed values: "yes", "no"; default value: "yes" +OBFUSCATE_BELOW_10_MODE = "1" # The mode of obfuscating values below 10: 0 - return zero, 1 - return ten, 2 - obfuscate using Laplace distribution and rounding, has no effect if OBFUSCATE = "no"; default value: 1 +DELTA_PATIENT = "1." # Sensitivity parameter for obfuscating the counts in the Patient stratifier, has no effect if OBFUSCATE = "no"; default value: 1 +DELTA_SPECIMEN = "20." # Sensitivity parameter for obfuscating the counts in the Specimen stratifier, has no effect if OBFUSCATE = "no"; default value: 20 +DELTA_DIAGNOSIS = "3." # Sensitivity parameter for obfuscating the counts in the Diagnosis stratifier, has no effect if OBFUSCATE = "no"; default value: 3 +DELTA_PROCEDURES = "1.7" # Sensitivity parameter for obfuscating the counts in the Procedures stratifier, has no effect if OBFUSCATE = "no"; default value: 1.7 +DELTA_MEDICATION_STATEMENTS = "2.1" # Sensitivity parameter for obfuscating the counts in the Medication Statements stratifier, has no effect if OBFUSCATE = "no"; default value: 2.1 +DELTA_HISTO = "20." # Sensitivity parameter for obfuscating the counts in the Histo stratifier, has no effect if OBFUSCATE = "no"; default value: 20 +EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no"; default value: 0.1 +ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no"; default value: 10 +PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects for which the results are not to be obfuscated, separated by ";" ; default value: "exliquid;dktk_supervisors;exporter;ehds2" +QUERIES_TO_CACHE = "queries_to_cache.conf" # The path to a file containing base64 encoded queries whose results are to be cached. If not set, no results are cached +PROVIDER = "name" #EUCAIM provider name +PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" # Base64 encoded EUCAIM provider icon AUTH_HEADER = "ApiKey XXXX" #Authorization header +POSTGRES_CONNECTION_STRING = "postgresql://postgres:Test.123@localhost:5432/postgres" # Postgres connection string ``` Obfuscating zero counts is by default switched off. To enable obfuscating zero counts, set the env. variable `OBFUSCATE_ZERO`. Optionally, you can provide the `TLS_CA_CERTIFICATES_DIR` environment variable to add additional trusted certificates, e.g., if you have a TLS-terminating proxy server in place. The application respects the `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, `NO_PROXY`, and their respective lowercase equivalents. +Log level can be set using the `RUST_LOG` environment variable. + ## Usage -Creating a sample focus healthcheck task using CURL (body can be any string and is ignored): +Creating a sample focus healthcheck task using curl (body can be any string and is ignored): ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"focus-healthcheck"},"body":"wie geht es"}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample task containing a [Blaze](https://github.com/samply/blaze) query using CURL: +Creating a sample task containing a [Blaze](https://github.com/samply/blaze) query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-fefbffffeeff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"ewoJImxhbmciOiAiY3FsIiwKCSJsaWIiOiB7CgkJImNvbnRlbnQiOiBbCgkJCXsKCQkJCSJjb250ZW50VHlwZSI6ICJ0ZXh0L2NxbCIsCgkJCQkiZGF0YSI6ICJiR2xpY21GeWVTQlNaWFJ5YVdWMlpRcDFjMmx1WnlCR1NFbFNJSFpsY25OcGIyNGdKelF1TUM0d0p3cHBibU5zZFdSbElFWklTVkpJWld4d1pYSnpJSFpsY25OcGIyNGdKelF1TUM0d0p3b0tZMjlrWlhONWMzUmxiU0JzYjJsdVl6b2dKMmgwZEhBNkx5OXNiMmx1WXk1dmNtY25DbU52WkdWemVYTjBaVzBnYVdOa01UQTZJQ2RvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNOcFpDOXBZMlF0TVRBbkNtTnZaR1Z6ZVhOMFpXMGdVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxPaUFuYUhSMGNITTZMeTltYUdseUxtSmliWEpwTG1SbEwwTnZaR1ZUZVhOMFpXMHZVMkZ0Y0d4bFRXRjBaWEpwWVd4VWVYQmxKd29LQ21OdmJuUmxlSFFnVUdGMGFXVnVkQW9LUWtKTlVrbGZVMVJTUVZSZlIwVk9SRVZTWDFOVVVrRlVTVVpKUlZJS0NrSkNUVkpKWDFOVVVrRlVYMFJGUmw5VFVFVkRTVTFGVGdwcFppQkpia2x1YVhScFlXeFFiM0IxYkdGMGFXOXVJSFJvWlc0Z1cxTndaV05wYldWdVhTQmxiSE5sSUh0OUlHRnpJRXhwYzNROFUzQmxZMmx0Wlc0K0NncENRazFTU1Y5VFZGSkJWRjlUUVUxUVRFVmZWRmxRUlY5VFZGSkJWRWxHU1VWU0NncENRazFTU1Y5VFZGSkJWRjlEVlZOVVQwUkpRVTVmVTFSU1FWUkpSa2xGVWdvS1FrSk5Va2xmVTFSU1FWUmZSRWxCUjA1UFUwbFRYMU5VVWtGVVNVWkpSVklLQ2tKQ1RWSkpYMU5VVWtGVVgwRkhSVjlUVkZKQlZFbEdTVVZTQ2dwQ1FrMVNTVjlUVkZKQlZGOUVSVVpmU1U1ZlNVNUpWRWxCVEY5UVQxQlZURUZVU1U5T0NuUnlkV1U9IgoJCQl9CgkJXSwKCQkicmVzb3VyY2VUeXBlIjogIkxpYnJhcnkiLAoJCSJzdGF0dXMiOiAiYWN0aXZlIiwKCQkidHlwZSI6IHsKCQkJImNvZGluZyI6IFsKCQkJCXsKCQkJCQkiY29kZSI6ICJsb2dpYy1saWJyYXJ5IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbGlicmFyeS10eXBlIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjdmZjUzMmFkLTY5ZTQtNDhlZC1hMmQzLTllZmFmYjYwOWY2MiIKCX0sCgkibWVhc3VyZSI6IHsKCQkiZ3JvdXAiOiBbCgkJCXsKCQkJCSJjb2RlIjogewoJCQkJCSJ0ZXh0IjogInBhdGllbnRzIgoJCQkJfSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkluSW5pdGlhbFBvcHVsYXRpb24iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsLWlkZW50aWZpZXIiCgkJCQkJCX0KCQkJCQl9CgkJCQldLAoJCQkJInN0cmF0aWZpZXIiOiBbCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkdlbmRlciIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiR2VuZGVyIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbCIKCQkJCQkJfQoJCQkJCX0sCgkJCQkJewoJCQkJCQkiY29kZSI6IHsKCQkJCQkJCSJ0ZXh0IjogIkFnZSIKCQkJCQkJfSwKCQkJCQkJImNyaXRlcmlhIjogewoJCQkJCQkJImV4cHJlc3Npb24iOiAiQWdlQ2xhc3MiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfSwKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiQ3VzdG9kaWFuIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJDdXN0b2RpYW4iLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJkaWFnbm9zaXMiCgkJCQl9LAoJCQkJImV4dGVuc2lvbiI6IFsKCQkJCQl7CgkJCQkJCSJ1cmwiOiAiaHR0cDovL2hsNy5vcmcvZmhpci91cy9jcWZtZWFzdXJlcy9TdHJ1Y3R1cmVEZWZpbml0aW9uL2NxZm0tcG9wdWxhdGlvbkJhc2lzIiwKCQkJCQkJInZhbHVlQ29kZSI6ICJDb25kaXRpb24iCgkJCQkJfQoJCQkJXSwKCQkJCSJwb3B1bGF0aW9uIjogWwoJCQkJCXsKCQkJCQkJImNvZGUiOiB7CgkJCQkJCQkiY29kaW5nIjogWwoJCQkJCQkJCXsKCQkJCQkJCQkJImNvZGUiOiAiaW5pdGlhbC1wb3B1bGF0aW9uIiwKCQkJCQkJCQkJInN5c3RlbSI6ICJodHRwOi8vdGVybWlub2xvZ3kuaGw3Lm9yZy9Db2RlU3lzdGVtL21lYXN1cmUtcG9wdWxhdGlvbiIKCQkJCQkJCQl9CgkJCQkJCQldCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIkRpYWdub3NpcyIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAiZGlhZ25vc2lzIgoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJEaWFnbm9zaXNDb2RlIiwKCQkJCQkJCSJsYW5ndWFnZSI6ICJ0ZXh0L2NxbC1pZGVudGlmaWVyIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9LAoJCQl7CgkJCQkiY29kZSI6IHsKCQkJCQkidGV4dCI6ICJzcGVjaW1lbiIKCQkJCX0sCgkJCQkiZXh0ZW5zaW9uIjogWwoJCQkJCXsKCQkJCQkJInVybCI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3VzL2NxZm1lYXN1cmVzL1N0cnVjdHVyZURlZmluaXRpb24vY3FmbS1wb3B1bGF0aW9uQmFzaXMiLAoJCQkJCQkidmFsdWVDb2RlIjogIlNwZWNpbWVuIgoJCQkJCX0KCQkJCV0sCgkJCQkicG9wdWxhdGlvbiI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJImNvZGluZyI6IFsKCQkJCQkJCQl7CgkJCQkJCQkJCSJjb2RlIjogImluaXRpYWwtcG9wdWxhdGlvbiIsCgkJCQkJCQkJCSJzeXN0ZW0iOiAiaHR0cDovL3Rlcm1pbm9sb2d5LmhsNy5vcmcvQ29kZVN5c3RlbS9tZWFzdXJlLXBvcHVsYXRpb24iCgkJCQkJCQkJfQoJCQkJCQkJXQoJCQkJCQl9LAoJCQkJCQkiY3JpdGVyaWEiOiB7CgkJCQkJCQkiZXhwcmVzc2lvbiI6ICJTcGVjaW1lbiIsCgkJCQkJCQkibGFuZ3VhZ2UiOiAidGV4dC9jcWwtaWRlbnRpZmllciIKCQkJCQkJfQoJCQkJCX0KCQkJCV0sCgkJCQkic3RyYXRpZmllciI6IFsKCQkJCQl7CgkJCQkJCSJjb2RlIjogewoJCQkJCQkJInRleHQiOiAic2FtcGxlX2tpbmQiCgkJCQkJCX0sCgkJCQkJCSJjcml0ZXJpYSI6IHsKCQkJCQkJCSJleHByZXNzaW9uIjogIlNhbXBsZVR5cGUiLAoJCQkJCQkJImxhbmd1YWdlIjogInRleHQvY3FsIgoJCQkJCQl9CgkJCQkJfQoJCQkJXQoJCQl9CgkJXSwKCQkibGlicmFyeSI6ICJ1cm46dXVpZDo3ZmY1MzJhZC02OWU0LTQ4ZWQtYTJkMy05ZWZhZmI2MDlmNjIiLAoJCSJyZXNvdXJjZVR5cGUiOiAiTWVhc3VyZSIsCgkJInNjb3JpbmciOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiY29ob3J0IiwKCQkJCQkic3lzdGVtIjogImh0dHA6Ly90ZXJtaW5vbG9neS5obDcub3JnL0NvZGVTeXN0ZW0vbWVhc3VyZS1zY29yaW5nIgoJCQkJfQoJCQldCgkJfSwKCQkic3RhdHVzIjogImFjdGl2ZSIsCgkJInN1YmplY3RDb2RlYWJsZUNvbmNlcHQiOiB7CgkJCSJjb2RpbmciOiBbCgkJCQl7CgkJCQkJImNvZGUiOiAiUGF0aWVudCIsCgkJCQkJInN5c3RlbSI6ICJodHRwOi8vaGw3Lm9yZy9maGlyL3Jlc291cmNlLXR5cGVzIgoJCQkJfQoJCQldCgkJfSwKCQkidXJsIjogInVybjp1dWlkOjVlZThkZTczLTM0N2UtNDdjYS1hMDE0LWYyZTcxNzY3YWRmYyIKCX0KfQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample task containing an abstract syntax tree (AST) query using CURL: +Creating a sample task containing an abstract syntax tree (AST) query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"bbmri"},"body":"eyJsYW5nIjoiYXN0IiwicGF5bG9hZCI6ImV5SmhjM1FpT25zaWIzQmxjbUZ1WkNJNklrOVNJaXdpWTJocGJHUnlaVzRpT2x0N0ltOXdaWEpoYm1RaU9pSkJUa1FpTENKamFHbHNaSEpsYmlJNlczc2liM0JsY21GdVpDSTZJazlTSWl3aVkyaHBiR1J5Wlc0aU9sdDdJbXRsZVNJNkltZGxibVJsY2lJc0luUjVjR1VpT2lKRlVWVkJURk1pTENKemVYTjBaVzBpT2lJaUxDSjJZV3gxWlNJNkltMWhiR1VpZlN4N0ltdGxlU0k2SW1kbGJtUmxjaUlzSW5SNWNHVWlPaUpGVVZWQlRGTWlMQ0p6ZVhOMFpXMGlPaUlpTENKMllXeDFaU0k2SW1abGJXRnNaU0o5WFgxZGZWMTlMQ0pwWkNJNkltRTJaakZqWTJZekxXVmlaakV0TkRJMFppMDVaRFk1TFRSbE5XUXhNelZtTWpNME1DSjkifQ=="}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using CURL: +Creating a sample SQL task for a `SELECT_TABLES` query using curl: +```bash + curl -v -X POST -H "Content-Type: application/json" --data '{"id":"7fffefff-ffef-fcff-feef-feffffffffff","from":"app1.proxy1.broker","to":["app1.proxy1.broker"],"ttl":"10s","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"metadata":{"project":"exliquid"},"body":"eyJwYXlsb2FkIjoiU0VMRUNUX1RBQkxFUyJ9"}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks + ``` + +Creating a sample [Exporter](https://github.com/samply/exporter) "execute" task containing an Exporter query using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktY29udGV4dCIgOiAiVUZKUFNrVkRWQzFKUkQxa01qaGhZVEl5Wm1Wa01USTBNemM0T0RWallnPT0iLA0KICAicXVlcnktbGFiZWwiIDogIlRlc3QgMyIsDQogICJxdWVyeS1leGVjdXRpb24tY29udGFjdC1pZCIgOiAiYmstYWRtaW5AdGVzdC5kZmt6LmRlIiwNCiAgInF1ZXJ5LWRlc2NyaXB0aW9uIiA6ICJUaGlzIGlzIHRoZSB0ZXN0IDMiLA0KICAicXVlcnktZXhwaXJhdGlvbi1kYXRlIiA6ICIyMDI0LTA4LTE0IiwNCiAgIm91dHB1dC1mb3JtYXQiIDogIkVYQ0VMIiwNCiAgInF1ZXJ5IiA6ICJleUpzWVc1bklqb2lZM0ZzSWl3aWJHbGlJanA3SW5KbGMyOTFjbU5sVkhsd1pTSTZJa3hwWW5KaGNua2lMQ0oxY213aU9pSjFjbTQ2ZFhWcFpEcGpOelJrWmpJd05DMDFZalppTFRSaFpXUXRZakl5T0MwM1pqVXpNekE0TnpZME5UZ2lMQ0p6ZEdGMGRYTWlPaUpoWTNScGRtVWlMQ0owZVhCbElqcDdJbU52WkdsdVp5STZXM3NpYzNsemRHVnRJam9pYUhSMGNEb3ZMM1JsY20xcGJtOXNiMmQ1TG1oc055NXZjbWN2UTI5a1pWTjVjM1JsYlM5c2FXSnlZWEo1TFhSNWNHVWlMQ0pqYjJSbElqb2liRzluYVdNdGJHbGljbUZ5ZVNKOVhYMHNJbU52Ym5SbGJuUWlPbHQ3SW1OdmJuUmxiblJVZVhCbElqb2lkR1Y0ZEM5amNXd2lMQ0prWVhSaElqb2lZa2RzYVdOdFJubGxVMEpUV2xoU2VXRlhWakphVVhBeFl6SnNkVnA1UWtkVFJXeFRTVWhhYkdOdVRuQmlNalJuU25wUmRVMUROSGRLZDNCd1ltMU9jMlJYVW14SlJWcEpVMVpLU1ZwWGVIZGFXRXA2U1VoYWJHTnVUbkJpTWpSblNucFJkVTFETkhkS2QyOUxXVEk1YTFwWVRqVmpNMUpzWWxOQ2MySXliSFZaZW05blNqSm9NR1JJUVRaTWVUbHpZakpzZFZsNU5YWmpiV051UTJkd2FtSXlOVEJhV0dnd1NVWkNhR1JIYkd4aWJsRkxRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3daRVpVYTFKR1ZXdzVWRlpHU2tKV1JXeEhVMVZXVTBObmNFVlRNVkpNV0RGT1ZWVnJSbFZZTVVKVFUxVXhRbFZzYkdaU1JXeENVakExVUZVd2JGUllNVTVWVld0R1ZWTlZXa3BTVmtsTFVrVjBWVk14T1ZSV1JrcENWa1k1UWxJd1ZtWlJNSGhDVlRGT1psVXhVbE5SVmxKS1VtdHNSbFZuYjB0U1JYUlZVekU1VkZaR1NrSldSamxGVWxWT1JsRldUa1pTUmpsVVZrWktRbFpGYkVkVFZWWlRRMmR3UlZNeFVreFlNVTVWVld0R1ZWZ3dVa3BSVldSUFZERk9TbFV4T1ZSV1JrcENWa1ZzUjFOVlZsTkRaM0JGVXpGU1RGZ3hUbFZWYTBaVldERk9VVkpWVGtwVVZWWlBXREZPVlZWclJsVlRWVnBLVWxaSlMwTnJVa3hXUlhSbVZURlNVMUZXVW1aVlJrcFFVVEJXUlZaV1NrWllNVTVWVld0R1ZWTlZXa3BTVmtsTFEydFNURlpGZEdaVk1WSlRVVlpTWmxSVlZrVlRWVTVDVmtWc1VGUnNPVlJXUmtwQ1ZrVnNSMU5WVmxORGExSk1Wa1YwWmxVeFVsTlJWbEptVWtWV1IxZ3diRTlZTUd4UFUxWlNTbEZWZUdaVlJUbFJWbFY0UWxaRmJGQlViRUpvWkVkc2JHSnVVWFZhTWxaMVdrZFdlVWxFTUdkS01qRm9Za2RWYmlKOVhYMHNJbTFsWVhOMWNtVWlPbnNpY21WemIzVnlZMlZVZVhCbElqb2lUV1ZoYzNWeVpTSXNJblZ5YkNJNkluVnlianAxZFdsa09qaG1NMlV6WVRZeExXRXdPVGN0TkRoa05DMWlOMkZqTFRobE5ESTNZbVU0WVdNMFpDSXNJbk4wWVhSMWN5STZJbUZqZEdsMlpTSXNJbk4xWW1wbFkzUkRiMlJsWVdKc1pVTnZibU5sY0hRaU9uc2lZMjlrYVc1bklqcGJleUp6ZVhOMFpXMGlPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNKbGMyOTFjbU5sTFhSNWNHVnpJaXdpWTI5a1pTSTZJbEJoZEdsbGJuUWlmVjE5TENKc2FXSnlZWEo1SWpvaWRYSnVPblYxYVdRNll6YzBaR1l5TURRdE5XSTJZaTAwWVdWa0xXSXlNamd0TjJZMU16TXdPRGMyTkRVNElpd2ljMk52Y21sdVp5STZleUpqYjJScGJtY2lPbHQ3SW5ONWMzUmxiU0k2SW1oMGRIQTZMeTkwWlhKdGFXNXZiRzluZVM1b2JEY3ViM0puTDBOdlpHVlRlWE4wWlcwdmJXVmhjM1Z5WlMxelkyOXlhVzVuSWl3aVkyOWtaU0k2SW1OdmFHOXlkQ0o5WFgwc0ltZHliM1Z3SWpwYmV5SmpiMlJsSWpwN0luUmxlSFFpT2lKd1lYUnBaVzUwY3lKOUxDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa2x1U1c1cGRHbGhiRkJ2Y0hWc1lYUnBiMjRpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpIWlc1a1pYSWlmU3dpWTNKcGRHVnlhV0VpT25zaWJHRnVaM1ZoWjJVaU9pSjBaWGgwTDJOeGJDSXNJbVY0Y0hKbGMzTnBiMjRpT2lKSFpXNWtaWElpZlgwc2V5SmpiMlJsSWpwN0luUmxlSFFpT2lJM05URTROaTAzSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lSR1ZqWldGelpXUWlmWDBzZXlKamIyUmxJanA3SW5SbGVIUWlPaUpCWjJVaWZTd2lZM0pwZEdWeWFXRWlPbnNpYkdGdVozVmhaMlVpT2lKMFpYaDBMMk54YkNJc0ltVjRjSEpsYzNOcGIyNGlPaUpCWjJWRGJHRnpjeUo5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaVpHbGhaMjV2YzJsekluMHNJbVY0ZEdWdWMybHZiaUk2VzNzaWRYSnNJam9pYUhSMGNEb3ZMMmhzTnk1dmNtY3ZabWhwY2k5MWN5OWpjV1p0WldGemRYSmxjeTlUZEhKMVkzUjFjbVZFWldacGJtbDBhVzl1TDJOeFptMHRjRzl3ZFd4aGRHbHZia0poYzJseklpd2lkbUZzZFdWRGIyUmxJam9pUTI5dVpHbDBhVzl1SW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJa1JwWVdkdWIzTnBjeUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbVJwWVdkdWIzTnBjeUo5TENKamNtbDBaWEpwWVNJNmV5SnNZVzVuZFdGblpTSTZJblJsZUhRdlkzRnNMV2xrWlc1MGFXWnBaWElpTENKbGVIQnlaWE56YVc5dUlqb2lSR2xoWjI1dmMybHpRMjlrWlNKOWZWMTlMSHNpWTI5a1pTSTZleUowWlhoMElqb2ljM0JsWTJsdFpXNGlmU3dpWlhoMFpXNXphVzl1SWpwYmV5SjFjbXdpT2lKb2RIUndPaTh2YUd3M0xtOXlaeTltYUdseUwzVnpMMk54Wm0xbFlYTjFjbVZ6TDFOMGNuVmpkSFZ5WlVSbFptbHVhWFJwYjI0dlkzRm1iUzF3YjNCMWJHRjBhVzl1UW1GemFYTWlMQ0oyWVd4MVpVTnZaR1VpT2lKVGNHVmphVzFsYmlKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpUY0dWamFXMWxiaUo5ZlYwc0luTjBjbUYwYVdacFpYSWlPbHQ3SW1OdlpHVWlPbnNpZEdWNGRDSTZJbk5oYlhCc1pWOXJhVzVrSW4wc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3dpTENKbGVIQnlaWE56YVc5dUlqb2lVMkZ0Y0d4bFZIbHdaU0o5ZlYxOUxIc2lZMjlrWlNJNmV5SjBaWGgwSWpvaWNISnZZMlZrZFhKbGN5SjlMQ0psZUhSbGJuTnBiMjRpT2x0N0luVnliQ0k2SW1oMGRIQTZMeTlvYkRjdWIzSm5MMlpvYVhJdmRYTXZZM0ZtYldWaGMzVnlaWE12VTNSeWRXTjBkWEpsUkdWbWFXNXBkR2x2Ymk5amNXWnRMWEJ2Y0hWc1lYUnBiMjVDWVhOcGN5SXNJblpoYkhWbFEyOWtaU0k2SWxCeWIyTmxaSFZ5WlNKOVhTd2ljRzl3ZFd4aGRHbHZiaUk2VzNzaVkyOWtaU0k2ZXlKamIyUnBibWNpT2x0N0luTjVjM1JsYlNJNkltaDBkSEE2THk5MFpYSnRhVzV2Ykc5bmVTNW9iRGN1YjNKbkwwTnZaR1ZUZVhOMFpXMHZiV1ZoYzNWeVpTMXdiM0IxYkdGMGFXOXVJaXdpWTI5a1pTSTZJbWx1YVhScFlXd3RjRzl3ZFd4aGRHbHZiaUo5WFgwc0ltTnlhWFJsY21saElqcDdJbXhoYm1kMVlXZGxJam9pZEdWNGRDOWpjV3d0YVdSbGJuUnBabWxsY2lJc0ltVjRjSEpsYzNOcGIyNGlPaUpRY205alpXUjFjbVVpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpRY205alpXUjFjbVZVZVhCbEluMHNJbU55YVhSbGNtbGhJanA3SW14aGJtZDFZV2RsSWpvaWRHVjRkQzlqY1d3aUxDSmxlSEJ5WlhOemFXOXVJam9pVUhKdlkyVmtkWEpsVkhsd1pTSjlmVjE5TEhzaVkyOWtaU0k2ZXlKMFpYaDBJam9pYldWa2FXTmhkR2x2YmxOMFlYUmxiV1Z1ZEhNaWZTd2laWGgwWlc1emFXOXVJanBiZXlKMWNtd2lPaUpvZEhSd09pOHZhR3czTG05eVp5OW1hR2x5TDNWekwyTnhabTFsWVhOMWNtVnpMMU4wY25WamRIVnlaVVJsWm1sdWFYUnBiMjR2WTNGbWJTMXdiM0IxYkdGMGFXOXVRbUZ6YVhNaUxDSjJZV3gxWlVOdlpHVWlPaUpOWldScFkyRjBhVzl1VTNSaGRHVnRaVzUwSW4xZExDSndiM0IxYkdGMGFXOXVJanBiZXlKamIyUmxJanA3SW1OdlpHbHVaeUk2VzNzaWMzbHpkR1Z0SWpvaWFIUjBjRG92TDNSbGNtMXBibTlzYjJkNUxtaHNOeTV2Y21jdlEyOWtaVk41YzNSbGJTOXRaV0Z6ZFhKbExYQnZjSFZzWVhScGIyNGlMQ0pqYjJSbElqb2lhVzVwZEdsaGJDMXdiM0IxYkdGMGFXOXVJbjFkZlN3aVkzSnBkR1Z5YVdFaU9uc2liR0Z1WjNWaFoyVWlPaUowWlhoMEwyTnhiQzFwWkdWdWRHbG1hV1Z5SWl3aVpYaHdjbVZ6YzJsdmJpSTZJazFsWkdsallYUnBiMjVUZEdGMFpXMWxiblFpZlgxZExDSnpkSEpoZEdsbWFXVnlJanBiZXlKamIyUmxJanA3SW5SbGVIUWlPaUpOWldScFkyRjBhVzl1Vkhsd1pTSjlMQ0pqY21sMFpYSnBZU0k2ZXlKc1lXNW5kV0ZuWlNJNkluUmxlSFF2WTNGc0lpd2laWGh3Y21WemMybHZiaUk2SWxCeWIyTmxaSFZ5WlZSNWNHVWlmWDFkZlYxOWZRPT0iLA0KICAicXVlcnktY29udGFjdC1pZCIgOiAicmVzZWFyY2hlckB0ZXN0LmRrZnouZGUiLA0KICAicXVlcnktZm9ybWF0IiA6ICJDUUxfREFUQSIsDQogICJ0ZW1wbGF0ZS1pZCIgOiAiY2NwIg0KfQ==","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"EXECUTE"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -Creating a sample [Exporter](https://github.com/samply/exporter) "status" task using CURL: +Creating a sample [Exporter](https://github.com/samply/exporter) "status" task using curl: ```bash curl -v -X POST -H "Content-Type: application/json" --data '{"body":"ew0KICAicXVlcnktZXhlY3V0aW9uLWlkIiA6ICIxOSINCn0=","failure_strategy":{"retry":{"backoff_millisecs":1000,"max_tries":5}},"from":"app1.proxy1.broker","id":"22e1ea3a-07f3-4592-a888-82f2226a44a2","metadata":{"project":"exporter","task_type":"STATUS"},"to":["app1.proxy1.broker"],"ttl":"10s","status":null,"task":null}' -H "Authorization: ApiKey app1.proxy1.broker App1Secret" http://localhost:8081/v1/tasks ``` -## Changes specific to EHDS2/ECDC - -The files ```resources/cql/EHDS2_*``` contain the CQL templates used in the EHDS2 ECDC pilot project. I have chosen to outsource all CQL to focus. EHDS2_UTIL provides utility functions used frequently. EHDS2_PATIENT and EHDS2_OBSERVATION contain the queries relating to Patient and Observation resources respectively. There are small but important differences between Patient and Observation in the way that queries work, which you will notice if you look closely at the definitions in these two files. Finally, EHDS2_IN_INITIAL_POPULATION is used to build the actual query. This file just contains a stub; the full query is built in Lens, comprising of calls to the functions and variable definitions found in EHDS2_PATIENT and EHDS2_OBSERVATION. - ## License This code is licensed under the Apache License 2.0. For details, please see [LICENSE](./LICENSE) diff --git a/src/main.rs b/src/main.rs index c7e057e..75528bb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,12 +8,12 @@ mod errors; mod graceful_shutdown; mod logger; +mod db; +mod exporter; mod intermediate_rep; +mod projects; mod task_processing; mod util; -mod projects; -mod exporter; - use base64::engine::general_purpose; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; @@ -21,6 +21,7 @@ use beam_lib::{TaskRequest, TaskResult}; use futures_util::future::BoxFuture; use futures_util::FutureExt; use laplace_rs::ObfCache; +use sqlx::PgPool; use tokio::sync::Mutex; use crate::blaze::{parse_blaze_query_payload_ast, AstQuery}; @@ -38,7 +39,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use std::{process::exit, time::Duration}; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, warn, info}; +use tracing::{debug, error, trace, warn}; // result cache type SearchQuery = String; @@ -52,7 +53,7 @@ type BeamResult = TaskResult; #[serde(tag = "lang", rename_all = "lowercase")] enum Language { Cql(CqlQuery), - Ast(AstQuery) + Ast(AstQuery), } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -70,13 +71,13 @@ impl ReportCache { pub fn new() -> Self { let mut cache = HashMap::new(); - if let Some(filename) = CONFIG.queries_to_cache_file_path.clone() { - let lines = util::read_lines(filename.clone().to_string()); + if let Some(filename) = CONFIG.queries_to_cache.clone() { + let lines = util::read_lines(&filename); match lines { Ok(ok_lines) => { for line in ok_lines { let Ok(ok_line) = line else { - warn!("A line in the file {} is not readable", filename); + warn!("A line in the file {} is not readable", filename.display()); continue; }; cache.insert((ok_line.clone(), false), ("".into(), UNIX_EPOCH)); @@ -84,7 +85,7 @@ impl ReportCache { } } Err(_) => { - error!("The file {} cannot be opened", filename); //This shouldn't stop focus from running, it's just going to go to blaze every time, but that's not too slow + error!("The file {} cannot be opened", filename.display()); //This shouldn't stop focus from running, it's just going to go to blaze every time, but that's not too slow } } } @@ -97,17 +98,15 @@ const REPORTCACHE_TTL: Duration = Duration::from_secs(86400); //24h #[tokio::main] pub async fn main() -> ExitCode { - info!("main: entered"); - if let Err(e) = logger::init_logger() { error!("Cannot initalize logger: {}", e); exit(1); }; banner::print_banner(); - let _ = CONFIG.api_key; // Initialize config + trace!("WARNING: You are running Focus in trace logging. This log level outputs unobfuscated result counts and is only intended for debugging the obfuscation. To avoid privacy risks, please check if that log level is appropriate. Consider using \"info\" or \"warn\"."); - info!("main: going to Tokio"); + let _ = CONFIG.api_key; // Initialize config tokio::select! { _ = graceful_shutdown::wait_for_signal() => { @@ -120,11 +119,22 @@ pub async fn main() -> ExitCode { } async fn main_loop() -> ExitCode { - info!("main_loop: entered"); - + let db_pool = if let Some(connection_string) = CONFIG.postgres_connection_string.clone() { + match db::get_pg_connection_pool(&connection_string, 8).await { + Err(e) => { + error!("Error connecting to database: {}", e); + return ExitCode::from(8); + } + Ok(pool) => Some(pool), + } + } else { + None + }; let endpoint_service_available: fn() -> BoxFuture<'static, bool> = match CONFIG.endpoint_type { EndpointType::Blaze => || blaze::check_availability().boxed(), EndpointType::Omop => || async { true }.boxed(), // TODO health check + EndpointType::BlazeAndSql => || blaze::check_availability().boxed(), + EndpointType::Sql => || async { true }.boxed(), }; let mut failures = 0; while !(beam::check_availability().await && endpoint_service_available().await) { @@ -139,10 +149,9 @@ async fn main_loop() -> ExitCode { tokio::time::sleep(Duration::from_secs(2)).await; warn!( "Retrying connection (attempt {}/{})", - failures, - CONFIG.retry_count + failures, CONFIG.retry_count ); - }; + } let report_cache = Arc::new(Mutex::new(ReportCache::new())); let obf_cache = Arc::new(Mutex::new(ObfCache { cache: Default::default(), @@ -150,8 +159,9 @@ async fn main_loop() -> ExitCode { task_processing::process_tasks(move |task| { let obf_cache = obf_cache.clone(); let report_cache = report_cache.clone(); - process_task(task, obf_cache, report_cache).boxed_local() - }).await; + process_task(task, obf_cache, report_cache, db_pool.clone()).boxed_local() + }) + .await; ExitCode::FAILURE } @@ -159,12 +169,13 @@ async fn process_task( task: &BeamTask, obf_cache: Arc>, report_cache: Arc>, + db_pool: Option, ) -> Result { debug!("Processing task {}", task.id); let metadata: Metadata = serde_json::from_value(task.metadata.clone()).unwrap_or(Metadata { project: "default_obfuscation".to_string(), - task_type: None + task_type: None, }); if metadata.project == "focus-healthcheck" { @@ -172,7 +183,7 @@ async fn process_task( CONFIG.beam_app_id_long.clone(), vec![task.from.clone()], task.id, - "healthy".into() + "healthy".into(), )); } if metadata.project == "exporter" { @@ -180,47 +191,128 @@ async fn process_task( return Err(FocusError::MissingExporterTaskType); }; let body = &task.body; - return Ok(run_exporter_query(task, body, task_type).await?); + return run_exporter_query(task, body, task_type).await; } - if CONFIG.endpoint_type == EndpointType::Blaze { - let mut generated_from_ast: bool = false; - let data = base64_decode(&task.body)?; - let query: CqlQuery = match serde_json::from_slice::(&data)? { - Language::Cql(cql_query) => cql_query, - Language::Ast(ast_query) => { - generated_from_ast = true; - serde_json::from_str(&cql::generate_body(parse_blaze_query_payload_ast(&ast_query.payload)?)?)? + match CONFIG.endpoint_type { + EndpointType::Blaze => { + let mut generated_from_ast: bool = false; + let data = base64_decode(&task.body)?; + let query: CqlQuery = match serde_json::from_slice::(&data)? { + Language::Cql(cql_query) => cql_query, + Language::Ast(ast_query) => { + generated_from_ast = true; + serde_json::from_str(&cql::generate_body(parse_blaze_query_payload_ast( + &ast_query.payload, + )?)?)? + } + }; + run_cql_query( + task, + &query, + obf_cache, + report_cache, + metadata.project, + generated_from_ast, + ) + .await + }, + EndpointType::BlazeAndSql => { + let mut generated_from_ast: bool = false; + let data = base64_decode(&task.body)?; + let query_maybe: Result = + serde_json::from_slice(&(data.clone())); + if let Ok(sql_query) = query_maybe { + if let Some(pool) = db_pool { + let rows = db::process_sql_task(&pool, &(sql_query.payload)).await?; + let rows_json = db::serialize_rows(rows)?; + trace!("result: {}", &rows_json); + + Ok(beam::beam_result::succeeded( + CONFIG.beam_app_id_long.clone(), + vec![task.from.clone()], + task.id, + BASE64.encode(serde_json::to_string(&rows_json)?), + )) + } else { + Err(FocusError::CannotConnectToDatabase( + "SQL task but no connection String in config".into(), + )) + } + } else { + let query: CqlQuery = match serde_json::from_slice::(&data)? { + Language::Cql(cql_query) => cql_query, + Language::Ast(ast_query) => { + generated_from_ast = true; + serde_json::from_str(&cql::generate_body(parse_blaze_query_payload_ast( + &ast_query.payload, + )?)?)? + } + }; + run_cql_query( + task, + &query, + obf_cache, + report_cache, + metadata.project, + generated_from_ast, + ) + .await } - }; - run_cql_query(task, &query, obf_cache, report_cache, metadata.project, generated_from_ast).await - - } else if CONFIG.endpoint_type == EndpointType::Omop { - let decoded = util::base64_decode(&task.body)?; - let intermediate_rep_query: intermediate_rep::IntermediateRepQuery = - serde_json::from_slice(&decoded)?; - //TODO check that the language is ast - let query_decoded = general_purpose::STANDARD - .decode(intermediate_rep_query.query) - .map_err(FocusError::DecodeError)?; - let ast: ast::Ast = - serde_json::from_slice(&query_decoded)?; - - Ok(run_intermediate_rep_query(task, ast).await?) - } else { - warn!( - "Can't run queries with endpoint type {}", - CONFIG.endpoint_type - ); - Ok(beam::beam_result::perm_failed( - CONFIG.beam_app_id_long.clone(), - vec![task.from.clone()], - task.id, - format!( - "Can't run queries with endpoint type {}", - CONFIG.endpoint_type - ), - )) + }, + EndpointType::Sql => { + let data = base64_decode(&task.body)?; + let query_maybe: Result = serde_json::from_slice(&(data)); + if let Ok(sql_query) = query_maybe { + if let Some(pool) = db_pool { + let result = db::process_sql_task(&pool, &(sql_query.payload)).await; + if let Ok(rows) = result { + let rows_json = db::serialize_rows(rows)?; + + Ok(beam::beam_result::succeeded( + CONFIG.beam_app_id_long.clone(), + vec![task.clone().from], + task.id, + BASE64.encode(serde_json::to_string(&rows_json)?), + )) + } else { + Err(FocusError::QueryResultBad( + "Query executed but result not readable".into(), + )) + } + } else { + Err(FocusError::CannotConnectToDatabase( + "SQL task but no connection String in config".into(), + )) + } + } else { + warn!( + "Wrong type of query for an SQL only store: {}, {:?}", + CONFIG.endpoint_type, data + ); + Ok(beam::beam_result::perm_failed( + CONFIG.beam_app_id_long.clone(), + vec![task.from.clone()], + task.id, + format!( + "Wrong type of query for an SQL only store: {}, {:?}", + CONFIG.endpoint_type, data + ), + )) + } + }, + EndpointType::Omop => { + let decoded = util::base64_decode(&task.body)?; + let intermediate_rep_query: intermediate_rep::IntermediateRepQuery = + serde_json::from_slice(&decoded)?; + //TODO check that the language is ast + let query_decoded = general_purpose::STANDARD + .decode(intermediate_rep_query.query) + .map_err(FocusError::DecodeError)?; + let ast: ast::Ast = serde_json::from_slice(&query_decoded)?; + + Ok(run_intermediate_rep_query(task, ast).await?) + } } } @@ -230,10 +322,8 @@ async fn run_cql_query( obf_cache: Arc>, report_cache: Arc>, project: String, - generated_from_ast: bool + generated_from_ast: bool, ) -> Result { - info!("run_cql_query: entered"); - let encoded_query = query.lib["content"][0]["data"] .as_str() @@ -244,13 +334,9 @@ async fn run_cql_query( let mut key_exists = false; - info!("run_cql_query: obfuscate"); - let obfuscate = CONFIG.obfuscate == config::Obfuscate::Yes && !CONFIG.unobfuscated.contains(&project); - info!("run_cql_query: get report"); - let report_from_cache = match report_cache .lock() .await @@ -268,20 +354,19 @@ async fn run_cql_query( None => None, }; - info!("run_cql_query: match report_from_cache"); - let cql_result_new = match report_from_cache { Some(some_report_from_cache) => some_report_from_cache.to_string(), None => { - let query = - if generated_from_ast { - query.clone() + let query = if generated_from_ast { + query.clone() } else { replace_cql_library(query.clone())? }; let cql_result = blaze::run_cql_query(&query.lib, &query.measure).await?; + trace!("MeasureReport with unobfuscated values: {}", &cql_result); + let cql_result_new: String = match obfuscate { true => obfuscate_counts_mr( &cql_result, @@ -310,8 +395,6 @@ async fn run_cql_query( } }; - info!("run_cql_query: beam_result"); - let result = beam_result(task.to_owned(), cql_result_new).unwrap_or_else(|e| { beam::beam_result::perm_failed( CONFIG.beam_app_id_long.clone(), @@ -321,8 +404,6 @@ async fn run_cql_query( ) }); - info!("run_cql_query: finished"); - Ok(result) } @@ -386,12 +467,8 @@ async fn run_exporter_query( } fn replace_cql_library(mut query: CqlQuery) -> Result { - info!("replace_cql_library: entered"); - let old_data_value = &query.lib["content"][0]["data"]; - info!("replace_cql_library: look for Field .content[0].data in old data"); - let old_data_string = old_data_value .as_str() .ok_or(FocusError::ParsingError(format!( @@ -399,18 +476,11 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { query.lib )))?; - info!("replace_cql_library: do a base 64 decode on old data"); - let decoded_cql = util::base64_decode(old_data_string)?; - info!("replace_cql_library: convert to UTF8"); - let decoded_string = str::from_utf8(&decoded_cql) .map_err(|_| FocusError::ParsingError("CQL query was invalid".into()))?; - info!("replace_cql_library: check for unexpected defines"); - info!("replace_cql_library: decoded_string {:?}", decoded_string); - match is_cql_tampered_with(decoded_string) { false => debug!("CQL not tampered with"), true => { @@ -421,23 +491,14 @@ fn replace_cql_library(mut query: CqlQuery) -> Result { } }; - info!("replace_cql_library: multiple operations"); - let replaced_cql_str = util::replace_cql(decoded_string); - - info!("replace_cql_library: replaced_cql_str {:?}", replaced_cql_str); - let replaced_cql_str_base64 = BASE64.encode(replaced_cql_str); let new_data_value = serde_json::to_value(replaced_cql_str_base64) .expect("unable to turn base64 string into json value - this should not happen"); - info!("replace_cql_library: Update the CqlQuery with the new data value"); - let a = &mut query.lib["content"][0]["data"]; *a = new_data_value; - info!("replace_cql_library: finished"); - Ok(query) } @@ -451,7 +512,6 @@ fn beam_result(task: BeamTask, measure_report: String) -> Result Date: Mon, 29 Jul 2024 11:21:37 +0200 Subject: [PATCH 24/34] Updated Rust build image --- DockerfileWithBuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DockerfileWithBuild b/DockerfileWithBuild index 30d18b2..d53a2f3 100644 --- a/DockerfileWithBuild +++ b/DockerfileWithBuild @@ -1,7 +1,7 @@ # This Dockerfile builds the Rust project and creates the final image. # Use the official Rust image as the build environment -FROM rust:1.76.0-bookworm as builder +FROM rust:1.80.0-bookworm as builder # Set the working directory inside the container WORKDIR /usr/src/your_project_name From 2ae5644f8aeb22ec0b1420fb0701f75f9e572a86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 12:07:09 +0000 Subject: [PATCH 25/34] Update thiserror requirement from 1.0.38 to 2.0.3 Updates the requirements on [thiserror](https://github.com/dtolnay/thiserror) to permit the latest version. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.38...2.0.3) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8ad8cae..c1068c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ base64 = "0.22.1" reqwest = { version = "0.12", default-features = false, features = ["json", "default-tls"] } serde = { version = "1.0.152", features = ["serde_derive"] } serde_json = "1.0" -thiserror = "1.0.38" +thiserror = "2.0.3" chrono = "0.4.31" indexmap = "2.1.0" tokio = { version = "1.25.0", default-features = false, features = ["signal", "rt-multi-thread", "macros"] } From 03f7d7598c0779ee6df114f927679560f06b44e1 Mon Sep 17 00:00:00 2001 From: Enola Knezevic <115070135+enola-dkfz@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:11:14 +0100 Subject: [PATCH 26/34] Feature/molecular (#183) * DKTK genetic variant CQL snippet * GeneticVariantCount DKTK stratifier --- resources/cql/DKTK_STRAT_GENETIC_VARIANT | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 resources/cql/DKTK_STRAT_GENETIC_VARIANT diff --git a/resources/cql/DKTK_STRAT_GENETIC_VARIANT b/resources/cql/DKTK_STRAT_GENETIC_VARIANT new file mode 100644 index 0000000..2d6b470 --- /dev/null +++ b/resources/cql/DKTK_STRAT_GENETIC_VARIANT @@ -0,0 +1,4 @@ +define GeneticVariantCount: [Observation: Code '69548-6' from loinc] + +define GeneticVariantCode: +First (from [Observation: Code '69548-6' from loinc] O return O.component.where(code.coding contains Code '48018-6' from loinc).value.coding.code.first()) From 4070594c41f34e4be5c7e8b4a2ca65b7c581e507 Mon Sep 17 00:00:00 2001 From: Torben Brenner Date: Thu, 14 Nov 2024 10:29:48 +0100 Subject: [PATCH 27/34] fix: let GeneticVariantCount return a boolean --- resources/cql/DKTK_STRAT_GENETIC_VARIANT | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/cql/DKTK_STRAT_GENETIC_VARIANT b/resources/cql/DKTK_STRAT_GENETIC_VARIANT index 2d6b470..1021262 100644 --- a/resources/cql/DKTK_STRAT_GENETIC_VARIANT +++ b/resources/cql/DKTK_STRAT_GENETIC_VARIANT @@ -1,4 +1,5 @@ -define GeneticVariantCount: [Observation: Code '69548-6' from loinc] +define GeneticVariantCount: +if InInitialPopulation then [Observation: Code '69548-6' from loinc] else {} as List define GeneticVariantCode: First (from [Observation: Code '69548-6' from loinc] O return O.component.where(code.coding contains Code '48018-6' from loinc).value.coding.code.first()) From 94f007f95dfd9e399968f27f49890bdad52bca0a Mon Sep 17 00:00:00 2001 From: Enola Knezevic Date: Thu, 14 Nov 2024 12:42:03 +0100 Subject: [PATCH 28/34] explanation of Exporter auth header --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e5c2b1..54a5828 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects QUERIES_TO_CACHE = "queries_to_cache.conf" # The path to a file containing base64 encoded queries whose results are to be cached. If not set, no results are cached PROVIDER = "name" #EUCAIM provider name PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" # Base64 encoded EUCAIM provider icon -AUTH_HEADER = "ApiKey XXXX" #Authorization header +AUTH_HEADER = "ApiKey XXXX" #Authorization header; if the endpoint type is Blaze or BlazeAndSql, this header is used for the Exporter target application, and the syntax is AUTH_HEADER = "XXXX" where "XXXX" is the API key ``` In order to use Postgres querying, a Docker image built with the feature "dktk" needs to be used and this optional variable set: From 2d479b4932262cf55793d8c659249ed0c533f4fa Mon Sep 17 00:00:00 2001 From: Enola Knezevic <115070135+enola-dkfz@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:31:46 +0100 Subject: [PATCH 29/34] Laplace version bump, focus version bump --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c1068c1..b9d6527 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "focus" -version = "0.8.0" +version = "0.9.0" edition = "2021" license = "Apache-2.0" @@ -16,7 +16,7 @@ chrono = "0.4.31" indexmap = "2.1.0" tokio = { version = "1.25.0", default-features = false, features = ["signal", "rt-multi-thread", "macros"] } beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] } -laplace_rs = {git = "https://github.com/samply/laplace-rs.git", tag = "v0.3.0" } +laplace_rs = {git = "https://github.com/samply/laplace-rs.git", tag = "v0.4.0" } uuid = "1.8.0" rand = { default-features = false, version = "0.8.5" } futures-util = { version = "0.3", default-features = false, features = ["std"] } From 28a88b16c7cde75796bd4b1150e5baca996cff97 Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Fri, 22 Nov 2024 09:19:12 +0100 Subject: [PATCH 30/34] Branch with EHDS2 CQL only --- DockerfileWithBuild | 27 --------------------------- build.sh | 1 - 2 files changed, 28 deletions(-) delete mode 100644 DockerfileWithBuild delete mode 100644 build.sh diff --git a/DockerfileWithBuild b/DockerfileWithBuild deleted file mode 100644 index d53a2f3..0000000 --- a/DockerfileWithBuild +++ /dev/null @@ -1,27 +0,0 @@ -# This Dockerfile builds the Rust project and creates the final image. - -# Use the official Rust image as the build environment -FROM rust:1.80.0-bookworm as builder - -# Set the working directory inside the container -WORKDIR /usr/src/your_project_name - -# Copy the project files into the container -COPY . . - -# Build the Rust project -RUN echo building project -RUN cargo build --release - -# Create a smaller final image -FROM gcr.io/distroless/cc-debian12 - -# Set the working directory inside the container -WORKDIR /usr/src/your_project_name - -# Copy the built binary from the builder stage -COPY --from=builder /usr/src/your_project_name/target/release/focus /usr/local/bin/ - -# Set the entry point for the final image -ENTRYPOINT ["/usr/local/bin/focus"] - diff --git a/build.sh b/build.sh deleted file mode 100644 index 738c070..0000000 --- a/build.sh +++ /dev/null @@ -1 +0,0 @@ -docker build --progress=plain -f DockerfileWithBuild -t samply/focus --no-cache . From a9279d1f3d8a88f7fcc77d089173cff2ec28a56c Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Fri, 22 Nov 2024 09:49:04 +0100 Subject: [PATCH 31/34] Reinstated .dockerignore from develop branch --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..37ad7ed --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!artifacts/ From 69e342e122d3dcfd0531a2612a16384fb2f2152f Mon Sep 17 00:00:00 2001 From: Martin Lablans <6804500+lablans@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:28:49 +0100 Subject: [PATCH 32/34] Disable daily rebuild (#189) --- .github/workflows/rust.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f7efcbb..2c47c8a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,9 +4,6 @@ on: push: workflow_dispatch: pull_request: - schedule: - # Fetch new base image updates every night at 1am - - cron: '0 1 * * *' jobs: build-with-samply: From 97cb1d6cc86744f2d7bc3e4121d6da029af1203e Mon Sep 17 00:00:00 2001 From: Enola Knezevic <115070135+enola-dkfz@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:26:38 +0100 Subject: [PATCH 33/34] separated exporter API key CLA from auth header (#191) --- README.md | 3 ++- src/config.rs | 7 +++++++ src/exporter.rs | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 54a5828..03f0914 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ PROJECTS_NO_OBFUSCATION = "exliquid;dktk_supervisors;exporter;ehds2" # Projects QUERIES_TO_CACHE = "queries_to_cache.conf" # The path to a file containing base64 encoded queries whose results are to be cached. If not set, no results are cached PROVIDER = "name" #EUCAIM provider name PROVIDER_ICON = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAAACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=" # Base64 encoded EUCAIM provider icon -AUTH_HEADER = "ApiKey XXXX" #Authorization header; if the endpoint type is Blaze or BlazeAndSql, this header is used for the Exporter target application, and the syntax is AUTH_HEADER = "XXXX" where "XXXX" is the API key +AUTH_HEADER = "[Auth Type] XXXX" #Authorization header for accessing the store; Auth Type e.g. ApiKey, Basic, ... +EXPORTER_API_KEY = "XXXX" # Value of header x-api-key for accessing the Exporter application ``` In order to use Postgres querying, a Docker image built with the feature "dktk" needs to be used and this optional variable set: diff --git a/src/config.rs b/src/config.rs index d45a408..9eca57d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -158,10 +158,15 @@ struct CliArgs { #[clap(long, env, value_parser)] provider_icon: Option, + // TODO - refactor to include multiple authorization headers for multiple stores / applications at the same time /// Authorization header #[clap(long, env, value_parser)] auth_header: Option, + /// Exporter API key + #[clap(long, env, value_parser)] + exporter_api_key: Option, + /// Postgres connection string #[cfg(feature = "query-sql")] #[clap(long, env, value_parser)] @@ -198,6 +203,7 @@ pub(crate) struct Config { pub provider: Option, pub provider_icon: Option, pub auth_header: Option, + pub exporter_api_key: Option, #[cfg(feature = "query-sql")] pub postgres_connection_string: Option, #[cfg(feature = "query-sql")] @@ -243,6 +249,7 @@ impl Config { provider: cli_args.provider, provider_icon: cli_args.provider_icon, auth_header: cli_args.auth_header, + exporter_api_key: cli_args.exporter_api_key, #[cfg(feature = "query-sql")] postgres_connection_string: cli_args.postgres_connection_string, #[cfg(feature = "query-sql")] diff --git a/src/exporter.rs b/src/exporter.rs index c3eb932..7208f3c 100644 --- a/src/exporter.rs +++ b/src/exporter.rs @@ -42,10 +42,10 @@ pub async fn post_exporter_query(body: &String, task_type: TaskType) -> Result Date: Fri, 6 Dec 2024 11:37:57 +0100 Subject: [PATCH 34/34] chore: update CHANGELOG.md (#192) --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb7e5a..a1e5cf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Samply.Focus v0.8.0 2024-11-04 +## Major changes +* EHDS2 query support + +## Minor changes +* Separated exporter API key CLA from authorization header CLA + + +# Samply.Focus v0.8.0 2024-11-04 + In this release, we are supporting 4 types of SQL queries for Exliquid and Organoids ## Major changes