From 786de341d4efe93c7837112517fa8081ad23e0c3 Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Tue, 5 Nov 2024 12:58:26 +0300 Subject: [PATCH 1/2] added Loadid to Metadata table --- .../org.ohdsi.cdm.presentation.lambdamerge/Function.cs | 2 +- .../org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs index 3389f667..00f9ebaf 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs @@ -213,7 +213,7 @@ private void SaveCdmSource() private void SaveMetadata() { - var mergeMetadata = new MergeMetadata(_settings); + var mergeMetadata = new MergeMetadata(_settings, _versionId); mergeMetadata.Start(); using var stream = mergeMetadata.GetMetadataCsvStream(); SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs index aac79722..49fc6810 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs @@ -18,15 +18,17 @@ namespace org.ohdsi.cdm.presentation.lambdamerge { - class MergeMetadata(Settings settings) + class MergeMetadata(Settings settings, int versionId) { private readonly Settings _settings = settings; private List _metadata; + private readonly int _versionId = versionId; public void Start() { _metadata = CreateMetadata(); + _metadata.Add(new MetadataOMOP { MetadataConceptId = 0, Name = "Loadid", ValueAsString = _versionId.ToString(), MetadataDate = DateTime.Now }); _metadata.Add(new MetadataOMOP { MetadataConceptId = 37116952, Name = "Source data citation", ValueAsString = _settings.Vendor.Citation, MetadataDate = DateTime.Now }); _metadata.Add(new MetadataOMOP { MetadataConceptId = 4123211, Name = "Publication review requirements", ValueAsString = _settings.Vendor.Publication, MetadataDate = DateTime.Now }); From 36745e998692aecf128b94dc9e48b25a1fae97a2 Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Wed, 6 Nov 2024 16:02:27 +0300 Subject: [PATCH 2/2] added native load_id --- .../Desktop/DbLayer/DbSource.cs | 23 +++++++++++++++++++ .../Program.cs | 9 ++++++++ .../Function.cs | 10 ++++---- .../MergeMetadata.cs | 8 +++---- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs b/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs index 30cc64a2..29d08eac 100644 --- a/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs +++ b/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs @@ -141,5 +141,28 @@ public string GetSourceReleaseDate() return DateTime.MinValue.ToShortDateString(); } + + public string GetSourceVersionId() + { + try + { + string query = "SELECT version_id FROM " + _schemaName + "._Version"; + using var connection = SqlConnectionHelper.OpenOdbcConnection(_connectionString); + using var c = new OdbcCommand(query, connection); + using var reader = c.ExecuteReader(); + + while (reader.Read()) + { + return reader.GetString("version_id"); + } + + } + catch (Exception) + { + return "unknown"; + } + + return "unknown"; + } } } \ No newline at end of file diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs index 8160bc75..b4a3a08a 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs @@ -8,11 +8,13 @@ using org.ohdsi.cdm.framework.common.Enums; using org.ohdsi.cdm.framework.common.Extensions; using org.ohdsi.cdm.framework.common.Helpers; +using org.ohdsi.cdm.framework.common.Omop; using org.ohdsi.cdm.framework.common.Utility; using org.ohdsi.cdm.framework.desktop.DbLayer; using org.ohdsi.cdm.framework.desktop.Settings; using org.ohdsi.cdm.framework.desktop3.Monitor; using System; +using System.Collections.Generic; using System.Configuration; using System.Data.Odbc; using System.Data.SqlClient; @@ -231,6 +233,7 @@ static int Main(string[] arguments) { var dbSource = new DbSource(sourceConnectionString, null, sourceSchema); var sourceReleaseDate = dbSource.GetSourceReleaseDate(); + var sourceVersionId = dbSource.GetSourceVersionId(); var vocabularyVersion = DbBuildingSettings.GetVocabularyVersion(vocabularyConnectionString, vocabularySchema); Console.WriteLine("SourceReleaseDate:" + sourceReleaseDate); @@ -241,6 +244,12 @@ static int Main(string[] arguments) var reader = new CdmSourceDataReader54(DateTime.Parse(sourceReleaseDate), vocabularyVersion); using var stream = reader.GetStreamCsv(); SaveToS3(stream, 0, "cdmCSV", "CDM_SOURCE", "gz", vendor, Settings.Current.Building.Id.Value); + + List metadata = []; + metadata.Add(new MetadataOMOP { Id = 0, MetadataConceptId = 0, Name = "NativeLoadId", ValueAsString = sourceVersionId, MetadataDate = DateTime.Now.Date }); + var metadataReader = new MetadataOMOPDataReader54(metadata); + + SaveToS3(metadataReader.GetStreamCsv(), 1, "cdmCSV", "METADATA", "gz", vendor, Settings.Current.Building.Id.Value); } else { diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs index 00f9ebaf..7e634985 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs @@ -201,13 +201,13 @@ private void SaveCdmSource() { var reader = new CdmSourceDataReader54(); using var stream = reader.GetStreamCsv(); - SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); + SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz"); } else { var reader = new CdmSourceDataReader(); using var stream = reader.GetStreamCsv(); - SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); + SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz"); } } @@ -216,14 +216,14 @@ private void SaveMetadata() var mergeMetadata = new MergeMetadata(_settings, _versionId); mergeMetadata.Start(); using var stream = mergeMetadata.GetMetadataCsvStream(); - SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); + SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz"); } private void SaveVersion() { var reader = new VersionDataReader(_versionId); using var stream = reader.GetStreamCsv(); - SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); + SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz"); } private void SaveFactRelationship() @@ -231,7 +231,7 @@ private void SaveFactRelationship() var mergeFactRelationship = new MergeFactRelationship(_settings); using var stream = mergeFactRelationship.GetFactRelationshipCsvStream(); - SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz"); + SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz"); } private void SaveToS3(Stream memoryStream, int index, string folder, string table, string extension) diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs index 49fc6810..f2262bf2 100644 --- a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs +++ b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs @@ -28,13 +28,13 @@ public void Start() { _metadata = CreateMetadata(); - _metadata.Add(new MetadataOMOP { MetadataConceptId = 0, Name = "Loadid", ValueAsString = _versionId.ToString(), MetadataDate = DateTime.Now }); - _metadata.Add(new MetadataOMOP { MetadataConceptId = 37116952, Name = "Source data citation", ValueAsString = _settings.Vendor.Citation, MetadataDate = DateTime.Now }); - _metadata.Add(new MetadataOMOP { MetadataConceptId = 4123211, Name = "Publication review requirements", ValueAsString = _settings.Vendor.Publication, MetadataDate = DateTime.Now }); + _metadata.Add(new MetadataOMOP { MetadataConceptId = 0, Name = "CDMLoadId", ValueAsString = _versionId.ToString(), MetadataDate = DateTime.Now.Date }); + _metadata.Add(new MetadataOMOP { MetadataConceptId = 37116952, Name = "Source data citation", ValueAsString = _settings.Vendor.Citation, MetadataDate = DateTime.Now.Date }); + _metadata.Add(new MetadataOMOP { MetadataConceptId = 4123211, Name = "Publication review requirements", ValueAsString = _settings.Vendor.Publication, MetadataDate = DateTime.Now.Date }); for (int i = 0; i < _metadata.Count; i++) { - _metadata[i].Id = i; + _metadata[i].Id = i + 1; // Starts with 1 } }