Merge pull request #129 from OHDSI/loadid-into-metadata

Loadid into metadata
OHDSI · Nov 7, 2024 · 60a56ae · 60a56ae
2 parents 539836b + 36745e9
commit 60a56ae
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 10 deletions.
diff --git a/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs b/sources/Framework/org.ohdsi.cdm.framework/Desktop/DbLayer/DbSource.cs
@@ -141,5 +141,28 @@ public string GetSourceReleaseDate()
 
             return DateTime.MinValue.ToShortDateString();
         }
+
+        public string GetSourceVersionId()
+        {
+            try
+            {
+                string query = "SELECT version_id FROM " + _schemaName + "._Version";
+                using var connection = SqlConnectionHelper.OpenOdbcConnection(_connectionString);
+                using var c = new OdbcCommand(query, connection);
+                using var reader = c.ExecuteReader();
+
+                while (reader.Read())
+                {
+                    return reader.GetString("version_id");
+                }
+
+            }
+            catch (Exception)
+            {
+                return "unknown";
+            }
+
+            return "unknown";
+        }
     }
 }
diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs b/sources/Presentation/org.ohdsi.cdm.presentation.etl2/Program.cs
@@ -8,11 +8,13 @@
 using org.ohdsi.cdm.framework.common.Enums;
 using org.ohdsi.cdm.framework.common.Extensions;
 using org.ohdsi.cdm.framework.common.Helpers;
+using org.ohdsi.cdm.framework.common.Omop;
 using org.ohdsi.cdm.framework.common.Utility;
 using org.ohdsi.cdm.framework.desktop.DbLayer;
 using org.ohdsi.cdm.framework.desktop.Settings;
 using org.ohdsi.cdm.framework.desktop3.Monitor;
 using System;
+using System.Collections.Generic;
 using System.Configuration;
 using System.Data.Odbc;
 using System.Data.SqlClient;
@@ -231,6 +233,7 @@ static int Main(string[] arguments)
                 {
                     var dbSource = new DbSource(sourceConnectionString, null, sourceSchema);
                     var sourceReleaseDate = dbSource.GetSourceReleaseDate();
+                    var sourceVersionId = dbSource.GetSourceVersionId();
                     var vocabularyVersion = DbBuildingSettings.GetVocabularyVersion(vocabularyConnectionString, vocabularySchema);
 
                     Console.WriteLine("SourceReleaseDate:" + sourceReleaseDate);
@@ -241,6 +244,12 @@ static int Main(string[] arguments)
                         var reader = new CdmSourceDataReader54(DateTime.Parse(sourceReleaseDate), vocabularyVersion);
                         using var stream = reader.GetStreamCsv();
                         SaveToS3(stream, 0, "cdmCSV", "CDM_SOURCE", "gz", vendor, Settings.Current.Building.Id.Value);
+
+                        List<MetadataOMOP> metadata = [];
+                        metadata.Add(new MetadataOMOP { Id = 0, MetadataConceptId = 0, Name = "NativeLoadId", ValueAsString = sourceVersionId, MetadataDate = DateTime.Now.Date });
+                        var metadataReader = new MetadataOMOPDataReader54(metadata);
+
+                        SaveToS3(metadataReader.GetStreamCsv(), 1, "cdmCSV", "METADATA", "gz", vendor, Settings.Current.Building.Id.Value);
                     }
                     else
                     {

diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/Function.cs
@@ -201,37 +201,37 @@ private void SaveCdmSource()
             {
                 var reader = new CdmSourceDataReader54();
                 using var stream = reader.GetStreamCsv();
-                SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz");
+                SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz");
             }
             else
             {
                 var reader = new CdmSourceDataReader();
                 using var stream = reader.GetStreamCsv();
-                SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz");
+                SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz");
             }
         }
 
         private void SaveMetadata()
         {
-            var mergeMetadata = new MergeMetadata(_settings);
+            var mergeMetadata = new MergeMetadata(_settings, _versionId);
             mergeMetadata.Start();
             using var stream = mergeMetadata.GetMetadataCsvStream();
-            SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz");
+            SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz");
         }
 
         private void SaveVersion()
         {
             var reader = new VersionDataReader(_versionId);
             using var stream = reader.GetStreamCsv();
-            SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz");
+            SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz");
         }
 
         private void SaveFactRelationship()
         {
             var mergeFactRelationship = new MergeFactRelationship(_settings);
 
             using var stream = mergeFactRelationship.GetFactRelationshipCsvStream();
-            SaveToS3(stream, 0, _settings.CdmFolder, _table, "txt.gz");
+            SaveToS3(stream, 0, _settings.CdmFolder, _table, "gz");
         }
 
         private void SaveToS3(Stream memoryStream, int index, string folder, string table, string extension)

diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdamerge/MergeMetadata.cs
@@ -18,21 +18,23 @@
 
 namespace org.ohdsi.cdm.presentation.lambdamerge
 {
-    class MergeMetadata(Settings settings)
+    class MergeMetadata(Settings settings, int versionId)
     {
         private readonly Settings _settings = settings;
         private List<MetadataOMOP> _metadata;
+        private readonly int _versionId = versionId;
 
         public void Start()
         {
             _metadata = CreateMetadata();
 
-            _metadata.Add(new MetadataOMOP { MetadataConceptId = 37116952, Name = "Source data citation", ValueAsString = _settings.Vendor.Citation, MetadataDate = DateTime.Now });
-            _metadata.Add(new MetadataOMOP { MetadataConceptId = 4123211, Name = "Publication review requirements", ValueAsString = _settings.Vendor.Publication, MetadataDate = DateTime.Now });
+            _metadata.Add(new MetadataOMOP { MetadataConceptId = 0, Name = "CDMLoadId", ValueAsString = _versionId.ToString(), MetadataDate = DateTime.Now.Date });
+            _metadata.Add(new MetadataOMOP { MetadataConceptId = 37116952, Name = "Source data citation", ValueAsString = _settings.Vendor.Citation, MetadataDate = DateTime.Now.Date });
+            _metadata.Add(new MetadataOMOP { MetadataConceptId = 4123211, Name = "Publication review requirements", ValueAsString = _settings.Vendor.Publication, MetadataDate = DateTime.Now.Date });
 
             for (int i = 0; i < _metadata.Count; i++)
             {
-                _metadata[i].Id = i;
+                _metadata[i].Id = i + 1; // Starts with 1
             }
         }