Finally integrate changes from #309 (#362)

amosproj · Jul 3, 2024 · 90ac564 · 90ac564
2 parents ca44202 + d9a39b5
commit 90ac564
Show file tree

Hide file tree

Showing 42 changed files with 920 additions and 304 deletions.
diff --git a/.github/workflows/backend-CI.yml b/.github/workflows/backend-CI.yml
@@ -36,6 +36,10 @@ jobs:
         run: dotnet restore
         working-directory: backend\src\BIE.DataPipeline
 
+      - name: Build required Libraries
+        run: dotnet build "./BieMetadata.csproj" -c Release
+        working-directory: backend\lib\BieMetadata
+
       - name: Run unit tests
-        run: dotnet test
-        working-directory: backend\src\BIE.DataPipeline
+        run: dotnet test ".\src\BIE.DataPipeline"
+        working-directory: backend
diff --git a/.github/workflows/deploy_production.yml b/.github/workflows/deploy_production.yml
@@ -20,16 +20,22 @@ jobs:
         service:
           - name: frontend
             context: ./frontend
+            docker_dir: 
           - name: api-gateway
-            context: ./backend/api-gateway
+            context: ./backend
+            docker_dir: /api-gateway
           - name: api-composer
-            context: ./backend/src/BIE.Core
+            context: ./backend
+            docker_dir: /src/BIE.Core
           - name: datapipeline
-            context: ./backend/src/BIE.DataPipeline
+            context: ./backend
+            docker_dir: /src/BIE.DataPipeline
           - name: sql-database
-            context: ./backend/sql-database
+            context: ./backend
+            docker_dir: /sql-database
           - name: metadata-database
-            context: ./backend/metadata-database
+            context: ./backend
+            docker_dir: /metadata-database
 
     steps:
       - name: Checkout repository
@@ -59,7 +65,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: ${{ matrix.service.context }}
-          file: ${{ matrix.service.context }}/Dockerfile
+          file: ${{ matrix.service.context }}${{ matrix.service.docker_dir }}/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -86,6 +92,6 @@ jobs:
           scp ./docker-compose.yml ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie
           scp ./.env.production ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie/.env
       - name: Connect and Pull
-        run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
+        run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose down -v && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
       - name: Cleanup
         run: rm -rf ~/.ssh
diff --git a/.github/workflows/deploy_test.yml b/.github/workflows/deploy_test.yml
@@ -23,17 +23,22 @@ jobs:
         service:
           - name: frontend
             context: ./frontend
+            docker_dir: 
           - name: api-gateway
-            context: ./backend/api-gateway
+            context: ./backend
+            docker_dir: /api-gateway
           - name: api-composer
-            context: ./backend/src/BIE.Core
+            context: ./backend
+            docker_dir: /src/BIE.Core
           - name: datapipeline
-            context: ./backend/src/BIE.DataPipeline
+            context: ./backend
+            docker_dir: /src/BIE.DataPipeline
           - name: sql-database
             context: ./backend/sql-database
+            docker_dir: 
           - name: metadata-database
             context: ./backend/metadata-database
-
+            docker_dir: 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -54,14 +59,15 @@ jobs:
             type=ref,event=tag
             type=sha
             type=raw,value=${{ env.STAGE }}
+            latest
           labels: |
             stage=${{ env.STAGE }}
 
       - name: Build and Publish Docker Image - ${{ matrix.service.name }}
         uses: docker/build-push-action@v5
         with:
           context: ${{ matrix.service.context }}
-          file: ${{ matrix.service.context }}/Dockerfile
+          file: ${{ matrix.service.context }}${{ matrix.service.docker_dir }}/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -89,7 +95,7 @@ jobs:
           scp ./.env.test ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }}:/var/lib/bie/.env
 
       - name: Connect and Pull
-        run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
+        run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose down -v && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit"
 
       - name: Cleanup
-        run: rm -rf ~/.ssh
+        run: rm -rf ~/.ssh
diff --git a/backend/api-gateway/APIGateway.csproj b/backend/api-gateway/APIGateway.csproj
@@ -13,4 +13,10 @@
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Reference Include="BieMetadata">
+      <HintPath>..\lib\BieMetadata\bin\Release\net6.0\BieMetadata.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+
 </Project>
diff --git a/backend/api-gateway/Controllers/APIGatewayController.cs b/backend/api-gateway/Controllers/APIGatewayController.cs
@@ -10,6 +10,7 @@
 using System.ComponentModel.DataAnnotations;
 using System.Net.Mime;
 using System.Text;
+using BieMetadata;
 
 namespace BIE.Core.API.Controllers
 {

diff --git a/backend/api-gateway/Dockerfile b/backend/api-gateway/Dockerfile
@@ -5,14 +5,20 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 WORKDIR /app
 
 # Copy the .csproj file and restore any dependencies
-COPY *.csproj ./
-RUN dotnet restore
+# This is done separately from the rest of the source to (potentially) 
+# speed up later builds.
+COPY ./api-gateway/APIGateway.csproj ./api-gateway/
+RUN dotnet restore ./api-gateway/APIGateway.csproj
 
 # Copy the remaining source code
-COPY . ./
+COPY ./api-gateway ./api-gateway
+# Copy the Metadata Library
+# Copy and build metadata library
+COPY ./lib/BieMetadata ./lib/BieMetadata
+RUN dotnet build "./lib/BieMetadata/BieMetadata.csproj" -c Release
 
 # Build the application
-RUN dotnet publish -c Release -o out
+RUN dotnet publish ./api-gateway/APIGateway.csproj -c Release -o out
 
 # Use the official Microsoft .NET runtime image to run the app
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime

diff --git a/backend/api-gateway/Models/MetadataObject.cs b/backend/api-gateway/Models/MetadataObject.cs
diff --git a/backend/api-gateway/Services/MongoDBService.cs b/backend/api-gateway/Services/MongoDBService.cs
@@ -1,6 +1,7 @@
 using MongoDB.Driver;
 using Microsoft.Extensions.Configuration;
 using APIGateway.Models;
+using BieMetadata;
 
 namespace BIE.Core.API.Services
 {

diff --git a/backend/lib/BieMetadata/.nuspec b/backend/lib/BieMetadata/.nuspec
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
+    <metadata>
+        <!-- Required elements-->
+        <id>BieMetadata</id>
+        <version>1.0.0</version>
+        <description>The shared Metadata libraries used in the BIE project</description>
+        <authors>Code.ing</authors>
+
+        <!-- Optional elements -->
+        <!-- ... -->
+    </metadata>
+    <!-- Optional 'files' node -->
+</package>
diff --git a/backend/lib/BieMetadata/BieMetadata.csproj b/backend/lib/BieMetadata/BieMetadata.csproj
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net6.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="MongoDB.Driver" Version="2.26.0" />
+    </ItemGroup>
+
+</Project>
diff --git a/backend/lib/BieMetadata/BieMetadata.sln b/backend/lib/BieMetadata/BieMetadata.sln
@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BieMetadata", "BieMetadata.csproj", "{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal
diff --git a/backend/lib/BieMetadata/BoundingBox.cs b/backend/lib/BieMetadata/BoundingBox.cs
@@ -0,0 +1,10 @@
+// ReSharper disable InconsistentNaming
+namespace BieMetadata;
+
+public struct BoundingBox
+{
+    public float minX;
+    public float minY;
+    public float maxX;
+    public float maxY;
+}
diff --git a/...DataPipeline/Metadata/MetadataDbHelper.cs → backend/lib/BieMetadata/MetadataDbHelper.cs b/...DataPipeline/Metadata/MetadataDbHelper.cs → backend/lib/BieMetadata/MetadataDbHelper.cs
@@ -1,14 +1,14 @@
-using BIE.DataPipeline.Import;
-using MongoDB.Bson;
-using MongoDB.Driver;
+using MongoDB.Driver;
 
-namespace BIE.DataPipeline.Metadata;
+namespace BieMetadata;
 
 public class MetadataDbHelper
 {
     private string mMetaDataDbUrl;
 
     private IMongoDatabase mDatabase;
+
+    public bool Connected { get; private set; }
 
     public MetadataDbHelper()
     {
@@ -30,6 +30,7 @@ public bool CreateConnection()
             var connectionString = $"mongodb://datapipeline:datapipeline@{mMetaDataDbUrl}/bci-metadata";
             var client = new MongoClient(connectionString);
             mDatabase = client.GetDatabase("bci-metadata");
+            Connected = true;
             return true;
         }
         catch (Exception e)
@@ -39,45 +40,65 @@ public bool CreateConnection()
         }
     }
 
-    public MetadataObject GetMetadata(DataSourceDescription description)
+    /// <summary>
+    /// get the Metadata for a specified dataset. Returns null if dataset is not found.
+    /// </summary>
+    /// <param name="dataset"></param>
+    /// <returns></returns>
+    public MetadataObject? GetMetadata(string dataset)
     {
         // Get the collection
         var collection = mDatabase.GetCollection<MetadataObject>("datasets");
 
         // Find the dataset object with the given ID
         var metadataObject = collection
-            .Find(g => g.basicData.DatasetId == description.dataset)
+            .Find(g => g.basicData.DatasetId == dataset)
             .FirstOrDefault();
 
         return metadataObject;
     }
 
-    public void UpdateMetadata(DataSourceDescription description, int numberOfLines)
+    public bool UpdateMetadata(string dataset, string tableName, int numberOfLines, BoundingBox boundingBox)
     {
         // Load the collection
         var collection = mDatabase.GetCollection<MetadataObject>("datasets");
 
         // Find the dataset object
         var metadataObject = collection
-            .Find(g => g.basicData.DatasetId == description.dataset)
+            .Find(g => g.basicData.DatasetId == dataset)
             .FirstOrDefault();
 
+        if (metadataObject == null)
+        {
+            Console.WriteLine($"Could not find Metadata for dataset {dataset}.");
+            return false;
+        }
+
         // Load the existing table
-        var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == description.table_name);
+        var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == tableName);
         if (existingTable == null)
         {
             // Create a new table object if not present
-            var newTable = new MetadataObject.TableData() { Name = description.table_name, NumberOfLines = numberOfLines };
+            var newTable = new MetadataObject.TableData()
+            {
+                Name = tableName,
+                NumberOfLines = numberOfLines,
+                BoundingBox = boundingBox
+            };
             metadataObject.additionalData.Tables.Add(newTable);
-            collection.ReplaceOne(g => g.basicData.DatasetId == description.dataset, metadataObject);
-            return;
+            collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject);
+            return true;
         }
 
         // Table info already exists, for now just choose the larger number of lines number.
         existingTable.NumberOfLines = existingTable.NumberOfLines < numberOfLines
             ? numberOfLines
             : existingTable.NumberOfLines;
 
-        collection.ReplaceOne(g => g.basicData.DatasetId == description.dataset, metadataObject);
+        // always write the current Bounding box
+        existingTable.BoundingBox = boundingBox;
+
+        collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject);
+        return true;
     }
-}
+}