diff --git a/.github/workflows/backend-CI.yml b/.github/workflows/backend-CI.yml index 057824ed..1dd39de2 100644 --- a/.github/workflows/backend-CI.yml +++ b/.github/workflows/backend-CI.yml @@ -36,6 +36,10 @@ jobs: run: dotnet restore working-directory: backend\src\BIE.DataPipeline + - name: Build required Libraries + run: dotnet build "./BieMetadata.csproj" -c Release + working-directory: backend\lib\BieMetadata + - name: Run unit tests - run: dotnet test - working-directory: backend\src\BIE.DataPipeline + run: dotnet test ".\src\BIE.DataPipeline" + working-directory: backend diff --git a/.github/workflows/deploy_production.yml b/.github/workflows/deploy_production.yml index ff8c9498..78c7bd54 100644 --- a/.github/workflows/deploy_production.yml +++ b/.github/workflows/deploy_production.yml @@ -20,16 +20,22 @@ jobs: service: - name: frontend context: ./frontend + docker_dir: - name: api-gateway - context: ./backend/api-gateway + context: ./backend + docker_dir: /api-gateway - name: api-composer - context: ./backend/src/BIE.Core + context: ./backend + docker_dir: /src/BIE.Core - name: datapipeline - context: ./backend/src/BIE.DataPipeline + context: ./backend + docker_dir: /src/BIE.DataPipeline - name: sql-database - context: ./backend/sql-database + context: ./backend + docker_dir: /sql-database - name: metadata-database - context: ./backend/metadata-database + context: ./backend + docker_dir: /metadata-database steps: - name: Checkout repository @@ -59,7 +65,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} - file: ${{ matrix.service.context }}/Dockerfile + file: ${{ matrix.service.context }}${{ matrix.service.docker_dir }}/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -86,6 +92,6 @@ jobs: scp ./docker-compose.yml ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie scp ./.env.production ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}}:/var/lib/bie/.env - name: Connect and Pull - run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit" + run: ssh ${{secrets.PROD_ENV_SSH_USER}}@${{secrets.PROD_ENV_SSH_HOST}} "cd /var/lib/bie && docker compose down -v && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit" - name: Cleanup run: rm -rf ~/.ssh diff --git a/.github/workflows/deploy_test.yml b/.github/workflows/deploy_test.yml index ecac78ec..42af928e 100644 --- a/.github/workflows/deploy_test.yml +++ b/.github/workflows/deploy_test.yml @@ -23,17 +23,22 @@ jobs: service: - name: frontend context: ./frontend + docker_dir: - name: api-gateway - context: ./backend/api-gateway + context: ./backend + docker_dir: /api-gateway - name: api-composer - context: ./backend/src/BIE.Core + context: ./backend + docker_dir: /src/BIE.Core - name: datapipeline - context: ./backend/src/BIE.DataPipeline + context: ./backend + docker_dir: /src/BIE.DataPipeline - name: sql-database context: ./backend/sql-database + docker_dir: - name: metadata-database context: ./backend/metadata-database - + docker_dir: steps: - name: Checkout repository uses: actions/checkout@v4 @@ -54,6 +59,7 @@ jobs: type=ref,event=tag type=sha type=raw,value=${{ env.STAGE }} + latest labels: | stage=${{ env.STAGE }} @@ -61,7 +67,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} - file: ${{ matrix.service.context }}/Dockerfile + file: ${{ matrix.service.context }}${{ matrix.service.docker_dir }}/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} @@ -89,7 +95,7 @@ jobs: scp ./.env.test ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }}:/var/lib/bie/.env - name: Connect and Pull - run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit" + run: ssh ${{ secrets.TEST_ENV_SSH_USER }}@${{ secrets.TEST_ENV_SSH_HOST }} "cd /var/lib/bie && docker compose down -v && docker compose pull && docker compose up -d --remove-orphans --force-recreate && exit" - name: Cleanup - run: rm -rf ~/.ssh + run: rm -rf ~/.ssh \ No newline at end of file diff --git a/backend/api-gateway/APIGateway.csproj b/backend/api-gateway/APIGateway.csproj index 529242cf..d05161a4 100644 --- a/backend/api-gateway/APIGateway.csproj +++ b/backend/api-gateway/APIGateway.csproj @@ -13,4 +13,10 @@ + + + ..\lib\BieMetadata\bin\Release\net6.0\BieMetadata.dll + + + diff --git a/backend/api-gateway/Controllers/APIGatewayController.cs b/backend/api-gateway/Controllers/APIGatewayController.cs index 6da3c94e..92d679a8 100644 --- a/backend/api-gateway/Controllers/APIGatewayController.cs +++ b/backend/api-gateway/Controllers/APIGatewayController.cs @@ -10,6 +10,7 @@ using System.ComponentModel.DataAnnotations; using System.Net.Mime; using System.Text; +using BieMetadata; namespace BIE.Core.API.Controllers { diff --git a/backend/api-gateway/Dockerfile b/backend/api-gateway/Dockerfile index fe275f81..0918351f 100644 --- a/backend/api-gateway/Dockerfile +++ b/backend/api-gateway/Dockerfile @@ -5,14 +5,20 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build WORKDIR /app # Copy the .csproj file and restore any dependencies -COPY *.csproj ./ -RUN dotnet restore +# This is done separately from the rest of the source to (potentially) +# speed up later builds. +COPY ./api-gateway/APIGateway.csproj ./api-gateway/ +RUN dotnet restore ./api-gateway/APIGateway.csproj # Copy the remaining source code -COPY . ./ +COPY ./api-gateway ./api-gateway +# Copy the Metadata Library +# Copy and build metadata library +COPY ./lib/BieMetadata ./lib/BieMetadata +RUN dotnet build "./lib/BieMetadata/BieMetadata.csproj" -c Release # Build the application -RUN dotnet publish -c Release -o out +RUN dotnet publish ./api-gateway/APIGateway.csproj -c Release -o out # Use the official Microsoft .NET runtime image to run the app FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS runtime diff --git a/backend/api-gateway/Models/MetadataObject.cs b/backend/api-gateway/Models/MetadataObject.cs deleted file mode 100644 index e2c40876..00000000 --- a/backend/api-gateway/Models/MetadataObject.cs +++ /dev/null @@ -1,55 +0,0 @@ -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; - -namespace APIGateway.Models; - -public class MetadataObject -{ - [BsonRepresentation(BsonType.ObjectId)] - public string _id { get; set; } = string.Empty; - - [BsonElement("basicData")] - public BasicData basicData { get; set; } = new BasicData(); - - [BsonElement("additionalData")] - public AdditionalData additionalData { get; set; } = new AdditionalData(); - - // The general and most important data about a dataset. - public class BasicData - { - public string DatasetId { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string ShortDescription { get; set; } = string.Empty; - public string Icon { get; set; } = string.Empty; - } - - // The additional data for each of the datasets, queried at a request. - public class AdditionalData - { - public string Icon { get; set; } = string.Empty; - public string Type { get; set; } = string.Empty; - - public string LongDescription { get; set; } = string.Empty; - - // Zoom level is higher the closer you look at something. If current zoom level is below this, it shouldn't display any value. - public int MinZoomLevel { get; set; } = 0; - - // The zoom threshold where areas start to turn into markers - public int MarkersThreshold { get; set; } = 0; - - // The display property is the property that should be shown in a popup. - public string DisplayProperty { get; set; } = string.Empty; - - // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. - public List Tables { get; set; } = new List(); - } - - // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. - public class TableData - { - // The name of the .yaml file - public string Name { get; set; } = string.Empty; - // The number of lines of data in that file. - public int NumberOfLines { get; set; } = 0; - } -} \ No newline at end of file diff --git a/backend/api-gateway/Services/MongoDBService.cs b/backend/api-gateway/Services/MongoDBService.cs index b1e30684..a50e622a 100644 --- a/backend/api-gateway/Services/MongoDBService.cs +++ b/backend/api-gateway/Services/MongoDBService.cs @@ -1,6 +1,7 @@ using MongoDB.Driver; using Microsoft.Extensions.Configuration; using APIGateway.Models; +using BieMetadata; namespace BIE.Core.API.Services { diff --git a/backend/lib/BieMetadata/.nuspec b/backend/lib/BieMetadata/.nuspec new file mode 100644 index 00000000..806ef8f3 --- /dev/null +++ b/backend/lib/BieMetadata/.nuspec @@ -0,0 +1,14 @@ + + + + + BieMetadata + 1.0.0 + The shared Metadata libraries used in the BIE project + Code.ing + + + + + + \ No newline at end of file diff --git a/backend/lib/BieMetadata/BieMetadata.csproj b/backend/lib/BieMetadata/BieMetadata.csproj new file mode 100644 index 00000000..8c231b00 --- /dev/null +++ b/backend/lib/BieMetadata/BieMetadata.csproj @@ -0,0 +1,13 @@ + + + + net6.0 + enable + enable + + + + + + + diff --git a/backend/lib/BieMetadata/BieMetadata.sln b/backend/lib/BieMetadata/BieMetadata.sln new file mode 100644 index 00000000..87c08b33 --- /dev/null +++ b/backend/lib/BieMetadata/BieMetadata.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BieMetadata", "BieMetadata.csproj", "{C70EFDC8-4D31-4BBF-828C-80E47C4064DA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C70EFDC8-4D31-4BBF-828C-80E47C4064DA}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/backend/lib/BieMetadata/BoundingBox.cs b/backend/lib/BieMetadata/BoundingBox.cs new file mode 100644 index 00000000..3c468280 --- /dev/null +++ b/backend/lib/BieMetadata/BoundingBox.cs @@ -0,0 +1,10 @@ +// ReSharper disable InconsistentNaming +namespace BieMetadata; + +public struct BoundingBox +{ + public float minX; + public float minY; + public float maxX; + public float maxY; +} diff --git a/backend/src/BIE.DataPipeline/Metadata/MetadataDbHelper.cs b/backend/lib/BieMetadata/MetadataDbHelper.cs similarity index 62% rename from backend/src/BIE.DataPipeline/Metadata/MetadataDbHelper.cs rename to backend/lib/BieMetadata/MetadataDbHelper.cs index 84484c42..75cfb856 100644 --- a/backend/src/BIE.DataPipeline/Metadata/MetadataDbHelper.cs +++ b/backend/lib/BieMetadata/MetadataDbHelper.cs @@ -1,14 +1,14 @@ -using BIE.DataPipeline.Import; -using MongoDB.Bson; -using MongoDB.Driver; +using MongoDB.Driver; -namespace BIE.DataPipeline.Metadata; +namespace BieMetadata; public class MetadataDbHelper { private string mMetaDataDbUrl; private IMongoDatabase mDatabase; + + public bool Connected { get; private set; } public MetadataDbHelper() { @@ -30,6 +30,7 @@ public bool CreateConnection() var connectionString = $"mongodb://datapipeline:datapipeline@{mMetaDataDbUrl}/bci-metadata"; var client = new MongoClient(connectionString); mDatabase = client.GetDatabase("bci-metadata"); + Connected = true; return true; } catch (Exception e) @@ -39,38 +40,54 @@ public bool CreateConnection() } } - public MetadataObject GetMetadata(DataSourceDescription description) + /// + /// get the Metadata for a specified dataset. Returns null if dataset is not found. + /// + /// + /// + public MetadataObject? GetMetadata(string dataset) { // Get the collection var collection = mDatabase.GetCollection("datasets"); // Find the dataset object with the given ID var metadataObject = collection - .Find(g => g.basicData.DatasetId == description.dataset) + .Find(g => g.basicData.DatasetId == dataset) .FirstOrDefault(); return metadataObject; } - public void UpdateMetadata(DataSourceDescription description, int numberOfLines) + public bool UpdateMetadata(string dataset, string tableName, int numberOfLines, BoundingBox boundingBox) { // Load the collection var collection = mDatabase.GetCollection("datasets"); // Find the dataset object var metadataObject = collection - .Find(g => g.basicData.DatasetId == description.dataset) + .Find(g => g.basicData.DatasetId == dataset) .FirstOrDefault(); + if (metadataObject == null) + { + Console.WriteLine($"Could not find Metadata for dataset {dataset}."); + return false; + } + // Load the existing table - var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == description.table_name); + var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == tableName); if (existingTable == null) { // Create a new table object if not present - var newTable = new MetadataObject.TableData() { Name = description.table_name, NumberOfLines = numberOfLines }; + var newTable = new MetadataObject.TableData() + { + Name = tableName, + NumberOfLines = numberOfLines, + BoundingBox = boundingBox + }; metadataObject.additionalData.Tables.Add(newTable); - collection.ReplaceOne(g => g.basicData.DatasetId == description.dataset, metadataObject); - return; + collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject); + return true; } // Table info already exists, for now just choose the larger number of lines number. @@ -78,6 +95,10 @@ public void UpdateMetadata(DataSourceDescription description, int numberOfLines) ? numberOfLines : existingTable.NumberOfLines; - collection.ReplaceOne(g => g.basicData.DatasetId == description.dataset, metadataObject); + // always write the current Bounding box + existingTable.BoundingBox = boundingBox; + + collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject); + return true; } -} \ No newline at end of file +} diff --git a/backend/lib/BieMetadata/MetadataObject.cs b/backend/lib/BieMetadata/MetadataObject.cs new file mode 100644 index 00000000..7dc23a84 --- /dev/null +++ b/backend/lib/BieMetadata/MetadataObject.cs @@ -0,0 +1,92 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +// ReSharper disable InconsistentNaming + +namespace BieMetadata; + +public class MetadataObject +{ + [BsonRepresentation(BsonType.ObjectId)] + public string _id { get; set; } = string.Empty; + + [BsonElement("basicData")] + public BasicData basicData { get; set; } = new BasicData(); + + [BsonElement("additionalData")] + public AdditionalData additionalData { get; set; } = new AdditionalData(); + + /// + /// The general and most important data about a dataset. + /// + public class BasicData + { + /// + /// The Id of the dataset + /// + public string DatasetId { get; set; } = string.Empty; + + /// + /// The displayname of the dataset + /// + public string Name { get; set; } = string.Empty; + + /// + /// a short description of the dataset + /// + public string ShortDescription { get; set; } = string.Empty; + + /// + /// the icon used to display dataset + /// + public string Icon { get; set; } = string.Empty; + } + + /// + /// The additional data for each of the datasets, queried at a request. + /// + public class AdditionalData + { + public string Icon { get; set; } = string.Empty; + public string Type { get; set; } = string.Empty; + + /// + /// the Datatype of the dataset. Used to determine how to handle Dataset + /// + public string DataType { get; set; } = string.Empty; + + public string LongDescription { get; set; } = string.Empty; + + /// + /// Zoom level is higher the closer you look at something. If current zoom level is below this, it shouldn't display any value. + /// + public int MinZoomLevel { get; set; } = 0; + + /// + /// The zoom threshold where areas start to turn into markers + /// + public int MarkersThreshold { get; set; } = 0; + + /// + /// The display property is the property that should be shown in a popup. + /// + public string DisplayProperty { get; set; } = string.Empty; + + /// + /// Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. + /// + public List Tables { get; set; } = new List(); + } + + /// + /// Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. + /// + public class TableData + { + // The name of the .yaml file + public string Name { get; set; } = string.Empty; + // The number of lines of data in that file. + public int NumberOfLines { get; set; } = 0; + + public BoundingBox? BoundingBox { get; set; } + } +} diff --git a/backend/metadata-database/init-db.js b/backend/metadata-database/init-db.js index 1909cf58..33c51566 100644 --- a/backend/metadata-database/init-db.js +++ b/backend/metadata-database/init-db.js @@ -24,6 +24,7 @@ const datasets = [ }, additionalData: { Type: "none", + DataType: "none", LongDescription: `An empty, default map of Germany, with no data loaded. Useful for exploring the map.`, MinZoomLevel: -1, MarkersThreshold: -1, @@ -41,6 +42,7 @@ const datasets = [ additionalData: { Icon: '', Type: "markers", + DataType: "CSV", LongDescription: `A map of EV charging stations displays the locations of electric vehicle charging points located in Germany, helping drivers plan routes and manage charging needs. It is essential for supporting the adoption and convenience of electric vehicles.`, MinZoomLevel: 11, MarkersThreshold: -1, @@ -58,6 +60,7 @@ const datasets = [ additionalData: { Icon: '', Type: "areas", + DataType: "SHAPE", LongDescription: `House footprints refer to the outline or ground area covered by a house, typically measured from the exterior walls of the structure. This footprint includes all parts of the house that are in contact with the ground, and is important for planning and zoning purposes, calculating property taxes, and designing land use.`, MinZoomLevel: 11, MarkersThreshold: 17, @@ -75,6 +78,7 @@ const datasets = [ additionalData: { Icon: '', Type: "areas", + DataType: "SHAPE", LongDescription: `The Actual Use map describes the use of the earth's surface in four main groups (settlement, traffic, vegetation and water bodies). The division of these main groups into almost 140 different types of use, such as residential areas, road traffic, agriculture or flowing water, enables detailed evaluations and analyses of the use of the earth's surface.`, MinZoomLevel: 11, MarkersThreshold: 17, diff --git a/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs b/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs new file mode 100644 index 00000000..f1a362ea --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Accord.Math; +using BIE.Core.API.Controllers; +using BieMetadata; + +namespace BIE.Core.API; + +public static class ApiHelper +{ + private static CultureInfo sCultureInfo = new CultureInfo("en-US"); + + /// + /// Get the Bounding Box from the query parameters. + /// + /// + /// + public static BoundingBox GetBoundingBoxFromParameters(DatasetController.QueryParameters parameters) + { + var boundingBox = new BoundingBox() + { + minX = parameters.BottomLong, + minY = parameters.BottomLat, + maxX = parameters.TopLong, + maxY = parameters.TopLat + }; + + return boundingBox; + } + + /// + /// get the WKT (well known text) polygon from a bounding box. + /// + /// + /// + public static string GetPolygonFromBoundingBox(BoundingBox boundingBox) + { + // Lat = Y Long = X + var culture = new CultureInfo("en-US"); + var bottomLong = boundingBox.minX.ToString(culture); + var bottomLat = boundingBox.minY.ToString(culture); + var topLong = boundingBox.maxX.ToString(culture); + var topLat = boundingBox.maxY.ToString(culture); + + // Create polygon WKT from bounding box + return + $"POLYGON(({bottomLong} {bottomLat}," + + $" {topLong} {bottomLat}," + + $" {topLong} {topLat}," + + $" {bottomLong} {topLat}," + + $" {bottomLong} {bottomLat}))"; + } + + /// + /// Get the polygon of the bounding box given in the queryparameters + /// + /// + /// + public static string GetPolygonFromQueryParameters(DatasetController.QueryParameters parameters) + { + var boundingBox = GetBoundingBoxFromParameters(parameters); + + // Create polygon WKT from bounding box + return GetPolygonFromBoundingBox(boundingBox); + } + + /// + /// returns true if two boxes are intersecting. + /// + /// + /// + /// + public static bool BoxIntersection(BoundingBox box1, BoundingBox box2) + { + var left1 = box1.minX; + var bottom1 = box1.minY; + var right1 = box1.maxX; + var top1 = box1.maxY; + + var left2 = box2.minX; + var bottom2 = box2.minY; + var right2 = box2.maxX; + var top2 = box2.maxY; + + Console.WriteLine($"left1: {left1}, left2: {left2}"); + + return !(right1 < left2 || right2 < left1 || top1 < bottom2 || top2 < bottom1); + } + + /// + /// Gets the Query to filter a table via polygon Intersection. Presents the FROM part of a Query. + /// + /// the name of the table to filter + /// the polygon string + /// + public static string FromTableIntersectsPolygon(string tableName, string polygon) + { + return $@" +FROM dbo.{tableName} +WHERE Location.STIntersects(geometry::STGeomFromText('{polygon}', 4326)) = 1;"; + } + + /// + /// Gets the float coordinates from the polygon string + /// + /// + /// + public static float[][][] GetCoordinatesFromPolygon(string polygon) + { + // example: + // POLYGON ((49.496927347229494 11.060226859896797, ..., 49.496927347229494 11.060226859896797)) + var substring = polygon.Substring(10, polygon.Length - 12); + var coordinatePairs = substring.Replace("(", "").Replace(")", "").Split(","); + var floatList = coordinatePairs.Select(pair => pair.Trim().Split(" ").Select(StringToFloat).ToArray()); + return new []{floatList.ToArray()}; + } + + /// + /// Casts a string to a Float using the en-US culture. + /// + /// + /// + public static float StringToFloat(string s) + { + return float.Parse(s, sCultureInfo); + } +} \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj index fee7a816..37fa139c 100644 --- a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj +++ b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj @@ -15,6 +15,7 @@ + @@ -26,4 +27,10 @@ + + + ..\..\..\lib\BieMetadata\bin\Release\net6.0\BieMetadata.dll + + + diff --git a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml index e1ce6d45..6ea0d60a 100644 --- a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml +++ b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml @@ -4,6 +4,57 @@ BIE.Core.API + + + Get the Bounding Box from the query parameters. + + + + + + + get the WKT (well known text) polygon from a bounding box. + + + + + + + Get the polygon of the bounding box given in the queryparameters + + + + + + + returns true if two boxes are intersecting. + + + + + + + + Gets the Query to filter a table via polygon Intersection. Presents the FROM part of a Query. + + the name of the table to filter + the polygon string + + + + + Gets the float coordinates from the polygon string + + + + + + + Casts a string to a Float using the en-US culture. + + + + Batch controller @@ -13,9 +64,7 @@ Get viewport data. so Data for a specific rectangle returned as featurecollection - Nice - @@ -84,6 +133,31 @@ + + + Handler for the CSV Datatype. Assumes a single table. Uses a Geometry row named + + + + + Handler for the CSV Datatype. Assumes a single table. Uses a Geometry row named + + + + + + Handler for the Shape Dataset Type. Scans multiple Tables and combines results. + + + + + + Get the data inside a bounding box + + + + + Global constatns diff --git a/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs b/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs index 29f5368b..0fc1e955 100644 --- a/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs +++ b/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs @@ -10,9 +10,10 @@ using Newtonsoft.Json; using System.Data; using System.ComponentModel.DataAnnotations; - using System.Globalization; using System.Text; +using BIE.Core.API.DatasetHandlers; +using BieMetadata; using Microsoft.Extensions.Logging; namespace BIE.Core.API.Controllers @@ -24,11 +25,36 @@ namespace BIE.Core.API.Controllers [ApiController] public class DatasetController : Controller { + // ReSharper disable once InconsistentNaming private readonly ILogger _logger; + private MetadataDbHelper mMetadataDbHelper; + + private MetadataDbHelper MetadataDbHelper + { + get + { + if (mMetadataDbHelper != null) + { + return mMetadataDbHelper; + } + + mMetadataDbHelper = new MetadataDbHelper(); + if (mMetadataDbHelper.CreateConnection()) + { + return mMetadataDbHelper; + } + + Console.WriteLine("could not establish Metadata-database Connection"); + throw new Exception("no metadata DB reachable!"); + } + } + public DatasetController(ILogger logger) { _logger = logger; + + Console.WriteLine("setting up Dataset Controller"); } /// @@ -41,24 +67,40 @@ public DatasetController(ILogger logger) [ProducesResponseType(500)] public ActionResult GetDatasetViewportData([FromQuery] QueryParameters parameters) { - _logger.LogInformation("Received request for GetDatasetViewportData with parameters: {parameters}", parameters); + _logger.LogInformation($"Received request for GetDatasetViewportData with parameters: {parameters}"); + Console.WriteLine("Get Viewport Data"); if (!ModelState.IsValid) { - _logger.LogWarning("Invalid model state: {ModelState}", ModelState); + _logger.LogWarning($"Invalid model state: {ModelState}"); return BadRequest(ModelState); } - switch (parameters.Id) + // check if the dataset is present in the Metadata + var metadata = MetadataDbHelper.GetMetadata(parameters.Id); + if (metadata == null) + { + return StatusCode(400, $"Unsupported dataset: {parameters.Id}"); + } + + // select the correct Handler + IDatasetHandler handler; + switch (metadata.additionalData.DataType) { - case "house_footprints": - return GetHouseFootprintsData(parameters); - case "EV_charging_stations": - return GetChargingStations(parameters); + case "SHAPE": + handler = new ShapeDatasetHandler(metadata); + break; + case "CSV": + // TODO + handler = new CsvDatasetHandler(metadata); + break; default: - _logger.LogWarning("Unsupported dataset ID: {Id}", parameters.Id); - return StatusCode(400, $"Unsupported dataset ID of {parameters.Id}"); + Console.WriteLine($"Datatype {metadata.additionalData.DataType} is not known."); + return StatusCode(400, $"Unsupported dataset type: {metadata.additionalData.DataType}"); } + + var boundingBox = ApiHelper.GetBoundingBoxFromParameters(parameters); + return Ok(handler.GetDataInsideArea(boundingBox)); } private ActionResult GetChargingStations(QueryParameters parameters) @@ -66,13 +108,9 @@ private ActionResult GetChargingStations(QueryParameters parameters) _logger.LogInformation("Fetching charging stations with parameters: {parameters}", parameters); try { - var bottomLat = parameters.BottomLat; - var bottomLong = parameters.BottomLong; - var topLat = parameters.TopLat; - var topLong = parameters.TopLong; - // Create polygon WKT from bounding box - var polygonWkt = $"POLYGON(({bottomLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}, {topLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}, {topLong.ToString(new CultureInfo("en-US"))} {topLat.ToString(new CultureInfo("en-US"))}, {bottomLong.ToString(new CultureInfo("en-US"))} {topLat.ToString(new CultureInfo("en-US"))}, {bottomLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}))"; + var polygonWkt = ApiHelper.GetPolygonFromQueryParameters(parameters); + // SQL Query to find intersecting points var sqlQuery = $@" @@ -126,21 +164,15 @@ FROM dbo.EV_charging_stations } - private ActionResult GetHouseFootprintsData([FromQuery] QueryParameters parameters) { _logger.LogInformation("Fetching house footprints with parameters: {parameters}", parameters); try { - var bottomLong = parameters.BottomLat; - var bottomLat = parameters.BottomLong; - var topLong = parameters.TopLat; - var topLat = parameters.TopLong; - DbHelper.CreateDbConnection(); // Create polygon WKT from bounding box - var polygonWkt = $"POLYGON(({bottomLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}, {topLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}, {topLong.ToString(new CultureInfo("en-US"))} {topLat.ToString(new CultureInfo("en-US"))}, {bottomLong.ToString(new CultureInfo("en-US"))} {topLat.ToString(new CultureInfo("en-US"))}, {bottomLong.ToString(new CultureInfo("en-US"))} {bottomLat.ToString(new CultureInfo("en-US"))}))"; + var polygonWkt = ApiHelper.GetPolygonFromQueryParameters(parameters); // SQL Query to find intersecting points var sqlQuery = $@" @@ -365,18 +397,18 @@ WHERE Location.STIntersects(geography::STGeomFromText('POLYGON(( var location = new List(); try { - location = row["Location"] - .Replace("POLYGON ((", "").Replace("))", "") - .Split(',') - .Select(coord => coord.Trim().Split(' ') - .Select(double.Parse).ToArray()).ToList(); - + location = row["Location"] + .Replace("POLYGON ((", "").Replace("))", "") + .Split(',') + .Select(coord => coord.Trim().Split(' ') + .Select(double.Parse).ToArray()).ToList(); } catch (Exception ex) { _logger.LogWarning(ex, "Error parsing location coordinates."); continue; } + spatialDataList.Add(new SpatialData { Id = Convert.ToInt32(row["Id"]), @@ -384,11 +416,13 @@ WHERE Location.STIntersects(geography::STGeomFromText('POLYGON(( Type = row["Type"] }); } + if (spatialDataList.Count == 0) { _logger.LogInformation("No spatial data found."); return Ok("{\"type\":\"FeatureCollection\",\"features\":[]}"); } + var clusters = QueryParameters.ClusterData(spatialDataList, Convert.ToInt32(parameters.ZoomLevel)); var geoJson = QueryParameters.ConvertToGeoJson(clusters); _logger.LogInformation("Clustered data fetched successfully."); @@ -404,9 +438,9 @@ WHERE Location.STIntersects(geography::STGeomFromText('POLYGON(( public class SpatialData { public int Id { get; set; } - public List Coordinates { get; set; } + public List Coordinates { get; set; } public string Type { get; set; } - public int ClusterId { get; set; } + public int ClusterId { get; set; } } @@ -439,18 +473,19 @@ public static string GetPolygonCordinates(string cordinate) cordinate = cordinate.Replace("POLYGON ((", ""); cordinate = cordinate.Replace("))", ""); var lstcordinate = cordinate.Split(','); - for (int i = 0;i coordinates) { double centroidX = 0, centroidY = 0; @@ -464,6 +499,7 @@ public static double[] CalculateCentroid(List coordinates) return new double[] { centroidX / pointCount, centroidY / pointCount }; } + public static List> ClusterData(List data, int numberOfClusters) { var centroids = data.Select(d => CalculateCentroid(d.Coordinates)).ToArray(); @@ -473,7 +509,7 @@ public static List> ClusterData(List data, int nu for (int i = 0; i < labels.Length; i++) { - data[i].ClusterId = labels[i]; + data[i].ClusterId = labels[i]; } var clusteredData = new List>(); @@ -525,9 +561,7 @@ public static GeoJsonFeatureCollection ConvertToGeoJson(List> return featureCollection; } - } - } public class LocationDataRequest diff --git a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs new file mode 100644 index 00000000..fcc022ea --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using BIE.Core.API.Controllers; +using BIE.Core.DBRepository; +using BieMetadata; + +namespace BIE.Core.API.DatasetHandlers; + +/// +/// Handler for the CSV Datatype. Assumes a single table. Uses a Geometry row named +/// +public class CsvDatasetHandler : IDatasetHandler +{ + private MetadataObject mMetadata; + + /// + /// Handler for the CSV Datatype. Assumes a single table. Uses a Geometry row named + /// + /// + public CsvDatasetHandler(MetadataObject metadata) + { + mMetadata = metadata; + } + + public string GetDataInsideArea(BoundingBox boundingBox) + { + if (!mMetadata.additionalData.Tables.Any()) + { + Console.WriteLine($"Dataset {mMetadata.basicData.DatasetId} does not contain any tables!"); + return ""; + } + + var polygon = ApiHelper.GetPolygonFromBoundingBox(boundingBox); + var tableName = mMetadata.additionalData.Tables[0].Name; + + var query = "SELECT top 1000 operator, Location.AsTextZM() AS Location" + + ApiHelper.FromTableIntersectsPolygon(tableName, polygon); + + // the list of features from combined datasets. + var features = new List>(); + var culture = new CultureInfo("en-US"); + + foreach (var row in DbHelper.GetData(query)) + { + var location = row["Location"]; + // Extract the coordinates from the POINT string + var coordinates = + location + .Replace("POINT (", "") + .Replace(")", "") + .Split(' '); + + var longitude = float.Parse(coordinates[0],culture); + var latitude = float.Parse(coordinates[1],culture); + + var feature = new Dictionary + { + { "type", "Feature" }, + { + "geometry", new Dictionary + { + { "type", "Point" }, + { + "coordinates", new List{longitude, latitude} + } + } + }, + { + "properties", new Dictionary + { + { "text", $"{row["operator"]}" } + } + } + }; + + features.Add(feature); + } + + // the response object + var responseObj = new Dictionary + { + { "type", "FeatureCollection" }, + { "features", features } + }; + return JsonSerializer.Serialize(responseObj); + } +} \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/IDatasetHandler.cs b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/IDatasetHandler.cs new file mode 100644 index 00000000..1189d7fd --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/IDatasetHandler.cs @@ -0,0 +1,8 @@ +using BieMetadata; + +namespace BIE.Core.API.DatasetHandlers; + +public interface IDatasetHandler +{ + public string GetDataInsideArea(BoundingBox boundingBox); +} \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs new file mode 100644 index 00000000..b7522581 --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using BIE.Core.API.Controllers; +using BIE.Core.DBRepository; +using BieMetadata; +using Newtonsoft.Json; +using JsonSerializer = System.Text.Json.JsonSerializer; + +namespace BIE.Core.API.DatasetHandlers; + +public class ShapeDatasetHandler : IDatasetHandler +{ + private MetadataObject mMetadata; + + /// + /// Handler for the Shape Dataset Type. Scans multiple Tables and combines results. + /// + /// + public ShapeDatasetHandler(MetadataObject metadata) + { + mMetadata = metadata; + } + + /// + /// Get the data inside a bounding box + /// + /// + /// + /// + public string GetDataInsideArea(BoundingBox boundingBox) + { + var polygon = ApiHelper.GetPolygonFromBoundingBox(boundingBox); + + // the list of features from combined datasets. + var features = new List>(); + + foreach (var table in mMetadata.additionalData.Tables) + { + if (!table.BoundingBox.HasValue) + { + continue; + } + + if (!ApiHelper.BoxIntersection(boundingBox, table.BoundingBox.Value)) + { + var bb = table.BoundingBox.Value; + // Console.WriteLine($"request-- x: {boundingBox.minX}, y: {boundingBox.minY} || x: {boundingBox.maxX}, y: {boundingBox.maxY}"); + // Console.WriteLine($"x: {bb.minX}, y: {bb.minY} || x: {bb.maxX}, y: {bb.maxY}"); + continue; + } + + // bounding boxes intersect. + // get data + + // SQL Query to find intersecting points + + var sqlQuery = $"SELECT top 1000 Location.AsTextZM() AS Location, Location.STGeometryType() AS Type" + + ApiHelper.FromTableIntersectsPolygon(table.Name, polygon); + + foreach (var row in DbHelper.GetData(sqlQuery)) + { + var feature = new Dictionary + { + { "type", "Feature" }, + { + "geometry", new Dictionary + { + { "type", $"{row["Type"]}" }, + { + "coordinates", ApiHelper.GetCoordinatesFromPolygon(row["Location"]) + } + } + }, + { + "properties", new Dictionary + { + { "text", $"{row["Type"]}" } + } + } + }; + + features.Add(feature); + } + } + + // the response object + var responseObj = new Dictionary + { + { "type", "FeatureCollection" }, + { "features", features } + }; + + return JsonSerializer.Serialize(responseObj); + } +} \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/Program.cs b/backend/src/BIE.Core/BIE.Core.API/Program.cs index 7e063602..bc15fc3d 100644 --- a/backend/src/BIE.Core/BIE.Core.API/Program.cs +++ b/backend/src/BIE.Core/BIE.Core.API/Program.cs @@ -1,3 +1,4 @@ +using System; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; diff --git a/backend/src/BIE.Core/BIE.Core.API/Properties/launchsettings.json b/backend/src/BIE.Core/BIE.Core.API/Properties/launchsettings.json new file mode 100644 index 00000000..363f07d2 --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/Properties/launchsettings.json @@ -0,0 +1,16 @@ +{ + "profiles": { + "BIE.Core.API": { + "commandName": "Project", + "environmentVariables": { + "DB_SERVER": "localhost", + "DB_NAME": "BIEDB", + "DB_PASSWORD": "MyPass@1234", + "DB_USERNAME": "db_user1", + "DB_TYPE": "SQL", + "TRUSTED": "True", + "METADATA_DB_URL": "localhost:27017" + } + } + } +} \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj b/backend/src/BIE.Core/BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj index 0e3341f4..8fb280c0 100644 --- a/backend/src/BIE.Core/BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj +++ b/backend/src/BIE.Core/BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj @@ -9,4 +9,8 @@ + + + + diff --git a/backend/src/BIE.Core/BIE.Core.DBRepository/BIE.Core.DBRepository.csproj b/backend/src/BIE.Core/BIE.Core.DBRepository/BIE.Core.DBRepository.csproj index 13a8098b..c672d117 100644 --- a/backend/src/BIE.Core/BIE.Core.DBRepository/BIE.Core.DBRepository.csproj +++ b/backend/src/BIE.Core/BIE.Core.DBRepository/BIE.Core.DBRepository.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/backend/src/BIE.Core/BIE.Core.DataObjects/BIE.Core.DataObjects.csproj b/backend/src/BIE.Core/BIE.Core.DataObjects/BIE.Core.DataObjects.csproj index dbc15171..18660f4d 100644 --- a/backend/src/BIE.Core/BIE.Core.DataObjects/BIE.Core.DataObjects.csproj +++ b/backend/src/BIE.Core/BIE.Core.DataObjects/BIE.Core.DataObjects.csproj @@ -4,4 +4,8 @@ net6.0 + + + + diff --git a/backend/src/BIE.Core/BIE.Core.Services/BIE.Core.Services.csproj b/backend/src/BIE.Core/BIE.Core.Services/BIE.Core.Services.csproj index d5978793..86060c3c 100644 --- a/backend/src/BIE.Core/BIE.Core.Services/BIE.Core.Services.csproj +++ b/backend/src/BIE.Core/BIE.Core.Services/BIE.Core.Services.csproj @@ -6,6 +6,7 @@ + diff --git a/backend/src/BIE.Core/BIE.Data/BIE.Data.csproj b/backend/src/BIE.Core/BIE.Data/BIE.Data.csproj index 6847cb07..aeb4f5ad 100644 --- a/backend/src/BIE.Core/BIE.Data/BIE.Data.csproj +++ b/backend/src/BIE.Core/BIE.Data/BIE.Data.csproj @@ -6,6 +6,7 @@ + diff --git a/backend/src/BIE.Core/Dockerfile b/backend/src/BIE.Core/Dockerfile index c75103d6..2ac09f8e 100644 --- a/backend/src/BIE.Core/Dockerfile +++ b/backend/src/BIE.Core/Dockerfile @@ -6,19 +6,24 @@ EXPOSE 80 EXPOSE 443 ### COPY PROJECTS ### +ENV C_DIR=src/BIE.Core FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build -WORKDIR /src -COPY ["BIE.Core.API/BIE.Core.API.csproj", "BIE.Core.API/"] -COPY ["BIE.Core.DataObjects/BIE.Core.DataObjects.csproj", "BIE.Core.DataObjects/"] -COPY ["BIE.Core.Services/BIE.Core.Services.csproj", "BIE.Core.Services/"] -COPY ["BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj", "BIE.Core.BaseRepository/"] -COPY ["BIE.Core.DBRepository/BIE.Core.DBRepository.csproj", "BIE.Core.DBRepository/"] -COPY ["BIE.Data/BIE.Data.csproj", "BIE.Data/"] -RUN dotnet restore "BIE.Core.API/BIE.Core.API.csproj" -COPY . . +WORKDIR /source +COPY ["src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj", "src/BIE.Core/BIE.Core.API/"] +COPY ["src/BIE.Core/BIE.Core.DataObjects/BIE.Core.DataObjects.csproj", "src/BIE.Core/BIE.Core.DataObjects/"] +COPY ["src/BIE.Core/BIE.Core.Services/BIE.Core.Services.csproj", "src/BIE.Core/BIE.Core.Services/"] +COPY ["src/BIE.Core/BIE.Core.BaseRepository/BIE.Core.BaseRepository.csproj", "src/BIE.Core/BIE.Core.BaseRepository/"] +COPY ["src/BIE.Core/BIE.Core.DBRepository/BIE.Core.DBRepository.csproj", "src/BIE.Core/BIE.Core.DBRepository/"] +COPY ["src/BIE.Core/BIE.Data/BIE.Data.csproj", "src/BIE.Core/BIE.Data/"] +RUN dotnet restore "src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj" +COPY "./src/BIE.Core" "./src/BIE.Core" ### BUILD ### -WORKDIR "/src/BIE.Core.API" +# Copy and build metadata library +COPY ./lib/BieMetadata ./lib/BieMetadata +RUN dotnet build "./lib/BieMetadata/BieMetadata.csproj" -c Release + +WORKDIR "/source/src/BIE.Core/BIE.Core.API" RUN dotnet build "BIE.Core.API.csproj" -c Release -o /app/build ### PUBLISH ### diff --git a/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj b/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj index 31722b8b..76d8d4d2 100644 --- a/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj +++ b/backend/src/BIE.DataPipeline/BIE.DataPipeline.csproj @@ -41,6 +41,11 @@ + + + Always + + @@ -48,4 +53,11 @@ + + + + ..\..\lib\BieMetadata\bin\Release\net6.0\BieMetadata.dll + + + diff --git a/backend/src/BIE.DataPipeline/DbHelper.cs b/backend/src/BIE.DataPipeline/DbHelper.cs index 8fc5482f..629e3180 100644 --- a/backend/src/BIE.DataPipeline/DbHelper.cs +++ b/backend/src/BIE.DataPipeline/DbHelper.cs @@ -3,6 +3,7 @@ using System.Text; using BIE.Data; using BIE.DataPipeline.Import; +using BieMetadata; namespace BIE.DataPipeline { @@ -105,6 +106,7 @@ internal bool CreateTable(DataSourceDescription description) return false; } } + /// /// Check if location column exists /// @@ -116,7 +118,7 @@ public bool CheckIfColumnExists(DataSourceDescription description) FROM sys.columns c JOIN sys.tables t ON c.object_id = t.object_id JOIN sys.types ty ON c.user_type_id = ty.user_type_id - WHERE t.name = '"+description.table_name+"' AND c.name = 'Location' AND ty.name = 'geometry';"; + WHERE t.name = '" + description.table_name + "' AND c.name = 'Location' AND ty.name = 'geometry';"; var db = Database.Instance; using (var command = db.CreateCommand(query)) @@ -146,8 +148,8 @@ internal bool CreateIndexes(DataSourceDescription description) Console.WriteLine("Creating Index..."); var db = Database.Instance; - // Step 1: Check if the ID column exists, and add it if it doesn't - string addColumnQuery = @" + // Step 1: Check if the ID column exists, and add it if it doesn't, create it. + var addIdCollumnQuery = @" USE BIEDB; IF NOT EXISTS ( SELECT * @@ -161,89 +163,57 @@ ALTER TABLE dbo." + description.table_name + @" END; "; - var addColumnCmd = db.CreateCommand(addColumnQuery); - db.Execute(addColumnCmd); + var addIdCollumnCommand = db.CreateCommand(addIdCollumnQuery); + db.Execute(addIdCollumnCommand); // Step 2: Ensure the ID column is the primary key - string primaryKeyQuery = @" + var primaryKeyQuery = $@" USE BIEDB; IF NOT EXISTS ( SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS - WHERE TABLE_NAME = '" + description.table_name + @"' + WHERE TABLE_NAME = '{description.table_name}' AND CONSTRAINT_TYPE = 'PRIMARY KEY' ) BEGIN - ALTER TABLE dbo." + description.table_name + @" - ADD CONSTRAINT PK_" + description.table_name + @"_ID PRIMARY KEY CLUSTERED (ID); + ALTER TABLE dbo.{description.table_name} + ADD CONSTRAINT PK_{description.table_name}_ID PRIMARY KEY CLUSTERED (ID); END; "; - var pkCmd = db.CreateCommand(primaryKeyQuery); - db.Execute(pkCmd); + var primaryKeyCommand = db.CreateCommand(primaryKeyQuery); + db.Execute(primaryKeyCommand); - // Step 3: Calculate the bounding box - string bboxQuery = @" - USE BIEDB; - DECLARE @MinX FLOAT, @MinY FLOAT, @MaxX FLOAT, @MaxY FLOAT; - - WITH ConvertedGeography AS ( - SELECT Location.STAsText() AS WKT - FROM dbo." + description.table_name + @" - ) - SELECT - @MinX = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(1).STX, - @MinY = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(1).STY, - @MaxX = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(3).STX, - @MaxY = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(3).STY - FROM ConvertedGeography; - - SELECT @MinX AS MinX, @MinY AS MinY, @MaxX AS MaxX, @MaxY AS MaxY; -"; - - var bboxCmd = db.CreateCommand(bboxQuery); - var (bboxReader, bboxConnection) = db.ExecuteReader(bboxCmd); - - double minX = 0, minY = 0, maxX = 0, maxY = 0; - - if (bboxReader.Read()) - { - minX = (double)bboxReader["MinX"]; - minY = (double)bboxReader["MinY"]; - maxX = (double)bboxReader["MaxX"]; - maxY = (double)bboxReader["MaxY"]; - } - - var cultureInfo = new CultureInfo("en-US"); - Console.WriteLine($"BBox: MinX = {minX.ToString(cultureInfo)}, MinY = {minY.ToString(cultureInfo)}, MaxX = {maxX.ToString(cultureInfo)}, MaxY = {maxY.ToString(cultureInfo)}"); - - bboxReader.Close(); - bboxConnection.Close(); + var boundingBox = GetBoundingBox(description.table_name); + Console.WriteLine($"BBox: MinX = {boundingBox.minX}," + + $" MinY = {boundingBox.minY}," + + $" MaxX = {boundingBox.maxX}," + + $" MaxY = {boundingBox.maxY}"); // Step 4: Create the spatial index - string indexQuery = @" + var indexQuery = $@" USE BIEDB; SET QUOTED_IDENTIFIER ON; IF NOT EXISTS ( SELECT * FROM sys.indexes - WHERE name = 'SI_" + description.table_name + @"_Location' - AND object_id = OBJECT_ID('dbo." + description.table_name + @"') + WHERE name = 'SI_{description.table_name}_Location' + AND object_id = OBJECT_ID('dbo.{description.table_name}') ) BEGIN - CREATE SPATIAL INDEX SI_" + description.table_name + @"_Location - ON dbo." + description.table_name + @"(Location) + CREATE SPATIAL INDEX SI_{description.table_name}_Location + ON dbo.{description.table_name}(Location) USING GEOMETRY_AUTO_GRID WITH ( BOUNDING_BOX = ( - XMIN = " + minX.ToString(cultureInfo) + @", - YMIN = " + minY.ToString(cultureInfo) + @", - XMAX = " + maxX.ToString(cultureInfo) + @", - YMAX = " + maxY.ToString(cultureInfo) + @" + XMIN = {boundingBox.minX}, + YMIN = {boundingBox.minY}, + XMAX = {boundingBox.maxX}, + YMAX = {boundingBox.maxY} ) ); END; - UPDATE STATISTICS dbo." + description.table_name + @"; + UPDATE STATISTICS dbo.{description.table_name}; "; var cmd = db.CreateCommand(indexQuery); @@ -261,8 +231,52 @@ USING GEOMETRY_AUTO_GRID } } + public BoundingBox GetBoundingBox(string tableName) + { + var db = Database.Instance; + + // Step 3: Calculate the bounding box + string bboxQuery = $@" + USE BIEDB; + DECLARE @MinX FLOAT, @MinY FLOAT, @MaxX FLOAT, @MaxY FLOAT; + + WITH ConvertedGeography AS ( + SELECT Location.STAsText() AS WKT + FROM dbo.{tableName} + ) + SELECT + @MinX = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(1).STX, + @MinY = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(1).STY, + @MaxX = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(3).STX, + @MaxY = geometry::EnvelopeAggregate(geometry::STGeomFromText(WKT, 4326)).STPointN(3).STY + FROM ConvertedGeography; + + SELECT @MinX AS MinX, @MinY AS MinY, @MaxX AS MaxX, @MaxY AS MaxY; +"; + + var bboxCmd = db.CreateCommand(bboxQuery); + var (bboxReader, bboxConnection) = db.ExecuteReader(bboxCmd); + + float minX = 0 , minY = 0, maxX = 0, maxY = 0; + + if (bboxReader.Read()) + { + var culture = new CultureInfo("en-US"); + minX = float.Parse(bboxReader["MinX"].ToString()!, culture); + minY = float.Parse(bboxReader["MinY"].ToString()!, culture); + maxX = float.Parse(bboxReader["MaxX"].ToString()!, culture); + maxY = float.Parse(bboxReader["MaxY"].ToString()!, culture); + } + + + bboxReader.Close(); + bboxConnection.Close(); + + return new BoundingBox() { minX = minX, minY = minY, maxX = maxX, maxY = maxY }; + } + - private string GetCreationQuery(DataSourceDescription? description) + private string GetCreationQuery(DataSourceDescription description) { if (description.source.data_format == "SHAPE") { @@ -302,7 +316,7 @@ IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{desc query += $" {column.name_in_table} {column.type}, "; } - if(description.options.location_to_SQL_point != null) + if (description.options.location_to_SQL_point != null) { query += $" {description.options.location_to_SQL_point.name_in_table} GEOMETRY,"; } @@ -372,7 +386,7 @@ private void ConfigureEnvironmentVariables() Environment.GetEnvironmentVariable("DB_TYPE") ?? throw new Exception("Could not get EnvironmentVariable DB_TYPE")); - + if (dbServer == null || dbName == null || dbUser == null || dbPassword == null) { throw new ExternalException("Could not get Environment Variables."); diff --git a/backend/src/BIE.DataPipeline/Dockerfile b/backend/src/BIE.DataPipeline/Dockerfile index fdf1e0f8..e20773af 100644 --- a/backend/src/BIE.DataPipeline/Dockerfile +++ b/backend/src/BIE.DataPipeline/Dockerfile @@ -11,18 +11,22 @@ RUN apt-get update && apt-get install -y dos2unix FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build WORKDIR /app -COPY *.csproj . -RUN dotnet restore +COPY "./src/BIE.DataPipeline/BIE.DataPipeline.csproj" "./src/BIE.DataPipeline/" +RUN dotnet restore "./src/BIE.DataPipeline/BIE.DataPipeline.csproj" -COPY . ./ +COPY "./src/BIE.DataPipeline" "./src/BIE.DataPipeline" -RUN dotnet publish "BIE.DataPipeline.csproj" --output /app/ --configuration Release +# Copy and build metadata library +COPY ./lib/BieMetadata ./lib/BieMetadata +RUN dotnet build "./lib/BieMetadata/BieMetadata.csproj" -c Release + +RUN dotnet publish "src/BIE.DataPipeline/BIE.DataPipeline.csproj" --output /app/ --configuration Release # Run the project FROM base AS final WORKDIR /app COPY --from=build /app . -RUN dos2unix entrypoint.sh +RUN dos2unix "src/BIE.DataPipeline/entrypoint.sh" -ENTRYPOINT ["/bin/bash","entrypoint.sh"] +ENTRYPOINT ["/bin/bash","src/BIE.DataPipeline/entrypoint.sh"] diff --git a/backend/src/BIE.DataPipeline/Import/CsvImporter.cs b/backend/src/BIE.DataPipeline/Import/CsvImporter.cs index 4d7ed218..7eb6f1b9 100644 --- a/backend/src/BIE.DataPipeline/Import/CsvImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/CsvImporter.cs @@ -11,6 +11,7 @@ using BIE.DataPipeline; using Microsoft.VisualBasic.FileIO; using NetTopologySuite.Geometries; +// ReSharper disable InconsistentNaming [assembly: InternalsVisibleTo("BIE.Tests")] @@ -19,7 +20,7 @@ namespace BIE.DataPipeline.Import internal class CsvImporter : IImporter { private TextFieldParser parser; - private DataSourceDescription? dataSourceDescription; + private DataSourceDescription dataSourceDescription; private Type[] columnTypes; private string[] fileHeader; private string[] yamlHeader; @@ -29,7 +30,7 @@ internal class CsvImporter : IImporter private StringBuilder builder; - public CsvImporter(DataSourceDescription? dataSourceDescription) + public CsvImporter(DataSourceDescription dataSourceDescription) { // CultureInfo cultureInfo = new CultureInfo("en-US"); diff --git a/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs b/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs index 899ae019..1aec5813 100644 --- a/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs @@ -163,8 +163,10 @@ private Geometry ConvertUtmToLatLong(Geometry polygon) // Extract latitude and longitude double latitude = wgs84Point[1]; double longitude = wgs84Point[0]; - coordinate.X = latitude; - coordinate.Y = longitude; + // this has probably caused some issues... + // Lat = Y , Lon = X + coordinate.X = longitude; + coordinate.Y = latitude; } return polygon; diff --git a/backend/src/BIE.DataPipeline/Import/YamlImporter.cs b/backend/src/BIE.DataPipeline/Import/YamlImporter.cs index 8b1430a4..05beed54 100644 --- a/backend/src/BIE.DataPipeline/Import/YamlImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/YamlImporter.cs @@ -187,7 +187,7 @@ public InsertBehaviour if_table_exists [DefaultValue(null)] - public LocationToSQLPoint location_to_SQL_point { get; set; } + public LocationToSQLPoint? location_to_SQL_point { get; set; } public override bool Equals(object? obj) { diff --git a/backend/src/BIE.DataPipeline/Metadata/MetadataObject.cs b/backend/src/BIE.DataPipeline/Metadata/MetadataObject.cs deleted file mode 100644 index 5a142b67..00000000 --- a/backend/src/BIE.DataPipeline/Metadata/MetadataObject.cs +++ /dev/null @@ -1,55 +0,0 @@ -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; - -namespace BIE.DataPipeline.Metadata; - -public class MetadataObject -{ - [BsonRepresentation(BsonType.ObjectId)] - public string _id { get; set; } = string.Empty; - - [BsonElement("basicData")] - public BasicData basicData { get; set; } = new BasicData(); - - [BsonElement("additionalData")] - public AdditionalData additionalData { get; set; } = new AdditionalData(); - - // The general and most important data about a dataset. - public class BasicData - { - public string DatasetId { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string ShortDescription { get; set; } = string.Empty; - public string Icon { get; set; } = string.Empty; - } - - // The additional data for each of the datasets, queried at a request. - public class AdditionalData - { - public string Icon { get; set; } = string.Empty; - public string Type { get; set; } = string.Empty; - - public string LongDescription { get; set; } = string.Empty; - - // Zoom level is higher the closer you look at something. If current zoom level is below this, it shouldn't display any value. - public int MinZoomLevel { get; set; } = 0; - - // The zoom threshold where areas start to turn into markers - public int MarkersThreshold { get; set; } = 0; - - // The display property is the property that should be shown in a popup. - public string DisplayProperty { get; set; } = string.Empty; - - // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. - public List Tables { get; set; } = new List(); - } - - // Table data populated by the data pipeline. Contains the name and the size of the all .yaml files correlated to that specific dataset. - public class TableData - { - // The name of the .yaml file - public string Name { get; set; } = string.Empty; - // The number of lines of data in that file. - public int NumberOfLines { get; set; } = 0; - } -} \ No newline at end of file diff --git a/backend/src/BIE.DataPipeline/Program.cs b/backend/src/BIE.DataPipeline/Program.cs index c8a9a5f8..68175ee8 100644 --- a/backend/src/BIE.DataPipeline/Program.cs +++ b/backend/src/BIE.DataPipeline/Program.cs @@ -1,41 +1,50 @@ -using BIE.DataPipeline; +using System.Globalization; +using BIE.DataPipeline; using BIE.DataPipeline.Import; -using BIE.DataPipeline.Metadata; +using BieMetadata; using Mono.Options; using System.Xml; -// setup command line options. + +// set the culture to be always en-US +CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); + +// Setup command line options. var tableInsertBehaviour = InsertBehaviour.none; + var filename = HandleCliArguments(); +Console.WriteLine("Starting datapipeline"); + var description = GetDataSourceDescription(filename); if (description == null) { return 1; } -Console.WriteLine($@"Starting the Data Pipeline for {filename}: +Console.WriteLine($@"Running with {filename}: type: {description.source.type} format: {description.source.data_format} location: {description.source.location} table name: {description.table_name} + "); -// Check if the table insert behaviour is overwritten if (tableInsertBehaviour != InsertBehaviour.none) { description.options.if_table_exists = tableInsertBehaviour; - Console.WriteLine($"Overwriting description: Using {tableInsertBehaviour} Behaviour for insertion."); + Console.WriteLine($"Overwriting Description: Using {tableInsertBehaviour} Behaviour for insertion."); } -// Create the connection to the database +// create connection to database; var dbHelper = new DbHelper(); -// Exit when the connection is not possible +// End if Connection not possible. if (!dbHelper.CheckConnection()) { - Console.WriteLine("Could not establish database connection, exiting..."); + Console.WriteLine("Could not establish Database Connection, exiting..."); return 1; } + Console.WriteLine("Established the database connection."); // End if Dataset can be skipped @@ -49,12 +58,12 @@ if (!metadataDbHelper.CreateConnection()) { // maybe make optional? + Console.WriteLine("metadataDatabase could not found"); return 1; } -Console.WriteLine("Starting the importer..."); +Console.WriteLine("Starting Importer"); -// Import the data based on the specified data type IImporter importer; try { @@ -82,7 +91,7 @@ } catch (Exception e) { - Console.WriteLine("Error while setting up the importer."); + Console.WriteLine("Error While setting up Importer."); Console.WriteLine(e); return 1; } @@ -96,9 +105,9 @@ { var line = ""; var notEof = importer.ReadLine(out line); - Console.WriteLine("Inserting the data into the database..."); - // Read all lines + Console.WriteLine("Inserting into Database"); + var count = 0; while (notEof) { @@ -116,41 +125,33 @@ // Optionally, you can decide to break the loop or continue based on the type of error } } + Console.WriteLine($"Finished inserting {count} lines of data."); if (dbHelper.CheckIfColumnExists(description)) { - dbHelper.CreateIndexes(description); - } + } + Console.WriteLine("Updating the metadata..."); - metadataDbHelper.UpdateMetadata(description, count); + var boundingBox = dbHelper.GetBoundingBox(description.table_name); + if (!metadataDbHelper.UpdateMetadata(description.dataset, description.table_name, count, boundingBox)) + { + return 1; + } + Console.WriteLine("The metadata was updated."); Console.WriteLine("--------------------------------------------------------------"); } catch (Exception e) { - Console.WriteLine("Error inserting the data into the database:"); + Console.WriteLine("Error inserting into Database:"); Console.WriteLine(e); return 1; } -try -{ -} -catch (Exception e) -{ - Console.WriteLine("Could not insert into Metadata DB"); - Console.WriteLine(e); - throw; -} - return 0; -// ------------------------------------------------------- -// FUNCTIONS -// ------------------------------------------------------- - string HandleCliArguments() { var options = new OptionSet diff --git a/backend/src/BIE.DataPipeline/Properties/launchsettings.json b/backend/src/BIE.DataPipeline/Properties/launchsettings.json index 80e044e8..967d5a65 100644 --- a/backend/src/BIE.DataPipeline/Properties/launchsettings.json +++ b/backend/src/BIE.DataPipeline/Properties/launchsettings.json @@ -2,7 +2,7 @@ "profiles": { "BIE.DataPipeline": { "commandName": "Project", - "commandLineArgs": "--behavior=skip actual_use_Kreisfreie_Stadt_Schweinfurt.yaml", + "commandLineArgs": "--behavior=replace common/actual_use_Kreisfreie_Stadt_Nuernberg.yaml", "workingDirectory": "yaml", "environmentVariables": { "DB_SERVER": "localhost", diff --git a/backend/src/BIE.DataPipeline/tmp.json b/backend/src/BIE.DataPipeline/tmp.json new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/BIE.DataPipeline/yaml/common/actual_use_Kreisfreie_Stadt_Schweinfurt.yaml b/backend/src/BIE.DataPipeline/yaml/common/actual_use_Kreisfreie_Stadt_Schweinfurt.yaml new file mode 100644 index 00000000..c1389f11 --- /dev/null +++ b/backend/src/BIE.DataPipeline/yaml/common/actual_use_Kreisfreie_Stadt_Schweinfurt.yaml @@ -0,0 +1,18 @@ +# describe the source +source: + # link | filepath + type: URL + location: https://download1.bayernwolke.de/a/tn/lkr/tn_09662.zip + data_format: SHAPE +options: + # skip lines at the beginning + skip_lines: 0 + # discard any rows that have null values + discard_null_rows: false + # how to deal with existing table. Options: ignore, replace, skip (default). + if_table_exists: skip +table_name: actual_use_Kreisfreie_Stadt_Schweinfurt + +table_cols: + +dataset: actual_use diff --git a/docker-compose.yml b/docker-compose.yml index a6ca93f4..bb897402 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,8 +18,8 @@ services: # API Gateway for listening for the FE requests api-gateway: build: - context: ./backend/api-gateway - dockerfile: Dockerfile + context: ./backend + dockerfile: ./api-gateway/Dockerfile container_name: api-gateway image: ghcr.io/amosproj/amos2024ss04-building-information-enhancer-api-gateway:${DOCKER_COMPOSE_IMAGES_TAG} environment: @@ -32,8 +32,8 @@ services: # API Composer for composing the FE requests api-composer: build: - context: ./backend/src/BIE.Core - dockerfile: Dockerfile + context: ./backend + dockerfile: ./src/BIE.Core/Dockerfile container_name: api-composer image: ghcr.io/amosproj/amos2024ss04-building-information-enhancer-api-composer:${DOCKER_COMPOSE_IMAGES_TAG} environment: @@ -44,6 +44,7 @@ services: - DB_USERNAME=${SQL_DB_USERNAME} - DB_TYPE=${SQL_DB_TYPE} - TRUSTED=${SQL_TRUSTED} + - METADATA_DB_URL=metadata-database:${METADATA_DATABASE_PORT} ports: - ${API_COMPOSER_PORT}:80 networks: @@ -55,8 +56,8 @@ services: ## Data Pipeline for ingesting the datasets datapipeline: build: - context: ./backend/src/BIE.DataPipeline - dockerfile: Dockerfile + context: ./backend + dockerfile: ./src/BIE.DataPipeline/Dockerfile image: ghcr.io/amosproj/amos2024ss04-building-information-enhancer-datapipeline:${DOCKER_COMPOSE_IMAGES_TAG} environment: - DB_NAME=${SQL_DB_NAME}