diff --git a/backend/api-gateway/Controllers/APIGatewayController.cs b/backend/api-gateway/Controllers/APIGatewayController.cs index 92d679a8..1d97dd23 100644 --- a/backend/api-gateway/Controllers/APIGatewayController.cs +++ b/backend/api-gateway/Controllers/APIGatewayController.cs @@ -138,27 +138,6 @@ public async Task LoadLocationData([FromBody, Required] LocationD } - // Helper method to generate mock data - private List GenerateMockData(string mapId, int count) - { - var random = new Random(); - var data = new List(); - - for (int i = 0; i < count; i++) - { - var item = new DatasetItem - { - Id = Guid.NewGuid().ToString(), // Generate a unique ID - Key = $"{mapId} Item {i + 1} (distance)", - Value = $"{random.Next(50, 1000)} m", - MapId = mapId - }; - data.Add(item); - } - - return data; - } - /// /// Gets the dataset viewport data based on the provided parameters. /// diff --git a/backend/api-gateway/Models/LocationData.cs b/backend/api-gateway/Models/LocationData.cs index 2351ffb2..ae72ff58 100644 --- a/backend/api-gateway/Models/LocationData.cs +++ b/backend/api-gateway/Models/LocationData.cs @@ -18,17 +18,28 @@ public class Coordinate public class LocationDataResponse { - public List IndividualData { get; set; } = new List(); - public List GeneralData { get; set; } = new List(); + public List IndividualData { get; set; } = []; + public List SelectionData { get; set; } = []; } public class DatasetItem { - public string Id { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string DatasetId { get; set; } = string.Empty; + public double[] Coordinate { get; set; } = []; + public List PolygonCoordinates { get; set; } = []; + public List Subdata { get; set; } = []; + public string Value { get; set; } = string.Empty; // some items may not have subdata and should instead be directly displayed with a value + + } + + public class SubdataItem + { public string Key { get; set; } = string.Empty; + public string Value { get; set; } = string.Empty; - public string MapId { get; set; } = string.Empty; // Optional -> for "open in map" functionality + } diff --git a/backend/lib/BieMetadata/MetadataDbHelper.cs b/backend/lib/BieMetadata/MetadataDbHelper.cs index 75cfb856..4c2767d5 100644 --- a/backend/lib/BieMetadata/MetadataDbHelper.cs +++ b/backend/lib/BieMetadata/MetadataDbHelper.cs @@ -7,7 +7,7 @@ public class MetadataDbHelper private string mMetaDataDbUrl; private IMongoDatabase mDatabase; - + public bool Connected { get; private set; } public MetadataDbHelper() @@ -58,7 +58,7 @@ public bool CreateConnection() return metadataObject; } - public bool UpdateMetadata(string dataset, string tableName, int numberOfLines, BoundingBox boundingBox) + public bool UpdateMetadata(string dataset, MetadataObject.TableData tableData) { // Load the collection var collection = mDatabase.GetCollection("datasets"); @@ -74,31 +74,28 @@ public bool UpdateMetadata(string dataset, string tableName, int numberOfLines, return false; } - // Load the existing table - var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == tableName); - if (existingTable == null) + // Load and remove any existing table + var existingTable = metadataObject.additionalData.Tables.Find(t => t.Name == tableData.Name); + if (existingTable != null) { - // Create a new table object if not present - var newTable = new MetadataObject.TableData() - { - Name = tableName, - NumberOfLines = numberOfLines, - BoundingBox = boundingBox - }; - metadataObject.additionalData.Tables.Add(newTable); - collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject); - return true; + metadataObject.additionalData.Tables.Remove(existingTable); } - // Table info already exists, for now just choose the larger number of lines number. - existingTable.NumberOfLines = existingTable.NumberOfLines < numberOfLines - ? numberOfLines - : existingTable.NumberOfLines; - - // always write the current Bounding box - existingTable.BoundingBox = boundingBox; - + // Insert the new Table object. + metadataObject.additionalData.Tables.Add(tableData); collection.ReplaceOne(g => g.basicData.DatasetId == dataset, metadataObject); return true; } -} + + public bool UpdateMetadata(string dataset, string tableName, int numberOfLines, BoundingBox boundingBox) + { + return UpdateMetadata(dataset, + new MetadataObject.TableData() + { + Name = tableName, + NumberOfLines = numberOfLines, + BoundingBox = new BoundingBox(), + RowHeaders = new List() + }); + } +} \ No newline at end of file diff --git a/backend/lib/BieMetadata/MetadataObject.cs b/backend/lib/BieMetadata/MetadataObject.cs index 38df97c1..2e389db0 100644 --- a/backend/lib/BieMetadata/MetadataObject.cs +++ b/backend/lib/BieMetadata/MetadataObject.cs @@ -82,12 +82,25 @@ public class AdditionalData /// public class TableData { - // The name of the .yaml file + /// + /// the name of the table + /// public string Name { get; set; } = string.Empty; - // The number of lines of data in that file. + + /// + /// the number of lines in the table + /// public int NumberOfLines { get; set; } = 0; + /// + /// the bounding box of the geomtry data in the table + /// public BoundingBox? BoundingBox { get; set; } + + /// + /// the headers of the dataset. Should NOT include the special Location header. + /// + public List RowHeaders { get; set; } } /// diff --git a/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs b/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs index ce132dbe..4d7106ae 100644 --- a/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs +++ b/backend/src/BIE.Core/BIE.Core.API/ApiHelper.cs @@ -40,8 +40,9 @@ public static class ApiHelper AUTHORITY[""ESRI"",""102008""] "; // see http://epsg.io/27700 private static ICoordinateTransformation mTransformation = new CoordinateTransformationFactory(). - CreateFromCoordinateSystems(GeographicCoordinateSystem.WGS84, - (CoordinateSystem)CoordinateSystemWktReader.Parse(epsg27700)); + CreateFromCoordinateSystems(GeographicCoordinateSystem.WGS84, ProjectedCoordinateSystem.WGS84_UTM(32, true)); + private static ICoordinateTransformation transform = new CoordinateTransformationFactory().CreateFromCoordinateSystems(GeographicCoordinateSystem.WGS84, ProjectedCoordinateSystem.WGS84_UTM(32, true)); + public static double getDistance(double longitude, double latitude, Point p) { @@ -58,28 +59,32 @@ public static double getDistance(double longitude, double latitude, Point p) public static double getArea(Polygon p) { - // convert each point of the polygon into a new coordinate (revert x and y - // convert it into EPSG27700 - // return area + var coordinates = p.Coordinates; - Console.WriteLine($"coordinates {coordinates}"); + var transformedCoordinates = new Coordinate[coordinates.Length]; // Transform each coordinate for (int i = 0; i < coordinates.Length; i++) { - double[] pointWGS84 = { coordinates[i].Y, coordinates[i].X }; // Switch X and Y - double[] pointEPSG27700 = mTransformation.MathTransform.Transform(pointWGS84); + //Console.WriteLine($"FIRST X THEN Y coordinates before transformation (but switched xy) {coordinates[i].X},{coordinates[i].Y}"); + double[] pointWGS84 = { coordinates[i].X, coordinates[i].Y }; + double[] pointEPSG27700 = transform.MathTransform.Transform(pointWGS84); transformedCoordinates[i] = new Coordinate(pointEPSG27700[0], pointEPSG27700[1]); - Console.WriteLine($"transformedCoordinates[i] {transformedCoordinates[i]}"); + //Console.WriteLine($"transformedCoordinates[i] {transformedCoordinates[i]}"); } - var geometryFactory = new GeometryFactory(new PrecisionModel(), 27700); - var transformedPolygon = new Polygon(new LinearRing(transformedCoordinates), geometryFactory); - Console.WriteLine($"area {transformedPolygon.Area}"); - Console.WriteLine($"transformed coords {transformedPolygon.Coordinates}"); + // Ensure the polygon is closed + if (!transformedCoordinates[0].Equals2D(transformedCoordinates[transformedCoordinates.Length - 1])) + { + Array.Resize(ref transformedCoordinates, transformedCoordinates.Length + 1); + transformedCoordinates[transformedCoordinates.Length - 1] = transformedCoordinates[0]; + } + // Create a polygon with transformed coordinates + var geometryFactory = new GeometryFactory(); + var transformedPolygon = geometryFactory.CreatePolygon(transformedCoordinates); - // Return the area of the transformed polygon + // Calculate and return the area return transformedPolygon.Area; } @@ -132,23 +137,23 @@ public static List ConvertToWktPolygons(List>> locatio foreach (var polygon in locations) { var wktString = "POLYGON(("; - foreach (var point in polygon) + for (int i = 0; i < polygon.Count; i++) { + var point = polygon[i]; if (point.Count != 2) { throw new ArgumentException("Each point should have exactly two coordinates."); } - var longitude = point[0].ToString(culture); - var latitude = point[1].ToString(culture); - wktString += $"{longitude} {latitude}, "; + var longitude = point[0].ToString(sCultureInfo); + var latitude = point[1].ToString(sCultureInfo); + wktString += $"{longitude} {latitude}"; + if (i < polygon.Count - 1) + { + wktString += ", "; + } } - - // Close the polygon by repeating the first point - var firstPoint = polygon[0]; - var firstLongitude = firstPoint[0].ToString(culture); - var firstLatitude = firstPoint[1].ToString(culture); - wktString += $"{firstLongitude} {firstLatitude}))"; + wktString += "))"; wktPolygons.Add(wktString); } @@ -198,7 +203,7 @@ public static bool BoxIntersection(BoundingBox box1, BoundingBox box2) /// the name of the table to filter /// the polygon string /// - public static string FromTableIntersectsPolygon(string tableName, string polygon) + public static string FromTableWhereIntersectsPolygon(string tableName, string polygon) { return $@" FROM dbo.{tableName} diff --git a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj index fdd411a2..46f64f3b 100644 --- a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj +++ b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.csproj @@ -14,6 +14,7 @@ + diff --git a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml index 6ea0d60a..22101e5d 100644 --- a/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml +++ b/backend/src/BIE.Core/BIE.Core.API/BIE.Core.API.xml @@ -33,7 +33,7 @@ - + Gets the Query to filter a table via polygon Intersection. Presents the FROM part of a Query. @@ -74,13 +74,6 @@ In case of a single point a list with a single element. Data for the specified point/polygon as a list of key/values. - - - WIP: DO NOT USE, Get a record - - - - Get a record @@ -158,6 +151,13 @@ + + + Get the GeoJson Feature Object corresponding to the object + + + + Global constatns diff --git a/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs b/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs index c720bfe0..24ad21c8 100644 --- a/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs +++ b/backend/src/BIE.Core/BIE.Core.API/Controllers/DatasetController.cs @@ -30,6 +30,7 @@ using MySqlX.XDevAPI.Common; using static MongoDB.Bson.Serialization.Serializers.SerializerHelper; using Accord.Math; +using LinqStatistics; namespace BIE.Core.API.Controllers { @@ -97,14 +98,14 @@ public ActionResult GetDatasetViewportData([FromQuery] QueryParameters parameter // check if the dataset is present in the Metadata var metadata = MetadataDbHelper.GetMetadata(parameters.Id); if (metadata == null) - { return StatusCode(400, $"Unsupported dataset: {parameters.Id}"); - } + // select the correct Handler IDatasetHandler handler; switch (metadata.additionalData.DataType) { + case "CITYGML": case "SHAPE": handler = new ShapeDatasetHandler(metadata); break; @@ -121,8 +122,6 @@ public ActionResult GetDatasetViewportData([FromQuery] QueryParameters parameter return Ok(handler.GetDataInsideArea(boundingBox)); } - - /// /// Loads the location data for the given point or polygon. /// @@ -140,290 +139,22 @@ public IActionResult LoadLocationData([FromBody, Required] LocationDataRequest r return BadRequest(ModelState); } + bool onlySinglePoint = request.Location.Count == 1 && request.Location.ElementAt(0).Count == 1; + try { - var generalData = new List(); - var individualData = new List(); - - // can still be called even if we have a single point - var polygons = ApiHelper.ConvertToWktPolygons(request.Location); - - - if (request.Location.Count <= 0) - { - return BadRequest("at least a single point has to be given"); - } - - bool onlySinglePoint = request.Location.Count == 1 && request.Location.ElementAt(0).Count == 1; - - Point pointForCalculations = new Point(new Coordinate()); + LocationDataResponse data; if (onlySinglePoint) - { - var p = request.Location.First().First(); - pointForCalculations = new Point(p.ElementAt(0), p.ElementAt(1)); - } + data = LoadLocationHelper.loadLocationDataForSinglePoint(request); else - { - // Create a WKTReader instance - WKTReader reader = new WKTReader(); - Geometry geometry = reader.Read(polygons.First()); - if (geometry is Polygon polygon) - pointForCalculations = polygon.Centroid; - - } - - // closest charging stations and number of charging stations - var radius = 10000000; // Define the radius as needed - var columns = new HashSet { "operator", "Id", "rated_power_kw" }; - var metadata = MetadataDbHelper.GetMetadata("EV_charging_stations"); - if (metadata != null) - { - var tables = metadata.additionalData.Tables; - if (tables.Count == 1) - { - var chargingStations = getNclosestObjects(tables[0].Name, pointForCalculations.Y, pointForCalculations.X, radius, 3, columns); - - // Log chargingStations to inspect the content - foreach (var station in chargingStations) - { - _logger.LogInformation($"Station data: {string.Join(", ", station.Select(kv => $"{kv.Key}: {kv.Value}"))}"); - } - - var chargingStationsData = chargingStations.Select(h => new DatasetItem - { - Id = h.ContainsKey("Id") ? h["Id"] : null, - Key = "Charging station: " + (h.ContainsKey("operator") ? h["operator"] : "No operator defined"), - Value = (h.ContainsKey("Distance") ? h["Distance"] + "m" : "-1") + " | " + (h.ContainsKey("rated_power_kw") ? h["rated_power_kw"] + "kw" : "-1"), - MapId = "EV_charging_stations", - }); - - individualData.AddRange(chargingStationsData); - } - } - - - var housefootprints = new List>(); - if(onlySinglePoint) - { - // Additional section for house footprints - columns = new HashSet { "Id" }; - metadata = MetadataDbHelper.GetMetadata("house_footprints"); - if (metadata != null) - { - var p = request.Location.First().First(); - foreach (var table in metadata.additionalData.Tables) - housefootprints.AddRange(getMatchingObjects(table.Name, p.ElementAt(1), p.ElementAt(0), columns)); - - } - } - else - { - _logger.LogInformation("Do housefootprints for area"); - - foreach (var polygonWkt in polygons) - { - // then go through house footprints - metadata = MetadataDbHelper.GetMetadata("house_footprints"); - if (metadata != null) - { - _logger.LogInformation("metadata from housefootprints retrieved"); - foreach (var table in metadata.additionalData.Tables) - { - var sqlQuery = $"SELECT Id, Location.STAsText() AS Location" + - ApiHelper.FromTableIntersectsPolygon(table.Name, polygonWkt); - _logger.LogInformation(sqlQuery); - - housefootprints.AddRange(DbHelper.GetData(sqlQuery)); - } - } - - } - } - double totalBuildingArea = 0.0; - if (housefootprints.Any()) - { - var housefootprintsData = housefootprints.Select(h => new DatasetItem - { - Id = h.ContainsKey("Id") ? h["Id"] : null, - Key = "House footprints", - Value = h.ContainsKey("Location") ? ApiHelper.getArea(GeoReader.Read(h["Location"]) as Polygon).ToString("0.##") + "m²" : "-1", - MapId = "House footprints", - }); - individualData.AddRange(housefootprintsData); - - long totalCountHouseFootprints = housefootprints.Count(); - long totalCountChargingStations = 0l; - foreach (var hdata in housefootprints) - { - - totalBuildingArea += hdata.ContainsKey("Location") ? ApiHelper.getArea(GeoReader.Read(hdata["Location"]) as Polygon) : 0; - - } - if (!onlySinglePoint) - { - generalData.Add(new DatasetItem - { - Id = "Total number of buildings in area", - Key = "Total number of buildings in area", - Value = totalCountHouseFootprints.ToString(), - MapId = "" - }); - } - generalData.Add(new DatasetItem - { - Id = "Total building", - Key = "Total building area", - Value = totalBuildingArea.ToString("0.##") + "m²", - MapId = "" - }); - } - - - ////////////// - if (!onlySinglePoint) - { - WKTReader reader = new WKTReader(); - double totalAreaSearchPolygon = 0.0; - foreach(var polygonWkt in polygons) - { - Geometry geometry = reader.Read(polygonWkt); - if (geometry is Polygon polygon) - totalAreaSearchPolygon += ApiHelper.getArea(polygon); - } - generalData.Insert(0, new DatasetItem - { - Id = "Searched area", - Key = "Searched area", - Value = totalAreaSearchPolygon.ToString("0.##") + "m²", - MapId = "" - }); - generalData.Add(new DatasetItem - { - Id = "Potential Area for geothermal use", - Key = "Potential Area for geothermal use", - Value = Math.Max(totalAreaSearchPolygon - totalBuildingArea, 0).ToString("0.##") + "m²", - MapId = "" - }); - } - - LocationDataResponse locationDataResponse = new() - { - IndividualData = individualData, - GeneralData = generalData - }; - - return Ok(locationDataResponse); - } - catch (Exception ex) + data = LoadLocationHelper.loadLocationDataforPolygons(request); + return Ok(data); + } catch (Exception ex) { - return StatusCode(500, $"Internal server error: {ex.Message}"); + return BadRequest(ex); } } - private IEnumerable> getNclosestObjects(string datasetId, double longitude, double latitude, double radius, int n, IEnumerable columns) - { - string columnsString = string.Join(", ", columns); - - string command = $@" - SELECT TOP {n} - {columnsString}, - Location.STAsText() AS Location - FROM - dbo.{datasetId} - WHERE - geometry::Point({latitude}, {longitude}, 4326).STBuffer({radius}).STIntersects(Location) = 1 - ORDER BY - geometry::Point({latitude}, {longitude}, 4326).STDistance(Location);"; - - var results = DbHelper.GetData(command).ToList(); - - - foreach (var result in results) - { - if (result.ContainsKey("Location")) - { - var wkt = result["Location"]; - var point2_wrong = GeoReader.Read(wkt) as Point; - if (point2_wrong != null) - { - - var distance = ApiHelper.getDistance(longitude, latitude, point2_wrong); - result["Distance"] = distance.ToString("0.##"); - } - else - { - result["Distance"] = "-1"; - } - } - else - { - result["Distance"] = "-1"; - } - } - - return results; - } - - - //private long getObjectCount(string datasetId, ) - private IEnumerable> getMatchingObjects( - string datasetId, - double[] longitude, - double[] latitude, - IEnumerable columns) - { - return new List>(); - } - - private IEnumerable> getMatchingObjects( - string datasetId, - double longitude, - double latitude, - IEnumerable columns) - { - // Convert the columns set to a comma-separated string - string columnsString = string.Join(", ", columns); - - // Define the SQL command with the columns string - string command = $@" - SELECT TOP 1000 - {columnsString}, - Location.STAsText() AS Location - FROM - dbo.{datasetId} - WHERE - Location.STIntersects(geometry::Point({latitude}, {longitude}, 4326)) = 1 - ORDER BY - geometry::Point({latitude}, {longitude}, 4326).STDistance(Location);"; - - _logger.LogInformation(command); - - // Get data from the database - var results = DbHelper.GetData(command).ToList(); - - foreach (var result in results) - { - if (result.ContainsKey("Location")) - { - var wkt = result["Location"]; - var polygon = GeoReader.Read(wkt) as Polygon; - if (polygon != null) - { - result["Area"] = ApiHelper.getArea(polygon).ToString("0.##"); - } - else - result["Area"] = "-1"; - - } - else - result["Area"] = "-1 no location"; - - } - - return results; - } - - public static List> ClusterData(List data, int numberOfClusters) { var centroids = data.Select(d => QueryParameters.CalculateCentroid(d.Coordinates)).ToArray(); @@ -450,74 +181,6 @@ public static List> ClusterData(List data, int nu return clusteredData; } - - /// - /// WIP: DO NOT USE, Get a record - /// - /// - /// - [HttpGet("3/data")] - public ActionResult GetActualUseData([FromQuery] QueryParameters parameters) - { - _logger.LogInformation("Fetching actual use data with parameters: {parameters}", parameters); - try - { - var bottomLat = parameters.BottomLat; - var bottomLong = parameters.BottomLong; - var topLat = parameters.TopLat; - var topLong = parameters.TopLong; - var stateName = parameters.StateName; - - DbHelper.CreateDbConnection(); - - string command = @" - SELECT top 1000 Id, Location.STAsText() AS Location, Location.STGeometryType() AS Type - FROM dbo.actual_use_{4} - WHERE Location.STIntersects(geography::STGeomFromText('POLYGON(( - {0} {1}, - {0} {3}, - {2} {3}, - {2} {1}, - {0} {1} - ))', 4326)) = 1"; - - string formattedQuery = string.Format(command, bottomLong, bottomLat, topLong, topLat, stateName); - List features = new List(); - foreach (var row in DbHelper.GetData(formattedQuery)) - { - var feature = new - { - type = "Feature", - geometry = new - { - type = (string)row["Type"], - coordinates = QueryParameters.GetPolygonCordinates(row["Location"]) - }, - properties = new - { - id = (string)row["Id"] - } - }; - features.Add(feature); - } - - var featureCollection = new - { - type = "FeatureCollection", - features = features - }; - - var jsonResponse = JsonConvert.SerializeObject(featureCollection); - _logger.LogInformation("Actual use data fetched successfully."); - return Ok(jsonResponse); - } - catch (ServiceException se) - { - _logger.LogError(se, "ServiceException occurred while fetching actual use data."); - return BadRequest(se.Message); - } - } - /// /// Get a record /// @@ -768,16 +431,29 @@ public class LocationDataRequest public class LocationDataResponse { + public List SelectionData { get; set; } public List IndividualData { get; set; } - public List GeneralData { get; set; } + } public class DatasetItem { - public string Id { get; set; } + public string DisplayName { get; set; } + public string DatasetId { get; set; } // Optional -> for "open in map" functionality + + public double[] Coordinate { get; set; } + public List PolygonCoordinates { get; set; } + public List Subdata { get; set; } + public string Value { get; set; } // some items may not have subdata and should instead be directly displayed with a value + + } + public class SubdataItem + { public string Key { get; set; } + public string Value { get; set; } - public string MapId { get; set; } // Optional -> for "open in map" functionality + } + } \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs index fcc022ea..4ce27da6 100644 --- a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs +++ b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs @@ -37,7 +37,7 @@ public string GetDataInsideArea(BoundingBox boundingBox) var tableName = mMetadata.additionalData.Tables[0].Name; var query = "SELECT top 1000 operator, Location.AsTextZM() AS Location" + - ApiHelper.FromTableIntersectsPolygon(tableName, polygon); + ApiHelper.FromTableWhereIntersectsPolygon(tableName, polygon); // the list of features from combined datasets. var features = new List>(); diff --git a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs index b7522581..c0eda26b 100644 --- a/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs +++ b/backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/ShapeDatasetHandler.cs @@ -43,9 +43,6 @@ public string GetDataInsideArea(BoundingBox boundingBox) if (!ApiHelper.BoxIntersection(boundingBox, table.BoundingBox.Value)) { - var bb = table.BoundingBox.Value; - // Console.WriteLine($"request-- x: {boundingBox.minX}, y: {boundingBox.minY} || x: {boundingBox.maxX}, y: {boundingBox.maxY}"); - // Console.WriteLine($"x: {bb.minX}, y: {bb.minY} || x: {bb.maxX}, y: {bb.maxY}"); continue; } @@ -53,32 +50,16 @@ public string GetDataInsideArea(BoundingBox boundingBox) // get data // SQL Query to find intersecting points + var sqlQuery = $"SELECT top 1000 Location.AsTextZM() AS Location," + + $" Location.STGeometryType() AS Type" + + (table.RowHeaders.Count > 0 ? "," + string.Join(',', table.RowHeaders) : "") + + ApiHelper.FromTableWhereIntersectsPolygon(table.Name, polygon); - var sqlQuery = $"SELECT top 1000 Location.AsTextZM() AS Location, Location.STGeometryType() AS Type" + - ApiHelper.FromTableIntersectsPolygon(table.Name, polygon); - + Console.WriteLine($"requesting:\n{sqlQuery}"); + // create feature object foreach (var row in DbHelper.GetData(sqlQuery)) { - var feature = new Dictionary - { - { "type", "Feature" }, - { - "geometry", new Dictionary - { - { "type", $"{row["Type"]}" }, - { - "coordinates", ApiHelper.GetCoordinatesFromPolygon(row["Location"]) - } - } - }, - { - "properties", new Dictionary - { - { "text", $"{row["Type"]}" } - } - } - }; - + var feature = GetFeatureFromRow(row, table); features.Add(feature); } } @@ -92,4 +73,35 @@ public string GetDataInsideArea(BoundingBox boundingBox) return JsonSerializer.Serialize(responseObj); } + + /// + /// Get the GeoJson Feature Object corresponding to the object + /// + /// + /// + public Dictionary GetFeatureFromRow(Dictionary row, MetadataObject.TableData table) + { + var properties = new Dictionary(); + foreach (var header in table.RowHeaders) + { + properties[header] = row[header]; + } + + return new Dictionary + { + { "type", "Feature" }, + { + "geometry", new Dictionary + { + { "type", $"{row["Type"]}" }, + { + "coordinates", ApiHelper.GetCoordinatesFromPolygon(row["Location"]) + } + } + }, + { + "properties", properties + } + }; + } } \ No newline at end of file diff --git a/backend/src/BIE.Core/BIE.Core.API/LoadLocationHelper.cs b/backend/src/BIE.Core/BIE.Core.API/LoadLocationHelper.cs new file mode 100644 index 00000000..eedc0724 --- /dev/null +++ b/backend/src/BIE.Core/BIE.Core.API/LoadLocationHelper.cs @@ -0,0 +1,334 @@ +using BIE.Core.API.Controllers; +using BIE.Core.DBRepository; +using NetTopologySuite.Geometries; +using NetTopologySuite.IO; +using System.Collections.Generic; +using System; +using LinqStatistics; +using BieMetadata; +using System.Linq; +using NetTopologySuite; + + +namespace BIE.Core.API +{ + public class LoadLocationHelper + { + private static MetadataDbHelper mMetadataDbHelper; + private static readonly WKTReader GeoReader = new(new NtsGeometryServices(new PrecisionModel(), 4326)); + private static readonly GeometryFactory GeometryFactory = new(new PrecisionModel(), 4326); + + + private static MetadataDbHelper MetadataDbHelper + { + get + { + if (mMetadataDbHelper != null) + { + return mMetadataDbHelper; + } + + mMetadataDbHelper = new MetadataDbHelper(); + if (mMetadataDbHelper.CreateConnection()) + { + return mMetadataDbHelper; + } + + Console.WriteLine("could not establish Metadata-database Connection"); + throw new Exception("no metadata DB reachable!"); + } + } + + public static LocationDataResponse loadLocationDataforPolygons(LocationDataRequest request) + { + if (request.Location.Count < 1 || request.Location.First().Count <= 1) + throw new ArgumentException("Polygon data cannot be retrieved with the amount of request objects"); + + var generalData = new List(); + var individualData = new List(); + + var polygons = ApiHelper.ConvertToWktPolygons(request.Location); + WKTReader reader = new WKTReader(); + double totalAreaSearchPolygon = 0.0; + foreach (var polygonWkt in polygons) + { + Geometry geometry = reader.Read(polygonWkt); + if (geometry is Polygon polygon) + totalAreaSearchPolygon += ApiHelper.getArea(polygon); + } + generalData.Insert(0, new DatasetItem + { + DisplayName = "Searched area", + Value = totalAreaSearchPolygon.ToString("0.##") + "m²" + }); + + var housefootprints = new List>(); + foreach (var polygonWkt in polygons) + { + // then go through house footprints + var metadata = MetadataDbHelper.GetMetadata("house_footprints"); + if (metadata != null) + { + foreach (var table in metadata.additionalData.Tables) + { + var sqlQuery = $"SELECT Id, Location.STAsText() AS Location" + + ApiHelper.FromTableWhereIntersectsPolygon(table.Name, polygonWkt); + housefootprints.AddRange(DbHelper.GetData(sqlQuery)); + } + } + + } + + long totalCountHouseFootprints = housefootprints.Count(); + double totalBuildingArea = 0.0; + List totalBuildingAreas = new List(); + foreach (var h in housefootprints) + { + Polygon p = GeoReader.Read(h["Location"]) as Polygon; + double area = ApiHelper.getArea(p); + + totalBuildingAreas.Add(area); + totalBuildingArea += area; + + var item = new DatasetItem + { + DisplayName = "House footprint: " + (h.ContainsKey("Id") ? h["Id"] : ""), + DatasetId = "House_footprints", + Value = area.ToString("0.##") + "m²", + Coordinate = new double[] { p.Centroid.X, p.Centroid.Y }, + Subdata = new List() + }; + individualData.Add(item); + } + + generalData.Add(new DatasetItem + { + DisplayName = "Potential Area for geothermal use", + Value = Math.Max(totalAreaSearchPolygon - totalBuildingArea, 0).ToString("0.##") + "m²", + }); + generalData.Add(new DatasetItem + { + DisplayName = "Total number of buildings in area", + Value = totalCountHouseFootprints.ToString() + }); + generalData.Add(new DatasetItem + { + DisplayName = "Total building area", + Value = totalBuildingArea.ToString("0.##") + "m²", + Subdata = new List + { + new() { Key = "Average", Value = totalBuildingAreas.Average().ToString() }, + new() { Key = "Median", Value = totalBuildingAreas.Median().ToString() }, + new() { Key = "Variance", Value = totalBuildingAreas.Variance().ToString() } + } + }); + + LocationDataResponse locationDataResponse = new() + { + SelectionData = generalData, + IndividualData = individualData.Take(500).ToList() + + }; + + return locationDataResponse; + } + + public static LocationDataResponse loadLocationDataForSinglePoint(LocationDataRequest request) + { + var generalData = new List(); + + + if (request.Location.Count != 1 || request.Location.First().Count != 1) + throw new ArgumentException("Single point location data can't be retrieved with the amount of request objects"); + + var p = request.Location.First().First(); + Point pointForCalculations = new Point(p.ElementAt(0), p.ElementAt(1)); + + // closest charging stations and number of charging stations + var radius = 10000000; // Define the radius as needed + var columns = new HashSet { "operator", "Id", "rated_power_kw" }; + var metadata = MetadataDbHelper.GetMetadata("EV_charging_stations"); + if (metadata != null) + { + var tables = metadata.additionalData.Tables; + if (tables.Count == 1) + { + var chargingStations = getNclosestObjects(tables[0].Name, pointForCalculations.Y, pointForCalculations.X, radius, 3, columns); + + var chargingStationsData = chargingStations.Select(h => new DatasetItem + { + DisplayName = "Charging station: " + (h.ContainsKey("operator") ? h["operator"] : "No operator defined"), + Value = (h.ContainsKey("Distance") ? h["Distance"] + "m" : "-1") + " | " + (h.ContainsKey("rated_power_kw") ? h["rated_power_kw"] + "kw" : "-1"), + DatasetId = "EV_charging_stations", + }); + + generalData.AddRange(chargingStationsData); + } + } + + + var housefootprints = new List>(); + + // Additional section for house footprints + columns = new HashSet { "Id" }; + metadata = MetadataDbHelper.GetMetadata("house_footprints"); + if (metadata != null) + { + foreach (var table in metadata.additionalData.Tables) + housefootprints.AddRange(getMatchingObjects(table.Name, pointForCalculations.Y, pointForCalculations.X, columns)); + } + var housefootprintsData = housefootprints.Select(h => new DatasetItem + { + DisplayName = "House footprint" + (h.ContainsKey("Id") ? h["Id"] : ""), + Value = h.ContainsKey("Location") ? ApiHelper.getArea(GeoReader.Read(h["Location"]) as Polygon).ToString("0.##") + "m²" : "-1", + DatasetId = "House_footprints", + }); + generalData.AddRange(housefootprintsData); + + LocationDataResponse locationDataResponse = new() + { + SelectionData = generalData + }; + return locationDataResponse; + } + + private static IEnumerable> getMatchingObjects( + string datasetId, + double longitude, + double latitude, + IEnumerable columns) + { + // Convert the columns set to a comma-separated string + string columnsString = string.Join(", ", columns); + + // Define the SQL command with the columns string + string command = $@" + SELECT TOP 1000 + {columnsString}, + Location.STAsText() AS Location + FROM + dbo.{datasetId} + WHERE + Location.STIntersects(geometry::Point({latitude}, {longitude}, 4326)) = 1 + ORDER BY + geometry::Point({latitude}, {longitude}, 4326).STDistance(Location);"; + + // Get data from the database + var results = DbHelper.GetData(command).ToList(); + + foreach (var result in results) + { + if (result.ContainsKey("Location")) + { + var wkt = result["Location"]; + var polygon = GeoReader.Read(wkt) as Polygon; + if (polygon != null) + { + result["Area"] = ApiHelper.getArea(polygon).ToString("0.##"); + } + else + result["Area"] = "-1"; + + } + else + result["Area"] = "-1 no location"; + + } + + return results; + } + + private static IEnumerable> getNclosestObjects(string datasetId, double longitude, double latitude, double radius, int n, IEnumerable columns) + { + string columnsString = string.Join(", ", columns); + + string command = $@" + SELECT TOP {n} + {columnsString}, + Location.STAsText() AS Location + FROM + dbo.{datasetId} + WHERE + geometry::Point({latitude}, {longitude}, 4326).STBuffer({radius}).STIntersects(Location) = 1 + ORDER BY + geometry::Point({latitude}, {longitude}, 4326).STDistance(Location);"; + + var results = DbHelper.GetData(command).ToList(); + + + foreach (var result in results) + { + if (result.ContainsKey("Location")) + { + var wkt = result["Location"]; + var point2_wrong = GeoReader.Read(wkt) as Point; + if (point2_wrong != null) + { + + var distance = ApiHelper.getDistance(longitude, latitude, point2_wrong); + result["Distance"] = distance.ToString("0.##"); + } + else + { + result["Distance"] = "-1"; + } + } + else + { + result["Distance"] = "-1"; + } + } + + return results; + } + + private IEnumerable> GetMatchingObjects( + string datasetId, + double longitude, + double latitude, + IEnumerable columns) + { + // Convert the columns set to a comma-separated string + string columnsString = string.Join(", ", columns); + + // Define the SQL command with the columns string + string command = $@" + SELECT TOP 1000 + {columnsString}, + Location.STAsText() AS Location + FROM + dbo.{datasetId} + WHERE + Location.STIntersects(geometry::Point({latitude}, {longitude}, 4326)) = 1 + ORDER BY + geometry::Point({latitude}, {longitude}, 4326).STDistance(Location);"; + + + // Get data from the database + var results = DbHelper.GetData(command).ToList(); + + foreach (var result in results) + { + if (result.ContainsKey("Location")) + { + var wkt = result["Location"]; + var polygon = GeoReader.Read(wkt) as Polygon; + if (polygon != null) + { + result["Area"] = ApiHelper.getArea(polygon).ToString("0.##"); + } + else + result["Area"] = "-1"; + + } + else + result["Area"] = "-1 no location"; + + } + + return results; + } + } + + +} diff --git a/backend/src/BIE.Core/BIE.Core.API/Properties/launchSettings.json b/backend/src/BIE.Core/BIE.Core.API/Properties/launchSettings.json index 914174b4..df7c7bfb 100644 --- a/backend/src/BIE.Core/BIE.Core.API/Properties/launchSettings.json +++ b/backend/src/BIE.Core/BIE.Core.API/Properties/launchSettings.json @@ -8,7 +8,7 @@ "DB_NAME": "BIEDB", "DB_PASSWORD": "MyPass@1234", "DB_TYPE": "SQL", - "TRUSTED": "False", + "TRUSTED": "True", "DB_USERNAME": "db_user1", "METADATA_DB_URL": "localhost:27017" }, diff --git a/backend/src/BIE.DataPipeline/DbHelper.cs b/backend/src/BIE.DataPipeline/DbHelper.cs index c2d38601..b7b42445 100644 --- a/backend/src/BIE.DataPipeline/DbHelper.cs +++ b/backend/src/BIE.DataPipeline/DbHelper.cs @@ -128,6 +128,7 @@ FROM sys.columns c WHERE t.name = '" + description.table_name + "' AND c.name = 'Location' AND ty.name = 'geometry';"; var db = Database.Instance; + using (var command = db.CreateCommand(query)) { var (reader, connection) = db.ExecuteReader(command); @@ -305,7 +306,7 @@ IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{desc BEGIN CREATE TABLE {description.table_name} ( Id INT PRIMARY KEY IDENTITY(1,1), - Location GEOGRAPHY, + Location Geometry, XmlData XML, GroundHeight FLOAT, DistrictKey VARCHAR(255), diff --git a/backend/src/BIE.DataPipeline/Import/CityGmlImporter.cs b/backend/src/BIE.DataPipeline/Import/CityGmlImporter.cs index ac926b9b..f41e4ff6 100644 --- a/backend/src/BIE.DataPipeline/Import/CityGmlImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/CityGmlImporter.cs @@ -126,7 +126,7 @@ public bool ReadLine(out string nextLine) float livingArea = GetLivingArea(buildingNode, groundArea, buildingWallHeight); float roofArea = GetRoofArea(buildingNode); - nextLine = $"geography::STGeomFromText('{geometry.AsText()}', 4326)"; + nextLine = $"geometry::STGeomFromText('{geometry.AsText()}', 4326)"; nextLine += string.Format(",'{0}'", buildingNode.InnerXml); nextLine += string.Format(",'{0}'", groundHeight.ToString(culture)); nextLine += string.Format(",'{0}'", districtKey); @@ -156,6 +156,11 @@ public string GetInsertHeader() return ""; } + public IEnumerable GetHeaders() + { + return new List(); + } + private Geometry UtmCoordinatesToGeometry(string utmCoordinates) { //Console.WriteLine(utmCoordinates); diff --git a/backend/src/BIE.DataPipeline/Import/CsvImporter.cs b/backend/src/BIE.DataPipeline/Import/CsvImporter.cs index af3f5ccd..1a550464 100644 --- a/backend/src/BIE.DataPipeline/Import/CsvImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/CsvImporter.cs @@ -206,6 +206,11 @@ public string GetInsertHeader() return headerString; } + public IEnumerable GetHeaders() + { + return headerString.Split(",").Select(s => s.Trim()); + } + private static string RemoveLastComma(string input) { diff --git a/backend/src/BIE.DataPipeline/Import/IImporter.cs b/backend/src/BIE.DataPipeline/Import/IImporter.cs index d5cb9435..b2b21fd0 100644 --- a/backend/src/BIE.DataPipeline/Import/IImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/IImporter.cs @@ -13,5 +13,7 @@ internal interface IImporter public string GetCreationHeader(); public string GetInsertHeader(); + + public IEnumerable GetHeaders(); } } diff --git a/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs b/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs index c6adb1bd..83bedf95 100644 --- a/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs +++ b/backend/src/BIE.DataPipeline/Import/ShapeImporter.cs @@ -16,12 +16,14 @@ internal class ShapeImporter : IImporter private DbaseFileHeader mHeader; private readonly ICoordinateTransformation? mTransformation; + private StringBuilder mStringBuilder; + public ShapeImporter(DataSourceDescription? dataSourceDescription) { // YAML Arguments: - this.mDataSourceDescription = dataSourceDescription; + mDataSourceDescription = dataSourceDescription; - new StringBuilder(); + mStringBuilder = new StringBuilder(); SetupParser(); @@ -71,20 +73,16 @@ private void SetupParser() shpStream.Position = 0; dbfStream.Position = 0; - mParser = Shapefile.CreateDataReader( - new ShapefileStreamProviderRegistry( - new ByteStreamProvider(StreamTypes - .Shape, - shpStream), - new ByteStreamProvider(StreamTypes - .Data, - dbfStream), - true, - true), - GeometryFactory.Default); + var shpByteStream = new ByteStreamProvider(StreamTypes.Shape, shpStream); + var dbfByteStream = new ByteStreamProvider(StreamTypes.Data, dbfStream); + var providerRegistry = new ShapefileStreamProviderRegistry(shpByteStream, + dbfByteStream, + true, + true); + + mParser = Shapefile.CreateDataReader(providerRegistry, GeometryFactory.Default); mHeader = mParser.DbaseHeader; - } private void ExtractShapeFilesFromZip(ZipArchive zipArchive, MemoryStream shpStream, MemoryStream dbfStream) @@ -118,6 +116,7 @@ private void ExtractShapeFilesFromZip(ZipArchive zipArchive, MemoryStream shpStr /// public bool ReadLine(out string nextLine) { + mStringBuilder.Clear(); nextLine = ""; if (!mParser.Read()) { @@ -128,27 +127,29 @@ public bool ReadLine(out string nextLine) // Append geometry as WKT (Well-Known Text) var geometry = mParser.Geometry; geometry = ConvertUtmToLatLong(geometry); - - - // for (int i = 1; i < mHeader.Fields.Length; i++) - // { - // Console.Write($" {mParser.GetValue(i)};"); - // } - // Console.WriteLine(); - nextLine = $"GEOMETRY::STGeomFromText('{geometry.AsText()}', 4326)"; + mStringBuilder.Append($"GEOMETRY::STGeomFromText('"); + mStringBuilder.Append(geometry.AsText()); + mStringBuilder.Append("', 4326)"); // nextLine = $"GEOMETRY::STGeomFromText('POLYGON (11.060226859896797 49.496927347229494, 11.060276626123832 49.49695803564076)')', 4326)"; for (int i = 1; i < mHeader.Fields.Length + 1; i++) { - var value = (string)mParser.GetValue(i); - nextLine += $", \'{(value != "" ? value : "null")}\'"; + var value = mParser.GetValue(i); + mStringBuilder.Append(", '"); + var bytes = Encoding.GetEncoding("ISO-8859-1").GetBytes(value?.ToString() ?? ""); + mStringBuilder.Append(Encoding.UTF8.GetString(bytes)); + mStringBuilder.Append("'"); + // nextLine += $", \'{(value != "" ? value : "null")}\'"; } - nextLine += $", {CalculateAreaInSquareMeters(geometry)}"; + mStringBuilder.Append($",{CalculateAreaInSquareMeters(geometry)}"); + // nextLine += $",{CalculateAreaInSquareMeters(geometry)}"; + nextLine = mStringBuilder.ToString(); return true; } + /// /// Calculates Area of Polygon /// @@ -169,7 +170,8 @@ private double CalculateAreaInSquareMeters(Geometry geometry) for (int i = 0; i < coordinates.Length; i++) { - double[] transformed = transform.MathTransform.Transform(new double[] { coordinates[i].X, coordinates[i].Y }); + double[] transformed = + transform.MathTransform.Transform(new double[] { coordinates[i].X, coordinates[i].Y }); transformedCoordinates[i] = new Coordinate(transformed[0], transformed[1]); } @@ -196,7 +198,8 @@ private double CalculateAreaInSquareMeters(Geometry geometry) /// public string GetCreationHeader() { - return mHeader.Fields.Aggregate("Location GEOMETRY", (current, field) => current + $", {field.Name} VARCHAR(255)"); + return mHeader.Fields.Aggregate("Location GEOMETRY", + (current, field) => current + $", {field.Name} NVARCHAR(255)"); } /// @@ -207,7 +210,12 @@ public string GetInsertHeader() { return mHeader.Fields.Aggregate("Location", (current, field) => current + $", {field.Name}"); } - + + public IEnumerable GetHeaders() + { + return mHeader.Fields.Select(field => field.Name).Append("Area"); + } + /// /// Conver UTM coordinates to Latitude and Longitude /// diff --git a/backend/src/BIE.DataPipeline/Program.cs b/backend/src/BIE.DataPipeline/Program.cs index c385f60b..1a781ab7 100644 --- a/backend/src/BIE.DataPipeline/Program.cs +++ b/backend/src/BIE.DataPipeline/Program.cs @@ -77,13 +77,13 @@ case "SHAPE": importer = new ShapeImporter(description); - dbHelper.SetInfo(description.table_name, importer.GetInsertHeader() + ",Area"); - break; + case "CITYGML": importer = new CityGmlImporter(description, dbHelper); - dbHelper.SetInfo(description.table_name, "Location, XmlData, GroundHeight, DistrictKey, CheckDate, GroundArea, BuildingWallHeight, LivingArea, RoofArea"); + dbHelper.SetInfo(description.table_name, + "Location, XmlData, GroundHeight, DistrictKey, CheckDate, GroundArea, BuildingWallHeight, LivingArea, RoofArea"); break; default: @@ -136,8 +136,19 @@ } Console.WriteLine("Updating the metadata..."); + var boundingBox = dbHelper.GetBoundingBox(description.table_name); - if (!metadataDbHelper.UpdateMetadata(description.dataset, description.table_name, count, boundingBox)) + var rowHeaders = importer.GetHeaders(); + + var tableData = new MetadataObject.TableData() + { + Name = description.table_name, + NumberOfLines = count, + BoundingBox = boundingBox, + RowHeaders = rowHeaders.ToList() + }; + + if (!metadataDbHelper.UpdateMetadata(description.dataset, tableData)) { return 1; }