Merge pull request #406 from KNMI/get-layer-metadata

Get layer metadata

adagucserverEC/CRequest.cpp

@@ -42,6 +42,10 @@
 #include <CReadFile.h>
 #include "Definitions.h"
 #include "utils/LayerUtils.h"
+#include "utils/XMLGenUtils.h"
+#include "utils/LayerMetadataStore.h"
+#include <json_adaguc.h>
+#include "utils/LayerMetadataToJson.h"
 
 const char *CRequest::className = "CRequest";
 int CRequest::CGI = 0;
@@ -2144,6 +2148,7 @@ int CRequest::process_all_layers() {
         CT::string dumpString = CDF::dump(dataSources[j]->getDataObject(0)->cdfObject);
         printf("%s", dumpString.c_str());
         reader.close();
+        return 0;
       }
 
       if (srvParam->requestType == REQUEST_WMS_GETREFERENCETIMES) {
@@ -3199,6 +3204,16 @@ int CRequest::process_querystring() {
         CDBError("ADAGUC Server: GetMetaData is restricted");
         return 1;
       }
+
+      if (srvParam->Format.equals("application/json")) {
+        // GetMetadata for specific dataset and layer
+        json result;
+        getLayerMetadataAsJson(srvParam, result);
+        printf("%s%c%c\n", "Content-Type:application/json", 13, 10);
+        printf("%s", result.dump().c_str());
+        return 0;
+      }
+
       if (dFound_WMSLAYER == 0) {
         CDBError("ADAGUC Server: Parameter LAYER missing");
         dErrorOccured = 1;

adagucserverEC/CXMLGen.cpp

@@ -100,8 +100,8 @@ int CXMLGen::getWMS_1_0_0_Capabilities(CT::string *XMLDoc, std::vector<MetadataL
         // Dims
         for (auto dim : layer->layerMetadata.dimList) {
           if (dim.hidden) continue;
-          XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\"/>\n", dim.name.c_str(), dim.units.c_str());
-          XMLDoc->printconcat("<Extent name=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\">", dim.name.c_str(), dim.defaultValue.c_str(), 1);
+          XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\"/>\n", dim.serviceName.c_str(), dim.units.c_str());
+          XMLDoc->printconcat("<Extent name=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\">", dim.serviceName.c_str(), dim.defaultValue.c_str(), 1);
           XMLDoc->concat(dim.values.c_str());
           XMLDoc->concat("</Extent>\n");
         }
@@ -237,8 +237,8 @@ int CXMLGen::getWMS_1_1_1_Capabilities(CT::string *XMLDoc, std::vector<MetadataL
             // Dims
             for (auto dim : layer->layerMetadata.dimList) {
               if (dim.hidden) continue;
-              XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\"/>\n", dim.name.c_str(), dim.units.c_str());
-              XMLDoc->printconcat("<Extent name=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\">", dim.name.c_str(), dim.defaultValue.c_str(), 1);
+              XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\"/>\n", dim.serviceName.c_str(), dim.units.c_str());
+              XMLDoc->printconcat("<Extent name=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\">", dim.serviceName.c_str(), dim.defaultValue.c_str(), 1);
               XMLDoc->concat(dim.values.c_str());
               XMLDoc->concat("</Extent>\n");
             }
@@ -649,11 +649,12 @@ int CXMLGen::getWMS_1_3_0_Capabilities(CT::string *XMLDoc, std::vector<MetadataL
             // Dims
             for (auto dim : layer->layerMetadata.dimList) {
               if (dim.hidden) continue;
-              if (dim.name.indexOf("time") != -1) {
-                XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\" current=\"1\">", dim.name.c_str(), dim.units.c_str(),
+              if (dim.serviceName.indexOf("time") != -1) {
+                XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\" current=\"1\">", dim.serviceName.c_str(), dim.units.c_str(),
                                     dim.defaultValue.c_str(), 1);
               } else {
-                XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\" >", dim.name.c_str(), dim.units.c_str(), dim.defaultValue.c_str(), 1);
+                XMLDoc->printconcat("<Dimension name=\"%s\" units=\"%s\" default=\"%s\" multipleValues=\"%d\" nearestValue=\"0\" >", dim.serviceName.c_str(), dim.units.c_str(),
+                                    dim.defaultValue.c_str(), 1);
               }
               XMLDoc->concat(dim.values.c_str());
               XMLDoc->concat("</Dimension>\n");

adagucserverEC/LayerTypeLiveUpdate/LayerTypeLiveUpdate.cpp

@@ -54,7 +54,8 @@ int layerTypeLiveUpdateConfigureWMSLayerForGetCapabilities(MetadataLayer *metada
   CT::string startTime = timeInstance.dateToISOString(timeInstance.offsetToDate(startTimeOffset));
   CT::string stopTime = timeInstance.dateToISOString(timeInstance.offsetToDate(stopTimeOffset));
   CT::string resTime = "PT1S";
-  LayerMetadataDim dim = {.name = "time", .units = "ISO8601", .values = startTime + "/" + stopTime + "/" + resTime, .defaultValue = stopTime, .hasMultipleValues = true, .hidden = false};
+  LayerMetadataDim dim = {
+      .serviceName = "time", .cdfName = "time", .units = "ISO8601", .values = startTime + "/" + stopTime + "/" + resTime, .defaultValue = stopTime, .hasMultipleValues = true, .hidden = false};
   metadataLayer->layerMetadata.dimList.push_back(dim);
 
   return 0;

adagucserverEC/Types/LayerMetadataType.h

@@ -6,7 +6,8 @@
 #include <CDataSource.h>
 
 struct LayerMetadataDim {
-  CT::string name;
+  CT::string serviceName;
+  CT::string cdfName;
   CT::string units;
   CT::string values;
   CT::string defaultValue;
@@ -15,7 +16,7 @@ struct LayerMetadataDim {
 };
 
 struct LayerMetadataProjection {
-  LayerMetadataProjection(const CT::string& name, const double bbox[]) {
+  LayerMetadataProjection(const CT::string &name, const double bbox[]) {
     this->name = name;
     for (size_t j = 0; j < 4; j++) {
       this->dfBBOX[j] = bbox[j];

adagucserverEC/utils/LayerMetadataStore.cpp

@@ -12,10 +12,11 @@ int getDimensionListAsJson(MetadataLayer *metadataLayer, json &dimListJson) {
       item["defaultValue"] = dimension.defaultValue.c_str();
       item["hasMultipleValues"] = dimension.hasMultipleValues;
       item["hidden"] = dimension.hidden;
-      item["name"] = dimension.name.c_str();
+      item["serviceName"] = dimension.serviceName.c_str();
+      item["cdfName"] = dimension.cdfName.c_str();
       item["units"] = dimension.units.c_str();
       item["values"] = dimension.values.c_str();
-      dimListJson[dimension.name.c_str()] = item;
+      dimListJson[dimension.serviceName.c_str()] = item;
     }
   } catch (json::exception &e) {
     CDBWarning("Unable to build json structure");
@@ -26,7 +27,7 @@ int getDimensionListAsJson(MetadataLayer *metadataLayer, json &dimListJson) {
 
 int getLayerBaseMetadataAsJson(MetadataLayer *metadataLayer, json &layerMetadataItem) {
   try {
-    layerMetadataItem["name"] = metadataLayer->layerMetadata.name;
+    layerMetadataItem["layername"] = metadataLayer->layerMetadata.name;
     layerMetadataItem["title"] = metadataLayer->layerMetadata.title;
     layerMetadataItem["group"] = metadataLayer->layerMetadata.group;
     layerMetadataItem["abstract"] = metadataLayer->layerMetadata.abstract;
@@ -181,7 +182,7 @@ int loadLayerMetadataStructFromMetadataDb(MetadataLayer *metadataLayer) {
     }
     json a;
     auto i = a.parse(layerMetadataAsJson.c_str());
-    metadataLayer->layerMetadata.name = i["name"].get<std::string>().c_str();
+    metadataLayer->layerMetadata.name = i["layername"].get<std::string>().c_str();
     metadataLayer->layerMetadata.title = i["title"].get<std::string>().c_str();
     metadataLayer->layerMetadata.group = i["group"].get<std::string>().c_str();
     metadataLayer->layerMetadata.abstract = i["abstract"].get<std::string>().c_str();
@@ -366,7 +367,8 @@ int loadLayerDimensionListFromMetadataDb(MetadataLayer *metadataLayer) {
       auto dimensionProperties = d.value();
 
       LayerMetadataDim dimension = {
-          .name = dimensionProperties["name"].get<std::string>().c_str(),
+          .serviceName = dimensionProperties["serviceName"].get<std::string>().c_str(),
+          .cdfName = dimensionProperties["cdfName"].get<std::string>().c_str(),
           .units = dimensionProperties["units"].get<std::string>().c_str(),
           .values = dimensionProperties["values"].get<std::string>().c_str(),
           .defaultValue = dimensionProperties["defaultValue"].get<std::string>().c_str(),

adagucserverEC/utils/XMLGenUtils.cpp

@@ -127,11 +127,11 @@ int populateMetadataLayerStruct(MetadataLayer *metadataLayer, bool readFromDB) {
 
   std::map<std::string, LayerMetadataProjection> projectionMap;
   // Make a unique list of projections
-  for (const auto& p : metadataLayer->layerMetadata.projectionList) {
+  for (const auto &p : metadataLayer->layerMetadata.projectionList) {
     projectionMap.emplace(p.name.c_str(), p);
   }
   metadataLayer->layerMetadata.projectionList.clear();
-  for (const auto& p : projectionMap) {
+  for (const auto &p : projectionMap) {
     metadataLayer->layerMetadata.projectionList.push_back(p.second);
   }
 
@@ -169,7 +169,8 @@ int getDimsForLayer(MetadataLayer *metadataLayer) {
         CT::string fileDate = CDirReader::getFileDate(metadataLayer->layer->FilePath[0]->value.c_str());
 
         LayerMetadataDim dim;
-        dim.name.copy("time");
+        dim.serviceName.copy("time");
+        dim.cdfName.copy("time");
         dim.units.copy("ISO8601");
         dim.values.copy(fileDate.c_str());
         dim.defaultValue.copy(fileDate.c_str());
@@ -188,7 +189,8 @@ int getDimsForLayer(MetadataLayer *metadataLayer) {
       // Create a new dim to store in the layer
       LayerMetadataDim dim;
       dim.hidden = false;
-      dim.name.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+      dim.serviceName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+      dim.cdfName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.name.c_str());
 
       // Get the tablename
       CT::string tableName;
@@ -404,14 +406,15 @@ int getDimsForLayer(MetadataLayer *metadataLayer) {
           // if(srvParam->requestType==REQUEST_WMS_GETCAPABILITIES)
           {
 
-            dim.name.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+            dim.serviceName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+            dim.cdfName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.name.c_str());
 
             // Try to get units from the variable
             dim.units.copy("NA");
             if (metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.units.empty()) {
               CT::string units;
               try {
-                metadataLayer->dataSource->getDataObject(0)->cdfObject->getVariable(dim.name.c_str())->getAttribute("units")->getDataAsString(&units);
+                metadataLayer->dataSource->getDataObject(0)->cdfObject->getVariable(dim.cdfName.c_str())->getAttribute("units")->getDataAsString(&units);
                 dim.units.copy(&units);
               } catch (int e) {
               }
@@ -518,7 +521,8 @@ int getDimsForLayer(MetadataLayer *metadataLayer) {
           if (metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.units.empty() == false) {
             dimUnits.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.units.c_str());
           }
-          dim.name.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+          dim.serviceName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->value.c_str());
+          dim.cdfName.copy(metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.name.c_str());
           dim.units.copy(dimUnits.c_str());
           dim.hasMultipleValues = 0;
           // metadataLayer->dataSource->cfgLayer->Dimension[i]->attr.defaultV.c_str()
@@ -689,7 +693,7 @@ int getStylesForLayer(MetadataLayer *metadataLayer) {
 bool compareStringCase(const std::string &s1, const std::string &s2) { return strcmp(s1.c_str(), s2.c_str()) < 0; }
 
 bool compareProjection(const LayerMetadataProjection &p1, const LayerMetadataProjection &p2) { return strcmp(p1.name.c_str(), p2.name.c_str()) < 0; }
-bool compareDim(const LayerMetadataDim &p2, const LayerMetadataDim &p1) { return strcmp(p1.name.c_str(), p2.name.c_str()) < 0; }
+bool compareDim(const LayerMetadataDim &p2, const LayerMetadataDim &p1) { return strcmp(p1.serviceName.c_str(), p2.serviceName.c_str()) < 0; }
 
 int getTitleForLayer(MetadataLayer *metadataLayer) {
 #ifdef CXMLGEN_DEBUG

python/lib/adaguc/runAdaguc.py

@@ -2,7 +2,6 @@
 import os
 from PIL import Image
 from io import BytesIO
-import brotli
 import shutil
 import random
 import string
@@ -314,7 +313,7 @@ async def response_to_cache(redis_pool, key, headers: str, data):
             entrytime
             + f"{len(cacheable_headers_json):06d}".encode("utf-8")
             + cacheable_headers_json
-            + brotli.compress(data.getvalue()),
+            + data.getvalue(),
             ex=ttl,
         )
         await redis_client.aclose()
@@ -335,5 +334,5 @@ async def get_cached_response(redis_pool, key):
     headers = json.loads(cached[16 : 16 + headers_len].decode("utf-8"))
     headers.append(f"age: {age}")
 
-    data = brotli.decompress(cached[16 + headers_len :])
+    data = cached[16 + headers_len :]
     return age, headers, BytesIO(data)

python/python_fastapi_server/routers/caching_middleware.py

@@ -10,7 +10,6 @@
 import redis.asyncio as redis  # This can also be used to connect to a Redis cluster
 
 import json
-import brotli
 
 ADAGUC_REDIS = os.environ.get("ADAGUC_REDIS")
 
@@ -32,7 +31,7 @@ async def get_cached_response(redis_pool, request):
     headers_len = int(cached[10:16].decode("utf-8"))
     headers = json.loads(cached[16 : 16 + headers_len].decode("utf-8"))
 
-    data = brotli.decompress(cached[16 + headers_len :])
+    data = cached[16 + headers_len :]
     return age, headers, data
 
 
@@ -51,15 +50,14 @@ async def response_to_cache(redis_pool, request, headers, data, ex: int):
     entrytime = f"{calendar.timegm(datetime.utcnow().utctimetuple()):10d}".encode(
         "utf-8"
     )
-    compressed_data = brotli.compress(data)
-    if len(compressed_data) < MAX_SIZE_FOR_CACHING:
+    if len(data) < MAX_SIZE_FOR_CACHING:
         redis_client = redis.Redis(connection_pool=redis_pool)
         await redis_client.set(
             key,
             entrytime
             + f"{len(headers_json):06d}".encode("utf-8")
             + headers_json
-            + brotli.compress(data),
+            + data,
             ex=ex,
         )
         await redis_client.aclose()

tests/AdagucTests/TestMetadataRequest.py

@@ -0,0 +1,56 @@
+# pylint: disable=invalid-name,missing-function-docstring
+"""
+This class contains tests to test the adaguc-server binary executable file. This is similar to black box testing, it tests the behaviour of the server software. It configures the server and checks if the response is OK.
+"""
+import json
+import os
+import os.path
+import unittest
+import sys
+from adaguc.AdagucTestTools import AdagucTestTools
+
+ADAGUC_PATH = os.environ["ADAGUC_PATH"]
+
+
+class TestMetadataRequest(unittest.TestCase):
+    """
+    TestMetadataRequest class to thest Web Map Service behaviour of adaguc-server.
+    """
+
+    testresultspath = "testresults/TestMetadataRequest/"
+    expectedoutputsspath = "expectedoutputs/TestMetadataRequest/"
+    env = {"ADAGUC_CONFIG": ADAGUC_PATH + "/data/config/adaguc.autoresource.xml"}
+
+    AdagucTestTools().mkdir_p(testresultspath)
+
+    def test_timeseries_adaguc_tests_arcus_uwcw_air_temperature_hagl(self):
+        AdagucTestTools().cleanTempDir()
+        if os.getenv("ADAGUC_DB").endswith(".db"):
+            print("SKIP: Only PSQL")
+            return
+        config = (
+            ADAGUC_PATH
+            + "/data/config/adaguc.tests.dataset.xml,"
+            + ADAGUC_PATH
+            + "/data/config/datasets/adaguc.tests.arcus_uwcw.xml"
+        )
+        # pylint: disable=unused-variable
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            args=["--updatedb", "--config", config], env=self.env, isCGI=False
+        )
+        self.assertEqual(status, 0)
+
+        filename = "test_GetMetadataRequest_arcus_uwcw.json"
+
+        status, data, headers = AdagucTestTools().runADAGUCServer(
+            "dataset=adaguc.tests.arcus_uwcw&service=WMS&request=GetMetadata&format=application/json",
+            {"ADAGUC_CONFIG": ADAGUC_PATH + "/data/config/adaguc.tests.dataset.xml"},
+        )
+        AdagucTestTools().writetofile(self.testresultspath + filename, data.getvalue())
+        self.assertEqual(status, 0)
+        self.assertEqual(
+            data.getvalue(),
+            AdagucTestTools().readfromfile(self.expectedoutputsspath + filename),
+        )
+
+