KNMI · maartenplieger · Sep 25, 2024 · Sep 2, 2024 · Sep 2, 2024 · Sep 3, 2024
diff --git a/CCDFDataModel/CCDFVariable.h b/CCDFDataModel/CCDFVariable.h
@@ -291,21 +291,29 @@ namespace CDF {
     void setSize(size_t size) { currentSize = size; }
     size_t getSize() { return currentSize; }
 
-    Attribute *getAttribute(const char *name) const {
+    bool hasAttribute(const char *name) const {
       for (size_t j = 0; j < attributes.size(); j++) {
         if (attributes[j]->name.equals(name)) {
-          return attributes[j];
+          return true;
         }
       }
-      throw(CDF_E_ATTNOTFOUND);
-      return NULL;
+      return false;
+    }
+
+    Attribute *getAttribute(const char *name) const {
+      Attribute *a = getAttributeNE(name);
+      if (a == nullptr) {
+        throw(CDF_E_ATTNOTFOUND);
+      }
+      return a;
     }
     Attribute *getAttributeNE(const char *name) const {
-      try {
-        return getAttribute(name);
-      } catch (int e) {
-        return NULL;
+      for (size_t j = 0; j < attributes.size(); j++) {
+        if (attributes[j]->name.equals(name)) {
+          return attributes[j];
+        }
       }
+      return nullptr;
     }
 
     /**

diff --git a/Dockerfile b/Dockerfile
@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="[email protected]"
 
 # Version should be same as in Definitions.h
-LABEL version="2.27.0"
+LABEL version="2.28.0"
 
 # Try to update image packages
 RUN apt-get -q -y update \
@@ -88,7 +88,7 @@ COPY python /adaguc/adaguc-server-master/python
 # To run the tests against a postgres db, see docs/test_postgesql.md
 FROM base AS test
 
-ENV TEST_IN_CONTAINER 1
+ENV TEST_IN_CONTAINER=1
 
 COPY requirements-dev.txt /adaguc/adaguc-server-master/requirements-dev.txt
 RUN pip install --no-cache-dir -r requirements-dev.txt

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,7 @@
+**Version 2.28.0 2024-09-62**
+- Metadata for layer items like the projections, dimensions and styles can now be stored in a metadatatable
+
+
 **Version 2.27.0 2024-09-02**
 - PostgreSQL query from `getFilesAndIndicesForDimensions` has been rewritten, which fixes https://github.com/KNMI/adaguc-server/issues/341.
 - Optimized existing PostgreSQL queries and reduced number of PostgreSQL queries in general. This results in better performance, the benchmark tool runs 9% faster.

diff --git a/adagucserverEC/CAutoConfigure.cpp b/adagucserverEC/CAutoConfigure.cpp
@@ -484,10 +484,11 @@ int CAutoConfigure::justLoadAFileHeader(CDataSource *dataSource) {
     return 0;
   }
 
-  CT::string foundFileName;
-
-  /* Use the file specified as header file */
-  foundFileName = dataSource->headerFileName.c_str();
+  CT::string foundFileName = dataSource->getFileName();
+  if (foundFileName.empty()) {
+    /* Use the file specified as header file */
+    foundFileName = dataSource->headerFileName.c_str();
+  }
   if (foundFileName.empty()) {
 
     /* Try to get a file from DB */

diff --git a/adagucserverEC/CDBAdapter.h b/adagucserverEC/CDBAdapter.h
@@ -16,7 +16,7 @@
 class CDBAdapter {
 public:
   CDBAdapter() {}
-  virtual ~CDBAdapter(){};
+  virtual ~CDBAdapter() {};
 
   class GeoOptions {
   public:
@@ -96,6 +96,9 @@ class CDBAdapter {
 
   /** First use setFile<type> as many times as you whish, second use addFilesToDataBase to make it final*/
   virtual int addFilesToDataBase() = 0;
+
+  virtual int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob) = 0;
+  virtual CT::string getLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey) = 0;
 };
 
 #endif
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.cpp b/adagucserverEC/CDBAdapterPostgreSQL.cpp
@@ -423,7 +423,7 @@ CDBStore::Store *CDBAdapterPostgreSQL::getFilesAndIndicesForDimensions(CDataSour
     CT::string whereStatement;
     std::vector<CT::string> requestedDimVals = requestedDimMap[m.first];
 
-    for (int i = 0; i < requestedDimVals.size(); ++i) {
+    for (size_t i = 0; i < requestedDimVals.size(); ++i) {
       if (requestedDimVals[i].equals("*")) continue;
 
       if (i != 0) {
@@ -717,7 +717,7 @@ std::map<CT::string, DimInfo> CDBAdapterPostgreSQL::getTableNamesForPathFilterAn
   }
 
   // Found all tablenames for requested dimensions in lookup table
-  int found = tableDimStore->size();
+  size_t found = tableDimStore->size();
   delete tableDimStore;
   if (found == mapping.size()) {
 #ifdef MEASURETIME
@@ -1080,3 +1080,87 @@ int CDBAdapterPostgreSQL::addFilesToDataBase() {
 #endif
   return 0;
 }
+
+int CDBAdapterPostgreSQL::storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadataBlob) {
+#ifdef MEASURETIME
+  StopWatch_Stop(">CDBAdapterPostgreSQL::storeLayerMetadata");
+#endif
+  CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+  if (dataBaseConnection == NULL) {
+    return -1;
+  }
+
+  CT::string tableColumns("datasetname varchar (255), layername varchar (255), metadatakey varchar (255), updatetime TIMESTAMPTZ, blob JSONB, PRIMARY KEY (datasetname, layername, metadatakey)");
+
+  int status = dataBaseConnection->checkTable("layermetadata", tableColumns.c_str());
+  if (status == 1) {
+    CDBError("\nFAIL: Table layermetadata could not be created: %s", tableColumns.c_str());
+    throw(__LINE__);
+  }
+
+  CT::string updateTime = CTime::currentDateTime().c_str();
+  updateTime.setSize(19);
+  CT::string query;
+  query.print("INSERT INTO layermetadata values (E'%s',E'%s', E'%s', E'%s', E'%s') "
+              "ON CONFLICT (datasetname, layername, metadatakey) "
+              "DO UPDATE SET blob = excluded.blob, updatetime = excluded.updatetime;",
+              datasetName, layerName, metadataKey, updateTime.c_str(), metadataBlob);
+
+  status = dataBaseConnection->query(query.c_str());
+  if (status != 0) {
+    CDBError("Unable to insert records: \"%s\"", query.c_str());
+    throw(__LINE__);
+  }
+#ifdef MEASURETIME
+  StopWatch_Stop("<CDBAdapterPostgreSQL::storeLayerMetadata");
+#endif
+  return 0;
+}
+
+CT::string CDBAdapterPostgreSQL::getLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey) {
+#ifdef MEASURETIME
+  StopWatch_Stop(">CDBAdapterPostgreSQL::getLayerMetadata");
+#endif
+
+  if (this->layerMetaDataStore == nullptr) {
+#ifdef CDBAdapterPostgreSQL_DEBUG
+    CDBDebug("Need to query layer metadata for %s/%s/%s", datasetName, layerName, metadataKey);
+#endif
+    CPGSQLDB *dataBaseConnection = getDataBaseConnection();
+    if (dataBaseConnection == NULL) {
+      CDBError("No database connection");
+      throw(__LINE__);
+    }
+
+    CT::string query;
+    query.print("SELECT layername, metadatakey, blob from layermetadata where datasetname = '%s';", datasetName);
+    this->layerMetaDataStore = dataBaseConnection->queryToStore(query.c_str());
+    if (layerMetaDataStore == nullptr) {
+#ifdef CDBAdapterPostgreSQL_DEBUG
+      CDBDebug("Unable query: \"%s\"", query.c_str());
+#endif
+      throw(__LINE__);
+    }
+    if (layerMetaDataStore->size() == 0) {
+      CDBDebug("No results \"%s\"", query.c_str());
+      throw(__LINE__);
+    }
+  }
+
+  auto records = layerMetaDataStore->getRecords();
+  for (auto record : records) {
+    if (record->get("layername")->equals(layerName) && record->get("metadatakey")->equals(metadataKey)) {
+#ifdef MEASURETIME
+      StopWatch_Stop("<CDBAdapterPostgreSQL::getLayerMetadata");
+#endif
+      return record->get("blob");
+    }
+  }
+  // CDBDebug(store->getRecord(0)->get("blob")->c_str());
+
+#ifdef MEASURETIME
+  StopWatch_Stop("<CDBAdapterPostgreSQL::getLayerMetadata");
+#endif
+  CDBDebug("No metadata entry found for %s %s %s", datasetName, layerName, metadataKey);
+  throw __LINE__;
+}
diff --git a/adagucserverEC/CDBAdapterPostgreSQL.h b/adagucserverEC/CDBAdapterPostgreSQL.h
@@ -43,6 +43,8 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   std::map<std::string, std::vector<std::string>> fileListPerTable;
   int createDimTableOfType(const char *dimname, const char *tablename, int type);
 
+  CDBStore::Store *layerMetaDataStore = nullptr;
+
 public:
   CDBAdapterPostgreSQL();
   ~CDBAdapterPostgreSQL();
@@ -87,4 +89,6 @@ class CDBAdapterPostgreSQL : public CDBAdapter {
   int setFileString(const char *tablename, const char *file, const char *dimvalue, int dimindex, const char *filedate, GeoOptions *geoOptions);
   int setFileTimeStamp(const char *tablename, const char *file, const char *dimvalue, int dimindex, const char *filedate, GeoOptions *geoOptions);
   int addFilesToDataBase();
+  int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
+  CT::string getLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey);
 };
diff --git a/adagucserverEC/CDBAdapterSQLLite.cpp b/adagucserverEC/CDBAdapterSQLLite.cpp
@@ -694,11 +694,6 @@ CDBStore::Store *CDBAdapterSQLLite::getFilesAndIndicesForDimensions(CDataSource
 
 int CDBAdapterSQLLite::autoUpdateAndScanDimensionTables(CDataSource *dataSource) {
   CServerParams *srvParams = dataSource->srvParams;
-  ;
-  //   if(srvParams->isAutoLocalFileResourceEnabled()==false){
-  //     CDBDebug("Auto update is not available");
-  //     return 0;
-  //   }
   CServerConfig::XMLE_Layer *cfgLayer = dataSource->cfgLayer;
   CSQLLiteDB *dataBaseConnection = getDataBaseConnection();
   if (dataBaseConnection == NULL) {
@@ -1310,4 +1305,7 @@ CDBStore::Store *CDBAdapterSQLLite::getFilesForIndices(CDataSource *dataSource,
   return store;
 }
 
+int CDBAdapterSQLLite::storeLayerMetadata(const char *, const char *, const char *, const char *) { return 0; }
+CT::string CDBAdapterSQLLite::getLayerMetadata(const char *, const char *, const char *) { return ""; }
+
 #endif
diff --git a/adagucserverEC/CDBAdapterSQLLite.h b/adagucserverEC/CDBAdapterSQLLite.h
@@ -119,5 +119,7 @@ class CDBAdapterSQLLite : public CDBAdapter {
   int setFileString(const char *tablename, const char *file, const char *dimvalue, int dimindex, const char *filedate, GeoOptions *geoOptions);
   int setFileTimeStamp(const char *tablename, const char *file, const char *dimvalue, int dimindex, const char *filedate, GeoOptions *geoOptions);
   int addFilesToDataBase();
+  int storeLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey, const char *metadatablob);
+  CT::string getLayerMetadata(const char *datasetName, const char *layerName, const char *metadataKey);
 };
 #endif
diff --git a/adagucserverEC/CDBFileScanner.cpp b/adagucserverEC/CDBFileScanner.cpp
@@ -31,6 +31,7 @@
 #include "CNetCDFDataWriter.h"
 #include "CCreateTiles.h"
 #include <set>
+#include "utils/LayerMetadataStore.h"
 const char *CDBFileScanner::className = "CDBFileScanner";
 std::vector<CT::string> CDBFileScanner::tableNamesDone;
 // #define CDBFILESCANNER_DEBUG
@@ -820,7 +821,7 @@ int CDBFileScanner::DBLoopFiles(CDataSource *dataSource, int removeNonExistingFi
               // delete cdfObject;cdfObject=NULL;
               // cdfObject=CDFObjectStore::getCDFObjectStore()->deleteCDFObject(&cdfObject);
             } catch (int linenr) {
-              CDBError("Exception in DBLoopFiles at line %d");
+              CDBError("Exception in DBLoopFiles at line %d", linenr);
               CDBError(" *** SKIPPING FILE %s ***", (*fileList)[j].c_str());
               // Close cdfObject. this is only needed if an exception occurs, otherwise it does nothing...
               // delete cdfObject;cdfObject=NULL;
@@ -1052,6 +1053,10 @@ int CDBFileScanner::updatedb(CDataSource *dataSource, CT::string *_tailPath, CT:
     }
   }
 
+  if (scanFlags & CDBFILESCANNER_UPDATEDB) {
+    updateMetaDataTable(dataSource);
+  }
+
   CDBDebug("  ==> *** Finished update layer [%s] ***", dataSource->cfgLayer->Name[0]->value.c_str());
   return 0;
 }
@@ -1101,7 +1106,7 @@ std::vector<std::string> CDBFileScanner::searchFileNames(const char *path, CT::s
       if (expr.empty() == false) { // dataSource->cfgLayer->FilePath[0]->attr.filter.c_str()
         fileFilterExpr.copy(&expr);
       }
-      CDBDebug("Reading directory %s with filter %s", filePath.c_str(), fileFilterExpr.c_str());
+      //      CDBDebug("Reading directory %s with filter %s", filePath.c_str(), fileFilterExpr.c_str());
 
       CDirReader *dirReader = CCachedDirReader::getDirReader(filePath.c_str(), fileFilterExpr.c_str());
       dirReader->listDirRecursive(filePath.c_str(), fileFilterExpr.c_str());

diff --git a/adagucserverEC/CDataPostProcessors/CDataPostProcessor_IncludeLayer.cpp b/adagucserverEC/CDataPostProcessors/CDataPostProcessor_IncludeLayer.cpp
@@ -1,6 +1,7 @@
 #include <ranges>
 #include "CDataPostProcessor_IncludeLayer.h"
 #include "CRequest.h"
+#include <utils/LayerUtils.h>
 /************************/
 /*CDPPIncludeLayer */
 /************************/
@@ -20,7 +21,7 @@ CDataSource *CDPPIncludeLayer::getDataSource(CServerConfig::XMLE_DataPostProc *p
   size_t additionalLayerNo = 0;
   for (size_t j = 0; j < dataSource->srvParams->cfg->Layer.size(); j++) {
     CT::string layerName;
-    dataSource->srvParams->makeUniqueLayerName(&layerName, dataSource->srvParams->cfg->Layer[j]);
+    makeUniqueLayerName(&layerName, dataSource->srvParams->cfg->Layer[j]);
     // CDBDebug("comparing for additionallayer %s==%s", additionalLayerName.c_str(), layerName.c_str());
     if (additionalLayerName.equals(layerName)) {
       additionalLayerNo = j;

diff --git a/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.cpp b/adagucserverEC/CDataPostProcessors/CDataPostProcessor_Operator.cpp
@@ -64,8 +64,8 @@ int CDPPOperator::execute(CServerConfig::XMLE_DataPostProc *proc, CDataSource *d
       newDataObject->cdfVariable->setAttributeText("units", proc->attr.units);
     }
 
-    short attrData[3];
-    attrData[0] = -1;
+    double attrData[1];
+    attrData[0] = 0;
     newDataObject->cdfVariable->setAttribute("_FillValue", newDataObject->cdfVariable->getType(), attrData, 1);
 
     newDataObject->cdfVariable->setCustomReader(CDF::Variable::CustomMemoryReaderInstance);

diff --git a/adagucserverEC/CDataPostProcessors/CDataPostProcessor_WFP.cpp b/adagucserverEC/CDataPostProcessors/CDataPostProcessor_WFP.cpp
@@ -1,6 +1,7 @@
 #include "CDataPostProcessor_WFP.h"
 #include "CRequest.h"
 #include "CGenericDataWarper.h"
+#include <utils/LayerUtils.h>
 
 /************************/
 /*      CDPPWFP  */
@@ -25,7 +26,7 @@ CDataSource *CDPPWFP::getDataSource(CDataSource *dataSource, CT::string baseLaye
   size_t additionalLayerNo = 0;
   for (size_t j = 0; j < dataSource->srvParams->cfg->Layer.size(); j++) {
     CT::string layerName;
-    dataSource->srvParams->makeUniqueLayerName(&layerName, dataSource->srvParams->cfg->Layer[j]);
+    makeUniqueLayerName(&layerName, dataSource->srvParams->cfg->Layer[j]);
     if (baseLayerName.equals(layerName)) {
       additionalLayerNo = j;
       break;

diff --git a/adagucserverEC/CDataReader.cpp b/adagucserverEC/CDataReader.cpp
@@ -421,7 +421,6 @@ int CDataReader::parseDimensions(CDataSource *dataSource, int mode, int x, int y
 #ifdef CDATAREADER_DEBUG
   CDBDebug("Number of dimensions = %d", dataSource->dNetCDFNumDims);
 #endif
-
   // Determine the X and Y dimensions and variables.
   determineXAndYDimIndices(dataSource, dataSourceVar);
   if (!determineXandYVars(dataSource, dataSourceVar, cdfObject)) {
@@ -847,15 +846,10 @@ int CDataReader::open(CDataSource *dataSource, int mode, int x, int y, int *grid
   if (x != -1 && y != -1) {
     singleCellMode = true;
   }
-
   CT::string dataSourceFilename;
   dataSourceFilename.copy(dataSource->getFileName());
-
-  // CCache cache;
-  // CT::string cacheFilename;
-  // pthread_mutex_lock(&CDataReader_open_lock);
   CStyleConfiguration *styleConfiguration = dataSource->getStyle();
-  // pthread_mutex_unlock(&CDataReader_open_lock);
+
   // Use autoscale of legendcolors when the legendscale factor has been set to zero.
   if (styleConfiguration != NULL) {
     if (dataSource->stretchMinMaxDone == false) {
@@ -867,7 +861,6 @@ int CDataReader::open(CDataSource *dataSource, int mode, int x, int y, int *grid
   }
 
   CDFObject *cdfObject = NULL;
-
 #ifdef CDATAREADER_DEBUG
   CDBDebug("Working on [%s] with mode %d and (%d,%d)", dataSourceFilename.c_str(), mode, x, y);
 
@@ -876,19 +869,16 @@ int CDataReader::open(CDataSource *dataSource, int mode, int x, int y, int *grid
     CDBDebug("Working on [%s]", dataSourceFilename.c_str());
   }
 #endif
-
   if (mode == CNETCDFREADER_MODE_OPEN_DIMENSIONS || mode == CNETCDFREADER_MODE_OPEN_HEADER) {
     cdfObject = CDFObjectStore::getCDFObjectStore()->getCDFObjectHeader(dataSource, dataSource->srvParams, dataSourceFilename.c_str(), enableObjectCache);
   }
-
   if (mode == CNETCDFREADER_MODE_OPEN_ALL || mode == CNETCDFREADER_MODE_GET_METADATA || mode == CNETCDFREADER_MODE_OPEN_EXTENT) {
     cdfObject = CDFObjectStore::getCDFObjectStore()->getCDFObject(dataSource, dataSourceFilename.c_str(), enableObjectCache);
   }
   if (cdfObject == NULL) {
     CDBError("Unable to get CDFObject from store");
     return 1;
   }
-
   if (dataSource->attachCDFObject(cdfObject) != 0) {
     CDBError("Unable to attach CDFObject");
     return 1;
@@ -914,7 +904,6 @@ int CDataReader::open(CDataSource *dataSource, int mode, int x, int y, int *grid
 #ifdef MEASURETIME
   StopWatch_Stop("parseDimensions done");
 #endif
-
   if (dataSource->useLonTransformation != -1 && gridExtent == NULL) {
     for (size_t varNr = 0; varNr < dataSource->getNumDataObjects(); varNr++) {
       Proc::swapPixelsAtLocation(dataSource, dataSource->getDataObject(varNr)->cdfVariable, 0);