Merge branch 'AliceO2Group:dev' into 3prong-st

.github/workflows/clean-test.yml

@@ -39,10 +39,6 @@ name: Clean PR checks
         description: build/O2/o2-cs8
         type: boolean
         default: true
-      'check_build/O2/o2-dataflow':
-        description: build/O2/o2-dataflow
-        type: boolean
-        default: true
       'check_build/O2/o2-dataflow-cs8':
         description: build/O2/o2-dataflow-cs8
         type: boolean

.github/workflows/pr-security-approval.yml

@@ -0,0 +1,18 @@
+---
+name: Security approval
+
+'on':
+  pull_request_review:
+    types:
+      - edited
+      - submitted
+
+permissions: {}
+
+jobs:
+  clean:
+    name: Security approval
+    uses: alisw/ali-bot/.github/workflows/pr-security-approval.yml@master
+    permissions:
+      pull-requests: read  # to get last commit for PR
+      statuses: write      # for set-github-status

Algorithm/include/Algorithm/PageParser.h

@@ -255,12 +255,12 @@ class PageParser
       return mElement;
     }
     // comparison
-    bool operator==(const SelfType& rh)
+    bool operator==(const SelfType& rh) const
     {
       return mPosition == rh.mPosition;
     }
     // comparison
-    bool operator!=(const SelfType& rh)
+    bool operator!=(const SelfType& rh) const
     {
       return mPosition != rh.mPosition;
     }

Algorithm/test/pageparser.cxx

@@ -50,7 +50,7 @@ struct ClusterData {
   {
   }
 
-  bool operator==(const ClusterData& rhs)
+  bool operator==(const ClusterData& rhs) const
   {
     return clusterid == rhs.clusterid && x == rhs.x && y == rhs.y && z == rhs.z && e == rhs.e;
   }

CCDB/CMakeLists.txt

@@ -45,6 +45,11 @@ o2_add_executable(upload
             SOURCES src/UploadTool.cxx
             PUBLIC_LINK_LIBRARIES O2::CCDB)
 
+o2_add_executable(cleansemaphores
+            COMPONENT_NAME ccdb
+            SOURCES src/CleanCCDBSemaphores.cxx
+            PUBLIC_LINK_LIBRARIES O2::CCDB)
+
 o2_add_executable(downloadccdbfile
             COMPONENT_NAME ccdb
             SOURCES src/DownloadCCDBFile.cxx

CCDB/include/CCDB/BasicCCDBManager.h

@@ -48,6 +48,8 @@ class CCDBManagerInstance
     std::string uuid;
     long startvalidity = 0;
     long endvalidity = -1;
+    size_t minSize = -1ULL;
+    size_t maxSize = 0;
     int queries = 0;
     int fetches = 0;
     int failures = 0;
@@ -176,6 +178,10 @@ class CCDBManagerInstance
 
   std::string getSummaryString() const;
 
+  size_t getFetchedSize() const { return mFetchedSize; }
+
+  void report(bool longrep = false);
+
   void endOfStream();
 
  private:
@@ -190,10 +196,11 @@ class CCDBManagerInstance
   bool mCanDefault = false;                             // whether default is ok --> useful for testing purposes done standalone/isolation
   bool mCachingEnabled = true;                          // whether caching is enabled
   bool mCheckObjValidityEnabled = false;                // wether the validity of cached object is checked before proceeding to a CCDB API query
+  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
   long mCreatedNotAfter = 0;                            // upper limit for object creation timestamp (TimeMachine mode) - If-Not-After HTTP header
   long mCreatedNotBefore = 0;                           // lower limit for object creation timestamp (TimeMachine mode) - If-Not-Before HTTP header
   long mTimerMS = 0;                                    // timer for queries
-  bool mFatalWhenNull = true;                           // if nullptr blob replies should be treated as fatal (can be set by user)
+  size_t mFetchedSize = 0;                              // total fetched size
   int mQueries = 0;                                     // total number of object queries
   int mFetches = 0;                                     // total number of succesful fetches from CCDB
   int mFailures = 0;                                    // total number of failed fetches
@@ -218,11 +225,16 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
       mFailures++;
     } else {
       mFetches++;
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+      }
     }
   } else {
     auto& cached = mCache[path];
+    cached.queries++;
     if (mCheckObjValidityEnabled && cached.isValid(timestamp)) {
-      cached.queries++;
       return reinterpret_cast<T*>(cached.noCleanupPtr ? cached.noCleanupPtr : cached.objPtr.get());
     }
     ptr = mCCDBAccessor.retrieveFromTFileAny<T>(path, mMetaData, timestamp, &mHeaders, cached.uuid,
@@ -238,12 +250,29 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
         cached.objPtr.reset(ptr);
       }
       cached.uuid = mHeaders["ETag"];
-      try { // this conversion can throw, better to catch immediately
-        cached.startvalidity = std::stol(mHeaders["Valid-From"]);
-        cached.endvalidity = std::stol(mHeaders["Valid-Until"]);
+      try {
+        if (mHeaders.find("Valid-From") != mHeaders.end()) {
+          cached.startvalidity = std::stol(mHeaders["Valid-From"]);
+        } else {
+          // if meta-information missing assume infinit validity
+          // (should happen only for locally created objects)
+          cached.startvalidity = 0;
+        }
+        if (mHeaders.find("Valid-Until") != mHeaders.end()) {
+          cached.endvalidity = std::stol(mHeaders["Valid-Until"]);
+        } else {
+          cached.endvalidity = std::numeric_limits<long>::max();
+        }
       } catch (std::exception const& e) {
         reportFatal("Failed to read validity from CCDB response (Valid-From :  " + mHeaders["Valid-From"] + std::string(" Valid-Until: ") + mHeaders["Valid-Until"] + std::string(")"));
       }
+      auto sh = mHeaders.find("fileSize");
+      if (sh != mHeaders.end()) {
+        size_t s = atol(sh->second.c_str());
+        mFetchedSize += s;
+        cached.minSize = std::min(s, cached.minSize);
+        cached.maxSize = std::max(s, cached.minSize);
+      }
     } else if (mHeaders.count("Error")) { // in case of errors the pointer is 0 and headers["Error"] should be set
       cached.failures++;
       cached.clear(); // in case of any error clear cache for this object

CCDB/include/CCDB/CCDBDownloader.h

@@ -11,6 +11,10 @@
 #ifndef O2_CCDBDOWNLOADER_H_
 #define O2_CCDBDOWNLOADER_H_
 
+#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
+#include "MemoryResources/MemoryResources.h"
+#endif
+
 #include <cstdio>
 #include <cstdlib>
 #include <curl/curl.h>
@@ -21,6 +25,8 @@
 #include <mutex>
 #include <condition_variable>
 #include <unordered_map>
+#include <map>
+#include <functional>
 
 typedef struct uv_loop_s uv_loop_t;
 typedef struct uv_timer_s uv_timer_t;
@@ -32,6 +38,25 @@ typedef struct uv_handle_s uv_handle_t;
 namespace o2::ccdb
 {
 
+#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
+struct HeaderObjectPair_t {
+  std::multimap<std::string, std::string> header;
+  o2::pmr::vector<char>* object = nullptr;
+  int counter = 0;
+};
+
+typedef struct DownloaderRequestData {
+  std::vector<std::string> hosts;
+  std::string path;
+  long timestamp;
+  HeaderObjectPair_t hoPair;
+  std::map<std::string, std::string>* headers;
+  std::string userAgent;
+
+  std::function<bool(std::string)> localContentCallback;
+} DownloaderRequestData;
+#endif
+
 /*
  Some functions below aren't member functions of CCDBDownloader because both curl and libuv require callback functions which have to be either static or non-member.
  Because non-static functions are used in the functions below, they must be non-member.
@@ -133,6 +158,14 @@ class CCDBDownloader
    */
   std::vector<CURLcode> batchBlockingPerform(std::vector<CURL*> const& handleVector);
 
+  /**
+   * Schedules an asynchronous transfer but doesn't perform it.
+   *
+   * @param handle Handle to be performed on.
+   * @param requestCounter Counter shared by a batch of CURL handles.
+   */
+  void asynchSchedule(CURL* handle, size_t* requestCounter);
+
   /**
    * Limits the number of parallel connections. Should be used only if no transfers are happening.
    */
@@ -175,7 +208,30 @@ class CCDBDownloader
    */
   void runLoop(bool noWait);
 
+  /**
+   * Returns a message describing the transfer an it's result.
+   */
+  std::string prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map<std::string, std::string>* headers, long httpCode) const;
+
+  /**
+   * Leaves only the protocol and host part of the url, discrading path and metadata.
+   */
+  std::string trimHostUrl(std::string full_host_url) const;
+
  private:
+  /**
+   * Recognizes whether the address is a full url, or a partial one (like for example "/Task/Detector/1") and combines it with potentialHost if needed.
+   */
+  std::string prepareRedirectedURL(std::string address, std::string potentialHost) const;
+
+  /**
+   * Returns a vector of possible content locations based on the redirect headers.
+   *
+   * @param baseUrl Content path.
+   * @param headerMap Map containing response headers.
+   */
+  std::vector<std::string> getLocations(std::multimap<std::string, std::string>* headerMap) const;
+
   std::string mUserAgentId = "CCDBDownloader";
   /**
    * Sets up internal UV loop.
@@ -207,8 +263,7 @@ class CCDBDownloader
    */
   enum RequestType {
     BLOCKING,
-    ASYNCHRONOUS,
-    ASYNCHRONOUS_WITH_CALLBACK
+    ASYNCHRONOUS
   };
 
   /**
@@ -230,20 +285,19 @@ class CCDBDownloader
 
   DataForSocket mSocketData;
 
+#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
   /**
    * Structure which is stored in a easy_handle. It carries information about the request which the easy_handle is part of.
-   * All easy handles coming from one request have an identical PerformData structure.
    */
   typedef struct PerformData {
-    std::condition_variable* cv;
-    bool* completionFlag;
     CURLcode* codeDestination;
-    void (*cbFun)(void*);
-    std::thread* cbThread;
-    void* cbData;
     size_t* requestsLeft;
     RequestType type;
+    int hostInd;
+    int locInd;
+    DownloaderRequestData* requestData;
   } PerformData;
+#endif
 
   /**
    * Called by CURL in order to close a socket. It will be called by CURL even if a timeout timer closed the socket beforehand.
@@ -253,6 +307,23 @@ class CCDBDownloader
    */
   static void closesocketCallback(void* clientp, curl_socket_t item);
 
+#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
+  // Returns a new location string or an empty string if all locations under current host have been accessedd
+  std::string getNewLocation(PerformData* performData, std::vector<std::string>& locations) const;
+
+  // Reschedules the transfer to be performed with a different host.
+  void tryNewHost(PerformData* performData, CURL* easy_handle);
+
+  // Retrieves content from either alien, cvmfs or local storage using a callback to CCDBApi.
+  void getLocalContent(PerformData* performData, std::string& newLocation, bool& contentRetrieved, std::vector<std::string>& locations);
+
+  // Continues a transfer via a http redirect.
+  void httpRedirect(PerformData* performData, std::string& newLocation, CURL* easy_handle);
+
+  // Continues a transfer via a redirect. The redirect can point to a local file, alien file or a http address.
+  void followRedirect(PerformData* performData, CURL* easy_handle, std::vector<std::string>& locations, bool& rescheduled, bool& contentRetrieved);
+#endif
+
   /**
    *  Is used to react to polling file descriptors in poll_handle.
    *
@@ -322,10 +393,12 @@ class CCDBDownloader
    */
   void checkMultiInfo();
 
+#if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
   /**
    * Set openSocketCallback and closeSocketCallback with appropriate arguments. Stores data inside the CURL handle.
    */
   void setHandleOptions(CURL* handle, PerformData* data);
+#endif
 
   /**
    * Create structure holding information about a socket including a poll handle assigned to it