Merge pull request #46632 from perrotta/introducePopConO2O_141X

Introduce OnlinePopCon mechanism and HLT-compatible LHCInfoPer* PopCon (duringFill mode) - 14_1_O2OTest

CondCore/PopCon/interface/Exception.h

@@ -7,6 +7,7 @@ namespace popcon {
   class Exception : public cms::Exception {
   public:
     explicit Exception(const std::string& message) : cms::Exception("PopCon", message) {}
+    Exception(const std::string& category, const std::string& message) : cms::Exception(category, message) {}
     ~Exception() throw() override {}
   };
 }  // namespace popcon

CondCore/PopCon/interface/OnlinePopCon.h

@@ -0,0 +1,111 @@
+#ifndef CONDCORE_POPCON_ONLINEPOPCON_H
+#define CONDCORE_POPCON_ONLINEPOPCON_H
+
+//
+// Authors:
+//  - Francesco Brivio (Milano-Bicocca)
+//  - Jan Chyczynski   (AGH University of Krakow)
+//
+
+#include "CondCore/DBOutputService/interface/OnlineDBOutputService.h"
+#include "CondCore/PopCon/interface/Exception.h"
+#include "FWCore/ParameterSet/interface/ParameterSetfwd.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+
+#include <algorithm>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace popcon {
+
+  //  Populator of the Condition DB.
+  //  Specific implementation for online lumi-based conditions (OnlineDBOutputService)
+
+  class OnlinePopCon {
+  public:
+    typedef cond::Time_t Time_t;
+
+    OnlinePopCon(const edm::ParameterSet& pset);
+
+    virtual ~OnlinePopCon();
+
+    template <typename Source>
+    void write(Source const& source);
+
+  private:
+    cond::persistency::Session initialize();
+    cond::persistency::Session preparePopCon();
+    void finalize();
+
+  private:
+    // DB service
+    edm::Service<cond::service::OnlineDBOutputService> m_dbService;
+
+    // PopCon infrastructure
+    cond::persistency::Session m_targetSession;
+    std::string m_targetConnectionString;
+    std::string m_authPath;
+    int m_authSys;
+    std::string m_recordName;
+    cond::TagInfo_t m_tagInfo;
+    cond::LogDBEntry_t m_logDBEntry;
+
+    // OnlinePopCon specific
+    int m_dbLoggerReturn_;  // DB logger return value
+    bool m_useLockRecors;   // whether to lock the records or not
+
+    // Version
+    static constexpr const char* const s_version = "1.0";
+  };
+
+  template <typename Source>
+  void OnlinePopCon::write(Source const& source) {
+    // Get data to be uploaded
+    typedef typename Source::Container Container;
+    std::pair<Container const*, std::string const> ret = source(initialize(), m_tagInfo, m_logDBEntry);
+    Container const& iovs = *ret.first;
+
+    // Check that only 1 iov/payload is provided
+    if (iovs.size() > 1) {
+      throw Exception("OnlinePopCon", "[write] More than one iov/payload provided!");
+    }
+
+    // If zero payloads to transfer, finalize and return
+    if (iovs.empty()) {
+      m_dbService->logger().logInfo() << "OnlinePopCon::write - Nothing to transfer";
+      finalize();
+      return;
+    }
+
+    // Upload
+    // Check if DB service is available
+    if (!m_dbService.isAvailable()) {
+      throw Exception("OnlinePopCon", "[write] DBService not available");
+    }
+
+    // Get the only payload to transfer
+    auto [originalSince, payload] = *iovs.begin();
+
+    // Log the original since
+    m_dbService->logger().logInfo() << "OnlinePopCon::write - original since: " << originalSince;
+
+    // Upload the payload
+    try {
+      auto targetSince = m_dbService->writeIOVForNextLumisection(*payload, m_recordName);
+      m_dbService->logger().logInfo() << "OnlinePopCon::write - writeForNextLumisection successful!";
+      m_dbService->logger().logInfo() << "OnlinePopCon::write - uploaded with since: " << targetSince;
+    } catch (const std::exception& e) {
+      m_dbService->logger().logError() << "OnlinePopCon::write - Error writing record: " << m_recordName;
+      m_dbService->logger().logError() << "Error is: " << e.what();
+      m_dbLoggerReturn_ = 2;
+    }
+
+    // Finalize
+    finalize();
+  }
+
+}  // namespace popcon
+
+#endif  // CONDCORE_POPCON_ONLINEPOPCON_H

CondCore/PopCon/interface/OnlinePopConAnalyzer.h

@@ -0,0 +1,45 @@
+#ifndef CONDCORE_POPCON_ONLINEPOPCONANALYZER_H
+#define CONDCORE_POPCON_ONLINEPOPCONANALYZER_H
+
+//
+// Authors:
+//  - Francesco Brivio (Milano-Bicocca)
+//  - Jan Chyczynski   (AGH University of Krakow)
+//
+
+#include "CondCore/PopCon/interface/OnlinePopCon.h"
+#include "FWCore/Framework/interface/Frameworkfwd.h"
+#include "FWCore/Framework/interface/one/EDAnalyzer.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+#include <vector>
+
+namespace popcon {
+  template <typename S>
+  class OnlinePopConAnalyzer : public edm::one::EDAnalyzer<> {
+  public:
+    typedef S SourceHandler;
+
+    OnlinePopConAnalyzer(const edm::ParameterSet& pset)
+        : m_populator(pset), m_source(pset.getParameter<edm::ParameterSet>("Source")) {}
+
+    ~OnlinePopConAnalyzer() override {}
+
+  protected:
+    SourceHandler& source() { return m_source; }
+
+  private:
+    void beginJob() override {}
+    void endJob() override { write(); }
+
+    void analyze(const edm::Event&, const edm::EventSetup&) override {}
+
+    void write() { m_populator.write(m_source); }
+
+  private:
+    OnlinePopCon m_populator;
+    SourceHandler m_source;
+  };
+
+}  // namespace popcon
+#endif  // CONDCORE_POPCON_ONLINEPOPCONANALYZER_H

CondCore/PopCon/src/OnlinePopCon.cc

@@ -0,0 +1,114 @@
+#include "CondCore/CondDB/interface/ConnectionPool.h"
+#include "CondCore/PopCon/interface/OnlinePopCon.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+//#include <iostream>
+
+namespace popcon {
+
+  constexpr const char* const OnlinePopCon::s_version;
+
+  OnlinePopCon::OnlinePopCon(const edm::ParameterSet& pset)
+      : m_targetSession(),
+        m_targetConnectionString(pset.getUntrackedParameter<std::string>("targetDBConnectionString", "")),
+        m_authPath(pset.getUntrackedParameter<std::string>("authenticationPath", "")),
+        m_authSys(pset.getUntrackedParameter<int>("authenticationSystem", 1)),
+        m_recordName(pset.getParameter<std::string>("record")),
+        m_useLockRecors(pset.getUntrackedParameter<bool>("useLockRecords", false)) {
+    edm::LogInfo("OnlinePopCon")
+        << "This is OnlinePopCon (Populator of Condition) v" << s_version << ".\n"
+        << "Please report any problem and feature request through the JIRA project CMSCONDDB.\n";
+  }
+
+  OnlinePopCon::~OnlinePopCon() {
+    if (!m_targetConnectionString.empty()) {
+      m_targetSession.transaction().commit();
+    }
+  }
+
+  cond::persistency::Session OnlinePopCon::preparePopCon() {
+    // Initialization almost identical to PopCon
+    const std::string& connectionStr = m_dbService->session().connectionString();
+    m_dbService->forceInit();
+    std::string tagName = m_dbService->tag(m_recordName);
+    m_tagInfo.name = tagName;
+    if (m_targetConnectionString.empty()) {
+      m_targetSession = m_dbService->session();
+      m_dbService->startTransaction();
+    } else {
+      cond::persistency::ConnectionPool connPool;
+      connPool.setAuthenticationPath(m_authPath);
+      connPool.setAuthenticationSystem(m_authSys);
+      connPool.configure();
+      m_targetSession = connPool.createSession(m_targetConnectionString);
+      m_targetSession.transaction().start();
+    }
+
+    m_dbService->logger().logInfo() << "OnlinePopCon::preparePopCon";
+    m_dbService->logger().logInfo() << "  destination DB: " << connectionStr;
+    m_dbService->logger().logInfo() << "  target DB: "
+                                    << (m_targetConnectionString.empty() ? connectionStr : m_targetConnectionString);
+
+    if (m_targetSession.existsDatabase() && m_targetSession.existsIov(tagName)) {
+      cond::persistency::IOVProxy iov = m_targetSession.readIov(tagName);
+      m_tagInfo.size = iov.sequenceSize();
+      if (m_tagInfo.size > 0) {
+        m_tagInfo.lastInterval = iov.getLast();
+      }
+      m_dbService->logger().logInfo() << "  TAG: " << tagName << ", last since/till: " << m_tagInfo.lastInterval.since
+                                      << "/" << m_tagInfo.lastInterval.till;
+      m_dbService->logger().logInfo() << "  size: " << m_tagInfo.size;
+    } else {
+      m_dbService->logger().logInfo() << "  TAG: " << tagName << "; First writer to this new tag.";
+    }
+    return m_targetSession;
+  }
+
+  cond::persistency::Session OnlinePopCon::initialize() {
+    // Check if DB service is available
+    if (!m_dbService.isAvailable()) {
+      throw Exception("OnlinePopCon", "[initialize] DBService not available");
+    }
+
+    // Start DB logging service
+    m_dbLoggerReturn_ = 0;
+    m_dbService->logger().start();
+    m_dbService->logger().logInfo() << "OnlinePopCon::initialize - begin logging for record: " << m_recordName;
+
+    // If requested, lock records
+    if (m_useLockRecors) {
+      m_dbService->logger().logInfo() << "OnlinePopCon::initialize - locking records";
+      m_dbService->lockRecords();
+    }
+
+    // Prepare the rest of PopCon infrastructure
+    auto session = preparePopCon();
+    return session;
+  }
+
+  void OnlinePopCon::finalize() {
+    // Check if DB service is available
+    if (!m_dbService.isAvailable()) {
+      throw Exception("OnlinePopCon", "[finalize] DBService not available");
+    }
+
+    // Release locks if previously locked
+    if (m_useLockRecors) {
+      m_dbService->logger().logInfo() << "OnlinePopCon::finalize - releasing locks";
+      m_dbService->releaseLocks();
+    }
+
+    // Finalize PopCon infrastructure
+    if (m_targetConnectionString.empty()) {
+      m_dbService->commitTransaction();
+    } else {
+      m_targetSession.transaction().commit();
+    }
+
+    // Stop DB logging service
+    m_dbService->logger().logInfo() << "OnlinePopCon::finalize - end logging for record: " << m_recordName;
+    m_dbService->logger().end(m_dbLoggerReturn_);
+  }
+
+}  // namespace popcon

CondTools/RunInfo/interface/LHCInfoHelper.h

@@ -11,6 +11,9 @@ namespace cond {
     // Large number of LS for the OMS query, covering around 25 hours
     static constexpr unsigned int kLumisectionsQueryLimit = 4000;
 
+    // last Run number and LS number of the specified Fill
+    std::pair<int, unsigned short> getFillLastRunAndLS(const cond::OMSService& oms, unsigned short fillId);
+
     // Returns lumi-type IOV from last LS of last Run of the specified Fill
     cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId);
 

CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h

@@ -0,0 +1,55 @@
+#include "CondCore/PopCon/interface/PopConSourceHandler.h"
+#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerFill> {
+public:
+  LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset);
+  ~LHCInfoPerFillPopConSourceHandler() override = default;
+
+  void getNewObjects() override;
+  std::string id() const override;
+
+private:
+  void addEmptyPayload(cond::Time_t iov);
+
+  // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
+  void addPayloadToBuffer(cond::OMSServiceResultRef& row);
+  void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid);
+
+  size_t getLumiData(const cond::OMSService& oms,
+                     unsigned short fillId,
+                     const boost::posix_time::ptime& beginFillTime,
+                     const boost::posix_time::ptime& endFillTime);
+
+  void getDipData(const cond::OMSService& oms,
+                  const boost::posix_time::ptime& beginFillTime,
+                  const boost::posix_time::ptime& endFillTime);
+
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime);
+
+  bool getEcalData(cond::persistency::Session& session,
+                   const boost::posix_time::ptime& lowerTime,
+                   const boost::posix_time::ptime& upperTime);
+
+private:
+  bool m_debug;
+  // starting date for sampling
+  boost::posix_time::ptime m_startTime;
+  boost::posix_time::ptime m_endTime;
+  bool m_endFillMode = true;
+  std::string m_name;
+  //for reading from relational database source
+  std::string m_connectionString, m_ecalConnectionString;
+  std::string m_authpath;
+  std::string m_omsBaseUrl;
+  std::unique_ptr<LHCInfoPerFill> m_fillPayload;
+  std::shared_ptr<LHCInfoPerFill> m_prevPayload;
+  std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;
+  bool m_lastPayloadEmpty = false;
+  // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs
+  std::map<cond::Time_t, cond::Time_t> m_timestampToLumiid;
+};