HLT-compatible LHCInfoPer* PopCon (duringFill mode)

adjust LHCInfoPer*PopConSourceHandler duringFill mode for HLT-compatibility

fix LHCInfoPerLS PopCon duringFill startSampleTime error

fix filtering in  duringFill mode (prevPayload error)

Remove adding empty payload to empty tag

Extract SourceHandler from LHCInfoPer*PopConAnalyzer

Make perLS and perFill duringFill use OnlinePopCon

Attempt to configure OnlineDBOutputService in LHCInfoPer*PopConAnalyzer_cfg.py

finish LHCInfoPer*PopConAnalyzer_cfg files for updated duringFill

For perLS duringfill implement adding defaultPayload and skipping invalid payloads

implement adding defaultPayload in duringFill mode

duringFill perLS: skip upload if invalid values

Adjust duringFill unit tests

remove duplicated kLumisectionsQueryLimit

fix unit tests, print logs of failed lhcInfoNewPopCon test

add connection strings as cfg parameters, fix frontier key parameter for perFill

CondTools/RunInfo/interface/LHCInfoHelper.h

@@ -11,6 +11,9 @@ namespace cond {
     // Large number of LS for the OMS query, covering around 25 hours
     static constexpr unsigned int kLumisectionsQueryLimit = 4000;
 
+    // last Run number and LS number of the specified Fill
+    std::pair<int, unsigned short> getFillLastRunAndLS(const cond::OMSService& oms, unsigned short fillId);
+
     // Returns lumi-type IOV from last LS of last Run of the specified Fill
     cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId);
 

CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h

@@ -0,0 +1,55 @@
+#include "CondCore/PopCon/interface/PopConSourceHandler.h"
+#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerFill> {
+public:
+  LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset);
+  ~LHCInfoPerFillPopConSourceHandler() override = default;
+
+  void getNewObjects() override;
+  std::string id() const override;
+
+private:
+  void addEmptyPayload(cond::Time_t iov);
+
+  // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
+  void addPayloadToBuffer(cond::OMSServiceResultRef& row);
+  void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid);
+
+  size_t getLumiData(const cond::OMSService& oms,
+                     unsigned short fillId,
+                     const boost::posix_time::ptime& beginFillTime,
+                     const boost::posix_time::ptime& endFillTime);
+
+  void getDipData(const cond::OMSService& oms,
+                  const boost::posix_time::ptime& beginFillTime,
+                  const boost::posix_time::ptime& endFillTime);
+
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime);
+
+  bool getEcalData(cond::persistency::Session& session,
+                   const boost::posix_time::ptime& lowerTime,
+                   const boost::posix_time::ptime& upperTime);
+
+private:
+  bool m_debug;
+  // starting date for sampling
+  boost::posix_time::ptime m_startTime;
+  boost::posix_time::ptime m_endTime;
+  bool m_endFillMode = true;
+  std::string m_name;
+  //for reading from relational database source
+  std::string m_connectionString, m_ecalConnectionString;
+  std::string m_authpath;
+  std::string m_omsBaseUrl;
+  std::unique_ptr<LHCInfoPerFill> m_fillPayload;
+  std::shared_ptr<LHCInfoPerFill> m_prevPayload;
+  std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;
+  bool m_lastPayloadEmpty = false;
+  // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs
+  std::map<cond::Time_t, cond::Time_t> m_timestampToLumiid;
+};

CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h

@@ -0,0 +1,82 @@
+#ifndef LHCInfoPerLSPopConSourceHandler_h
+#define LHCInfoPerLSPopConSourceHandler_h
+
+#include "CondCore/PopCon/interface/PopConSourceHandler.h"
+#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerLS> {
+public:
+  LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset);
+  ~LHCInfoPerLSPopConSourceHandler() override;
+  void getNewObjects() override;
+  std::string id() const override;
+
+private:
+  void populateIovs();
+  void filterInvalidPayloads();
+  bool isPayloadValid(const LHCInfoPerLS& payload) const;
+  void addEmptyPayload(cond::Time_t iov);
+  void addDefaultPayload(cond::Time_t iov, unsigned short fill, const cond::OMSService& oms);
+  void addDefaultPayload(cond::Time_t iov, unsigned short fill, int run, unsigned short lumi);
+  bool makeFillPayload(std::unique_ptr<LHCInfoPerLS>& targetPayload, const cond::OMSServiceResult& queryResult);
+  void addPayloadToBuffer(cond::OMSServiceResultRef& row);
+  size_t bufferAllLS(const cond::OMSServiceResult& queryResult);
+  size_t getLumiData(const cond::OMSService& oms,
+                     unsigned short fillId,
+                     const boost::posix_time::ptime& beginFillTime,
+                     const boost::posix_time::ptime& endFillTime);
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime);
+
+  bool m_debug;
+  // starting date for sampling
+  boost::posix_time::ptime m_startTime;
+  boost::posix_time::ptime m_endTime;
+  bool m_endFillMode = true;
+  std::string m_name;
+  // for reading from relational database source
+  std::string m_connectionString;
+  std::string m_authpath;
+  std::string m_omsBaseUrl;
+  // Allows for basic test of durigFill mode when there is no Stable Beams in LHC
+  // makes duringFill interpret fills as ongoing fill and writing their last LS
+  // (disabling the check if the last LS is in stable beams,
+  // although still only fills with stable beams are being processed
+  // also, still only up to one payload will be written)
+  const bool m_debugLogic;
+  // values for the default payload which is inserted after the last processed fill
+  // has ended and there's no ongoing stable beam yet:
+  float m_defaultCrossingAngleX;
+  float m_defaultCrossingAngleY;
+  float m_defaultBetaStarX;
+  float m_defaultBetaStarY;
+  float m_minBetaStar;       // meters
+  float m_maxBetaStar;       // meters
+  float m_minCrossingAngle;  // urad
+  float m_maxCrossingAngle;  // urad
+
+  std::unique_ptr<LHCInfoPerLS> m_fillPayload;
+  std::shared_ptr<LHCInfoPerLS> m_prevPayload;
+  cond::Time_t m_startFillTime;
+  cond::Time_t m_endFillTime;
+  cond::Time_t m_prevEndFillTime = 0;
+  cond::Time_t m_prevStartFillTime;
+  cond::Time_t m_startStableBeamTime;
+  cond::Time_t m_endStableBeamTime;
+  std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerLS>>> m_tmpBuffer;
+  bool m_lastPayloadEmpty = false;
+  // mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to
+  // the IDs they've been assignd from PPS DB value pair(-1, -1) means lumisection
+  // corresponding to the key exists in OMS but no lumisection was matched from PPS
+  std::map<std::pair<cond::Time_t, unsigned int>, std::pair<cond::Time_t, unsigned int>> m_lsIdMap;
+};
+
+#endif

CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h

@@ -18,8 +18,6 @@ class LHCInfoPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfo> {
   void getNewObjects() override;
   std::string id() const override;
 
-  static constexpr unsigned int kLumisectionsQueryLimit = 4000;  // enough for fills not exceeding 25h
-
 private:
   void addEmptyPayload(cond::Time_t iov);
 

CondTools/RunInfo/plugins/BuildFile.xml

@@ -59,6 +59,10 @@
   <flags EDM_PLUGIN="1"/>
 </library>
 
+<library file="LHCInfoPerLSWriter.cc" name="LHCInfoPerLSWriter">
+  <flags EDM_PLUGIN="1"/>
+</library>
+
 <library file="LHCInfoPerFillAnalyzer.cc" name="LHCInfoPerFillAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>
@@ -67,10 +71,14 @@
   <flags EDM_PLUGIN="1"/>
 </library>
 
+<library file="LHCInfoPerLSOnlinePopConAnalyzer.cc" name="CondToolsLHCInfoPerLSOnlinePopConAnalyzer">
+  <flags EDM_PLUGIN="1"/>
+</library>
+
 <library file="LHCInfoPerFillPopConAnalyzer.cc" name="CondToolsLHCInfoPerFillPopConAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>
 
-<library file="LHCInfoPerLSWriter.cc" name="LHCInfoPerLSWriter">
+<library file="LHCInfoPerFillOnlinePopConAnalyzer.cc" name="CondToolsLHCInfoPerFillOnlinePopConAnalyzer">
   <flags EDM_PLUGIN="1"/>
 </library>

CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc

@@ -0,0 +1,7 @@
+#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h"
+#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h"
+#include "FWCore/Framework/interface/MakerMacros.h"
+
+typedef popcon::OnlinePopConAnalyzer<LHCInfoPerFillPopConSourceHandler> LHCInfoPerFillOnlinePopConAnalyzer;
+
+DEFINE_FWK_MODULE(LHCInfoPerFillOnlinePopConAnalyzer);