From c8a3095138a7fbceed3423d993b939c3e3397d9d Mon Sep 17 00:00:00 2001 From: francescobrivio Date: Wed, 12 Jun 2024 16:25:41 +0200 Subject: [PATCH 1/2] Introduce OnlinePopCon mechanism --- CondCore/PopCon/interface/Exception.h | 1 + CondCore/PopCon/interface/OnlinePopCon.h | 111 +++++++++++++++++ .../PopCon/interface/OnlinePopConAnalyzer.h | 45 +++++++ CondCore/PopCon/src/OnlinePopCon.cc | 114 ++++++++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 CondCore/PopCon/interface/OnlinePopCon.h create mode 100644 CondCore/PopCon/interface/OnlinePopConAnalyzer.h create mode 100644 CondCore/PopCon/src/OnlinePopCon.cc diff --git a/CondCore/PopCon/interface/Exception.h b/CondCore/PopCon/interface/Exception.h index 7a4a4f43296eb..e4abec5e3c7d4 100644 --- a/CondCore/PopCon/interface/Exception.h +++ b/CondCore/PopCon/interface/Exception.h @@ -7,6 +7,7 @@ namespace popcon { class Exception : public cms::Exception { public: explicit Exception(const std::string& message) : cms::Exception("PopCon", message) {} + Exception(const std::string& category, const std::string& message) : cms::Exception(category, message) {} ~Exception() throw() override {} }; } // namespace popcon diff --git a/CondCore/PopCon/interface/OnlinePopCon.h b/CondCore/PopCon/interface/OnlinePopCon.h new file mode 100644 index 0000000000000..7720c58af0cd9 --- /dev/null +++ b/CondCore/PopCon/interface/OnlinePopCon.h @@ -0,0 +1,111 @@ +#ifndef CONDCORE_POPCON_ONLINEPOPCON_H +#define CONDCORE_POPCON_ONLINEPOPCON_H + +// +// Authors: +// - Francesco Brivio (Milano-Bicocca) +// - Jan Chyczynski (AGH University of Krakow) +// + +#include "CondCore/DBOutputService/interface/OnlineDBOutputService.h" +#include "CondCore/PopCon/interface/Exception.h" +#include "FWCore/ParameterSet/interface/ParameterSetfwd.h" +#include "FWCore/ServiceRegistry/interface/Service.h" + +#include +#include +#include +#include +#include + +namespace popcon { + + // Populator of the Condition DB. + // Specific implementation for online lumi-based conditions (OnlineDBOutputService) + + class OnlinePopCon { + public: + typedef cond::Time_t Time_t; + + OnlinePopCon(const edm::ParameterSet& pset); + + virtual ~OnlinePopCon(); + + template + void write(Source const& source); + + private: + cond::persistency::Session initialize(); + cond::persistency::Session preparePopCon(); + void finalize(); + + private: + // DB service + edm::Service m_dbService; + + // PopCon infrastructure + cond::persistency::Session m_targetSession; + std::string m_targetConnectionString; + std::string m_authPath; + int m_authSys; + std::string m_recordName; + cond::TagInfo_t m_tagInfo; + cond::LogDBEntry_t m_logDBEntry; + + // OnlinePopCon specific + int m_dbLoggerReturn_; // DB logger return value + bool m_useLockRecors; // whether to lock the records or not + + // Version + static constexpr const char* const s_version = "1.0"; + }; + + template + void OnlinePopCon::write(Source const& source) { + // Get data to be uploaded + typedef typename Source::Container Container; + std::pair ret = source(initialize(), m_tagInfo, m_logDBEntry); + Container const& iovs = *ret.first; + + // Check that only 1 iov/payload is provided + if (iovs.size() > 1) { + throw Exception("OnlinePopCon", "[write] More than one iov/payload provided!"); + } + + // If zero payloads to transfer, finalize and return + if (iovs.empty()) { + m_dbService->logger().logInfo() << "OnlinePopCon::write - Nothing to transfer"; + finalize(); + return; + } + + // Upload + // Check if DB service is available + if (!m_dbService.isAvailable()) { + throw Exception("OnlinePopCon", "[write] DBService not available"); + } + + // Get the only payload to transfer + auto [originalSince, payload] = *iovs.begin(); + + // Log the original since + m_dbService->logger().logInfo() << "OnlinePopCon::write - original since: " << originalSince; + + // Upload the payload + try { + auto targetSince = m_dbService->writeIOVForNextLumisection(*payload, m_recordName); + m_dbService->logger().logInfo() << "OnlinePopCon::write - writeForNextLumisection successful!"; + m_dbService->logger().logInfo() << "OnlinePopCon::write - uploaded with since: " << targetSince; + } catch (const std::exception& e) { + m_dbService->logger().logError() << "OnlinePopCon::write - Error writing record: " << m_recordName; + m_dbService->logger().logError() << "Error is: " << e.what(); + m_dbLoggerReturn_ = 2; + } + + // Finalize + finalize(); + } + +} // namespace popcon + +#endif // CONDCORE_POPCON_ONLINEPOPCON_H diff --git a/CondCore/PopCon/interface/OnlinePopConAnalyzer.h b/CondCore/PopCon/interface/OnlinePopConAnalyzer.h new file mode 100644 index 0000000000000..e42ef4f4b4f73 --- /dev/null +++ b/CondCore/PopCon/interface/OnlinePopConAnalyzer.h @@ -0,0 +1,45 @@ +#ifndef CONDCORE_POPCON_ONLINEPOPCONANALYZER_H +#define CONDCORE_POPCON_ONLINEPOPCONANALYZER_H + +// +// Authors: +// - Francesco Brivio (Milano-Bicocca) +// - Jan Chyczynski (AGH University of Krakow) +// + +#include "CondCore/PopCon/interface/OnlinePopCon.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/one/EDAnalyzer.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +#include + +namespace popcon { + template + class OnlinePopConAnalyzer : public edm::one::EDAnalyzer<> { + public: + typedef S SourceHandler; + + OnlinePopConAnalyzer(const edm::ParameterSet& pset) + : m_populator(pset), m_source(pset.getParameter("Source")) {} + + ~OnlinePopConAnalyzer() override {} + + protected: + SourceHandler& source() { return m_source; } + + private: + void beginJob() override {} + void endJob() override { write(); } + + void analyze(const edm::Event&, const edm::EventSetup&) override {} + + void write() { m_populator.write(m_source); } + + private: + OnlinePopCon m_populator; + SourceHandler m_source; + }; + +} // namespace popcon +#endif // CONDCORE_POPCON_ONLINEPOPCONANALYZER_H diff --git a/CondCore/PopCon/src/OnlinePopCon.cc b/CondCore/PopCon/src/OnlinePopCon.cc new file mode 100644 index 0000000000000..5a9a9a76332b9 --- /dev/null +++ b/CondCore/PopCon/src/OnlinePopCon.cc @@ -0,0 +1,114 @@ +#include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondCore/PopCon/interface/OnlinePopCon.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +//#include + +namespace popcon { + + constexpr const char* const OnlinePopCon::s_version; + + OnlinePopCon::OnlinePopCon(const edm::ParameterSet& pset) + : m_targetSession(), + m_targetConnectionString(pset.getUntrackedParameter("targetDBConnectionString", "")), + m_authPath(pset.getUntrackedParameter("authenticationPath", "")), + m_authSys(pset.getUntrackedParameter("authenticationSystem", 1)), + m_recordName(pset.getParameter("record")), + m_useLockRecors(pset.getUntrackedParameter("useLockRecords", false)) { + edm::LogInfo("OnlinePopCon") + << "This is OnlinePopCon (Populator of Condition) v" << s_version << ".\n" + << "Please report any problem and feature request through the JIRA project CMSCONDDB.\n"; + } + + OnlinePopCon::~OnlinePopCon() { + if (!m_targetConnectionString.empty()) { + m_targetSession.transaction().commit(); + } + } + + cond::persistency::Session OnlinePopCon::preparePopCon() { + // Initialization almost identical to PopCon + const std::string& connectionStr = m_dbService->session().connectionString(); + m_dbService->forceInit(); + std::string tagName = m_dbService->tag(m_recordName); + m_tagInfo.name = tagName; + if (m_targetConnectionString.empty()) { + m_targetSession = m_dbService->session(); + m_dbService->startTransaction(); + } else { + cond::persistency::ConnectionPool connPool; + connPool.setAuthenticationPath(m_authPath); + connPool.setAuthenticationSystem(m_authSys); + connPool.configure(); + m_targetSession = connPool.createSession(m_targetConnectionString); + m_targetSession.transaction().start(); + } + + m_dbService->logger().logInfo() << "OnlinePopCon::preparePopCon"; + m_dbService->logger().logInfo() << " destination DB: " << connectionStr; + m_dbService->logger().logInfo() << " target DB: " + << (m_targetConnectionString.empty() ? connectionStr : m_targetConnectionString); + + if (m_targetSession.existsDatabase() && m_targetSession.existsIov(tagName)) { + cond::persistency::IOVProxy iov = m_targetSession.readIov(tagName); + m_tagInfo.size = iov.sequenceSize(); + if (m_tagInfo.size > 0) { + m_tagInfo.lastInterval = iov.getLast(); + } + m_dbService->logger().logInfo() << " TAG: " << tagName << ", last since/till: " << m_tagInfo.lastInterval.since + << "/" << m_tagInfo.lastInterval.till; + m_dbService->logger().logInfo() << " size: " << m_tagInfo.size; + } else { + m_dbService->logger().logInfo() << " TAG: " << tagName << "; First writer to this new tag."; + } + return m_targetSession; + } + + cond::persistency::Session OnlinePopCon::initialize() { + // Check if DB service is available + if (!m_dbService.isAvailable()) { + throw Exception("OnlinePopCon", "[initialize] DBService not available"); + } + + // Start DB logging service + m_dbLoggerReturn_ = 0; + m_dbService->logger().start(); + m_dbService->logger().logInfo() << "OnlinePopCon::initialize - begin logging for record: " << m_recordName; + + // If requested, lock records + if (m_useLockRecors) { + m_dbService->logger().logInfo() << "OnlinePopCon::initialize - locking records"; + m_dbService->lockRecords(); + } + + // Prepare the rest of PopCon infrastructure + auto session = preparePopCon(); + return session; + } + + void OnlinePopCon::finalize() { + // Check if DB service is available + if (!m_dbService.isAvailable()) { + throw Exception("OnlinePopCon", "[finalize] DBService not available"); + } + + // Release locks if previously locked + if (m_useLockRecors) { + m_dbService->logger().logInfo() << "OnlinePopCon::finalize - releasing locks"; + m_dbService->releaseLocks(); + } + + // Finalize PopCon infrastructure + if (m_targetConnectionString.empty()) { + m_dbService->commitTransaction(); + } else { + m_targetSession.transaction().commit(); + } + + // Stop DB logging service + m_dbService->logger().logInfo() << "OnlinePopCon::finalize - end logging for record: " << m_recordName; + m_dbService->logger().end(m_dbLoggerReturn_); + } + +} // namespace popcon From be1fc654066f64c2082238d0ae210616def58526 Mon Sep 17 00:00:00 2001 From: Jan Chyczynski Date: Fri, 17 May 2024 12:51:59 +0200 Subject: [PATCH 2/2] HLT-compatible LHCInfoPer* PopCon (duringFill mode) adjust LHCInfoPer*PopConSourceHandler duringFill mode for HLT-compatibility fix LHCInfoPerLS PopCon duringFill startSampleTime error fix filtering in duringFill mode (prevPayload error) Remove adding empty payload to empty tag Extract SourceHandler from LHCInfoPer*PopConAnalyzer Make perLS and perFill duringFill use OnlinePopCon Attempt to configure OnlineDBOutputService in LHCInfoPer*PopConAnalyzer_cfg.py finish LHCInfoPer*PopConAnalyzer_cfg files for updated duringFill For perLS duringfill implement adding defaultPayload and skipping invalid payloads implement adding defaultPayload in duringFill mode duringFill perLS: skip upload if invalid values Adjust duringFill unit tests remove duplicated kLumisectionsQueryLimit fix unit tests, print logs of failed lhcInfoNewPopCon test add connection strings as cfg parameters, fix frontier key parameter for perFill --- CondTools/RunInfo/interface/LHCInfoHelper.h | 3 + .../LHCInfoPerFillPopConSourceHandler.h | 55 ++ .../LHCInfoPerLSPopConSourceHandler.h | 82 ++ .../interface/LHCInfoPopConSourceHandler.h | 2 - CondTools/RunInfo/plugins/BuildFile.xml | 10 +- .../LHCInfoPerFillOnlinePopConAnalyzer.cc | 7 + .../plugins/LHCInfoPerFillPopConAnalyzer.cc | 748 +----------------- .../LHCInfoPerLSOnlinePopConAnalyzer.cc | 7 + .../plugins/LHCInfoPerLSPopConAnalyzer.cc | 586 +------------- .../LHCInfoPerFillPopConAnalyzer_cfg.py | 104 ++- .../python/LHCInfoPerLSPopConAnalyzer_cfg.py | 174 +++- CondTools/RunInfo/src/LHCInfoHelper.cc | 11 +- .../src/LHCInfoPerFillPopConSourceHandler.cc | 744 +++++++++++++++++ .../src/LHCInfoPerLSPopConSourceHandler.cc | 659 +++++++++++++++ .../RunInfo/src/LHCInfoPopConSourceHandler.cc | 3 +- CondTools/RunInfo/test/last_lumi.txt | 1 + .../RunInfo/test/test_lhcInfoNewPopCon.sh | 45 +- 17 files changed, 1846 insertions(+), 1395 deletions(-) create mode 100644 CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h create mode 100644 CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h create mode 100644 CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc create mode 100644 CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc create mode 100644 CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc create mode 100644 CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc create mode 100644 CondTools/RunInfo/test/last_lumi.txt diff --git a/CondTools/RunInfo/interface/LHCInfoHelper.h b/CondTools/RunInfo/interface/LHCInfoHelper.h index 61d005c287a17..60d96ed89a475 100644 --- a/CondTools/RunInfo/interface/LHCInfoHelper.h +++ b/CondTools/RunInfo/interface/LHCInfoHelper.h @@ -11,6 +11,9 @@ namespace cond { // Large number of LS for the OMS query, covering around 25 hours static constexpr unsigned int kLumisectionsQueryLimit = 4000; + // last Run number and LS number of the specified Fill + std::pair getFillLastRunAndLS(const cond::OMSService& oms, unsigned short fillId); + // Returns lumi-type IOV from last LS of last Run of the specified Fill cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId); diff --git a/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h new file mode 100644 index 0000000000000..9a0a5185258fc --- /dev/null +++ b/CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h @@ -0,0 +1,55 @@ +#include "CondCore/PopCon/interface/PopConSourceHandler.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler { +public: + LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset); + ~LHCInfoPerFillPopConSourceHandler() override = default; + + void getNewObjects() override; + std::string id() const override; + +private: + void addEmptyPayload(cond::Time_t iov); + + // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map + void addPayloadToBuffer(cond::OMSServiceResultRef& row); + void convertBufferedIovsToLumiid(std::map timestampToLumiid); + + size_t getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + void getDipData(const cond::OMSService& oms, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool getEcalData(cond::persistency::Session& session, + const boost::posix_time::ptime& lowerTime, + const boost::posix_time::ptime& upperTime); + +private: + bool m_debug; + // starting date for sampling + boost::posix_time::ptime m_startTime; + boost::posix_time::ptime m_endTime; + bool m_endFillMode = true; + std::string m_name; + //for reading from relational database source + std::string m_connectionString, m_ecalConnectionString; + std::string m_authpath; + std::string m_omsBaseUrl; + std::unique_ptr m_fillPayload; + std::shared_ptr m_prevPayload; + std::vector>> m_tmpBuffer; + bool m_lastPayloadEmpty = false; + // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs + std::map m_timestampToLumiid; +}; diff --git a/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h new file mode 100644 index 0000000000000..0578663626030 --- /dev/null +++ b/CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h @@ -0,0 +1,82 @@ +#ifndef LHCInfoPerLSPopConSourceHandler_h +#define LHCInfoPerLSPopConSourceHandler_h + +#include "CondCore/PopCon/interface/PopConSourceHandler.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include +#include +#include +#include +#include + +class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler { +public: + LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset); + ~LHCInfoPerLSPopConSourceHandler() override; + void getNewObjects() override; + std::string id() const override; + +private: + void populateIovs(); + void filterInvalidPayloads(); + bool isPayloadValid(const LHCInfoPerLS& payload) const; + void addEmptyPayload(cond::Time_t iov); + void addDefaultPayload(cond::Time_t iov, unsigned short fill, const cond::OMSService& oms); + void addDefaultPayload(cond::Time_t iov, unsigned short fill, int run, unsigned short lumi); + bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult); + void addPayloadToBuffer(cond::OMSServiceResultRef& row); + size_t bufferAllLS(const cond::OMSServiceResult& queryResult); + size_t getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + bool getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime); + + bool m_debug; + // starting date for sampling + boost::posix_time::ptime m_startTime; + boost::posix_time::ptime m_endTime; + bool m_endFillMode = true; + std::string m_name; + // for reading from relational database source + std::string m_connectionString; + std::string m_authpath; + std::string m_omsBaseUrl; + // Allows for basic test of durigFill mode when there is no Stable Beams in LHC + // makes duringFill interpret fills as ongoing fill and writing their last LS + // (disabling the check if the last LS is in stable beams, + // although still only fills with stable beams are being processed + // also, still only up to one payload will be written) + const bool m_debugLogic; + // values for the default payload which is inserted after the last processed fill + // has ended and there's no ongoing stable beam yet: + float m_defaultCrossingAngleX; + float m_defaultCrossingAngleY; + float m_defaultBetaStarX; + float m_defaultBetaStarY; + float m_minBetaStar; // meters + float m_maxBetaStar; // meters + float m_minCrossingAngle; // urad + float m_maxCrossingAngle; // urad + + std::unique_ptr m_fillPayload; + std::shared_ptr m_prevPayload; + cond::Time_t m_startFillTime; + cond::Time_t m_endFillTime; + cond::Time_t m_prevEndFillTime = 0; + cond::Time_t m_prevStartFillTime; + cond::Time_t m_startStableBeamTime; + cond::Time_t m_endStableBeamTime; + std::vector>> m_tmpBuffer; + bool m_lastPayloadEmpty = false; + // mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to + // the IDs they've been assignd from PPS DB value pair(-1, -1) means lumisection + // corresponding to the key exists in OMS but no lumisection was matched from PPS + std::map, std::pair> m_lsIdMap; +}; + +#endif \ No newline at end of file diff --git a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h index 6139ba4becd26..4f5a2dd8a4e0b 100644 --- a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h +++ b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h @@ -18,8 +18,6 @@ class LHCInfoPopConSourceHandler : public popcon::PopConSourceHandler { void getNewObjects() override; std::string id() const override; - static constexpr unsigned int kLumisectionsQueryLimit = 4000; // enough for fills not exceeding 25h - private: void addEmptyPayload(cond::Time_t iov); diff --git a/CondTools/RunInfo/plugins/BuildFile.xml b/CondTools/RunInfo/plugins/BuildFile.xml index 9616cb9213904..fddb5283b8496 100644 --- a/CondTools/RunInfo/plugins/BuildFile.xml +++ b/CondTools/RunInfo/plugins/BuildFile.xml @@ -59,6 +59,10 @@ + + + + @@ -67,10 +71,14 @@ + + + + - + diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc new file mode 100644 index 0000000000000..700052917ccde --- /dev/null +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillOnlinePopConAnalyzer.cc @@ -0,0 +1,7 @@ +#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h" +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +typedef popcon::OnlinePopConAnalyzer LHCInfoPerFillOnlinePopConAnalyzer; + +DEFINE_FWK_MODULE(LHCInfoPerFillOnlinePopConAnalyzer); diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc index c87e36815e557..673ea289e788f 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc @@ -1,753 +1,7 @@ -#include "CondCore/CondDB/interface/ConnectionPool.h" #include "CondCore/PopCon/interface/PopConAnalyzer.h" -#include "CondCore/PopCon/interface/PopConSourceHandler.h" -#include "CondFormats/Common/interface/TimeConversions.h" -#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" -#include "CondTools/RunInfo/interface/LumiSectionFilter.h" -#include "CondTools/RunInfo/interface/LHCInfoHelper.h" -#include "CondTools/RunInfo/interface/OMSAccess.h" -#include "CoralBase/Attribute.h" -#include "CoralBase/AttributeList.h" -#include "CoralBase/AttributeSpecification.h" -#include "CoralBase/TimeStamp.h" +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" #include "FWCore/Framework/interface/MakerMacros.h" -#include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ParameterSet/interface/ParameterSetfwd.h" -#include "RelationalAccess/ICursor.h" -#include "RelationalAccess/IQuery.h" -#include "RelationalAccess/ISchema.h" -#include "RelationalAccess/ISessionProxy.h" - -class LHCInfoPerFillPopConSourceHandler; typedef popcon::PopConAnalyzer LHCInfoPerFillPopConAnalyzer; //define this as a plug-in DEFINE_FWK_MODULE(LHCInfoPerFillPopConAnalyzer); - -namespace cond { - namespace theLHCInfoPerFillPopConImpl { - - static const std::pair s_fillTypeMap[] = { - std::make_pair("PROTONS", LHCInfoPerFill::PROTONS), - std::make_pair("IONS", LHCInfoPerFill::IONS), - std::make_pair("COSMICS", LHCInfoPerFill::COSMICS), - std::make_pair("GAP", LHCInfoPerFill::GAP)}; - - static const std::pair s_particleTypeMap[] = { - std::make_pair("PROTON", LHCInfoPerFill::PROTON), - std::make_pair("PB82", LHCInfoPerFill::PB82), - std::make_pair("AR18", LHCInfoPerFill::AR18), - std::make_pair("D", LHCInfoPerFill::D), - std::make_pair("XE54", LHCInfoPerFill::XE54)}; - - LHCInfoPerFill::FillType fillTypeFromString(const std::string& s_fill_type) { - for (auto const& i : s_fillTypeMap) - if (s_fill_type == i.first) - return i.second; - return LHCInfoPerFill::UNKNOWN; - } - - LHCInfoPerFill::ParticleType particleTypeFromString(const std::string& s_particle_type) { - for (auto const& i : s_particleTypeMap) - if (s_particle_type == i.first) - return i.second; - return LHCInfoPerFill::NONE; - } - } // namespace theLHCInfoPerFillPopConImpl - - namespace impl { - - template <> - LHCInfoPerFill::FillType from_string(const std::string& attributeValue) { - return from_string_impl( - attributeValue, LHCInfoPerFill::UNKNOWN); - } - - template <> - LHCInfoPerFill::ParticleType from_string(const std::string& attributeValue) { - return from_string_impl( - attributeValue, LHCInfoPerFill::NONE); - } - - } // namespace impl -} // namespace cond - -namespace theLHCInfoPerFillImpl { - - bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { - bool ret = false; - if (!queryResult.empty()) { - auto row = *queryResult.begin(); - auto currentFill = row.get("fill_number"); - auto bunches1 = row.get("bunches_beam1"); - auto bunches2 = row.get("bunches_beam2"); - auto collidingBunches = row.get("bunches_colliding"); - auto targetBunches = row.get("bunches_target"); - auto fillType = row.get("fill_type_runtime"); - auto particleType1 = row.get("fill_type_party1"); - auto particleType2 = row.get("fill_type_party2"); - auto intensityBeam1 = row.get("intensity_beam1"); - auto intensityBeam2 = row.get("intensity_beam2"); - auto energy = row.get("energy"); - auto creationTime = row.get("start_time"); - auto stableBeamStartTime = row.get("start_stable_beam"); - std::string endTimeStr = row.get("end_time"); - auto beamDumpTime = - (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); - auto injectionScheme = row.get("injection_scheme"); - targetPayload = std::make_unique(); - targetPayload->setFillNumber(currentFill); - targetPayload->setBunchesInBeam1(bunches1); - targetPayload->setBunchesInBeam2(bunches2); - targetPayload->setCollidingBunches(collidingBunches); - targetPayload->setTargetBunches(targetBunches); - targetPayload->setFillType(fillType); - targetPayload->setParticleTypeForBeam1(particleType1); - targetPayload->setParticleTypeForBeam2(particleType2); - targetPayload->setIntensityForBeam1(intensityBeam1); - targetPayload->setIntensityForBeam2(intensityBeam2); - targetPayload->setEnergy(energy); - targetPayload->setCreationTime(cond::time::from_boost(creationTime)); - targetPayload->setBeginTime(cond::time::from_boost(stableBeamStartTime)); - targetPayload->setEndTime(beamDumpTime); - targetPayload->setInjectionScheme(injectionScheme); - ret = true; - } - return ret; - } -} // namespace theLHCInfoPerFillImpl - -namespace theLHCInfoPerFillImpl { - static const std::map vecMap = { - {"Beam1/beamPhaseMean", 1}, {"Beam2/beamPhaseMean", 2}, {"Beam1/cavPhaseMean", 3}, {"Beam2/cavPhaseMean", 4}}; - void setElementData(cond::Time_t since, - const std::string& dipVal, - unsigned int elementNr, - float value, - LHCInfoPerFill& payload, - std::set& initList) { - if (initList.find(since) == initList.end()) { - payload.beam1VC().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam2VC().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam1RF().resize(LHCInfoPerFill::bunchSlots, 0.); - payload.beam2RF().resize(LHCInfoPerFill::bunchSlots, 0.); - initList.insert(since); - } - // set the current values to all of the payloads of the lumi section samples after the current since - if (elementNr < LHCInfoPerFill::bunchSlots) { - switch (vecMap.at(dipVal)) { - case 1: - payload.beam1VC()[elementNr] = value; - break; - case 2: - payload.beam2VC()[elementNr] = value; - break; - case 3: - payload.beam1RF()[elementNr] = value; - break; - case 4: - payload.beam2RF()[elementNr] = value; - break; - default: - break; - } - } - } -} // namespace theLHCInfoPerFillImpl - -namespace theLHCInfoPerFillImpl { - bool comparePayloads(const LHCInfoPerFill& rhs, const LHCInfoPerFill& lhs) { - if (rhs.fillNumber() != lhs.fillNumber() || rhs.delivLumi() != lhs.delivLumi() || rhs.recLumi() != lhs.recLumi() || - rhs.instLumi() != lhs.instLumi() || rhs.instLumiError() != lhs.instLumiError() || - rhs.lhcState() != lhs.lhcState() || rhs.lhcComment() != lhs.lhcComment() || - rhs.ctppsStatus() != lhs.ctppsStatus()) { - return false; - } - return true; - } - - size_t transferPayloads(const std::vector>>& buffer, - std::map>& iovsToTransfer, - std::shared_ptr& prevPayload) { - size_t niovs = 0; - std::stringstream condIovs; - std::stringstream formattedIovs; - for (auto& iov : buffer) { - bool add = false; - auto payload = iov.second; - cond::Time_t since = iov.first; - if (iovsToTransfer.empty()) { - add = true; - } else { - LHCInfoPerFill& lastAdded = *iovsToTransfer.rbegin()->second; - if (!comparePayloads(lastAdded, *payload)) { - add = true; - } - } - if (add) { - niovs++; - condIovs << since << " "; - formattedIovs << boost::posix_time::to_iso_extended_string(cond::time::to_boost(since)) << " "; - iovsToTransfer.insert(std::make_pair(since, payload)); - prevPayload = iov.second; - } - } - edm::LogInfo("transferPayloads") << "TRANSFERED IOVS: " << condIovs.str(); - edm::LogInfo("transferPayloads") << "FORMATTED TRANSFERED IOVS: " << formattedIovs.str(); - return niovs; - } - -} // namespace theLHCInfoPerFillImpl -class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler { -public: - LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset) - : m_debug(pset.getUntrackedParameter("debug", false)), - m_startTime(), - m_endTime(), - m_endFillMode(pset.getUntrackedParameter("endFill", true)), - m_name(pset.getUntrackedParameter("name", "LHCInfoPerFillPopConSourceHandler")), - m_connectionString(pset.getUntrackedParameter("connectionString", "")), - m_ecalConnectionString(pset.getUntrackedParameter("ecalConnectionString", "")), - m_authpath(pset.getUntrackedParameter("authenticationPath", "")), - m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), - m_fillPayload(), - m_prevPayload(), - m_tmpBuffer() { - if (!pset.getUntrackedParameter("startTime").empty()) { - m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); - } - boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); - m_endTime = now; - if (!pset.getUntrackedParameter("endTime").empty()) { - m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); - if (m_endTime > now) - m_endTime = now; - } - } - - ~LHCInfoPerFillPopConSourceHandler() override = default; - - void getNewObjects() override { - //if a new tag is created, transfer fake fill from 1 to the first fill for the first time - if (tagInfo().size == 0) { - edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; - } else { - //check what is already inside the database - edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size - << ", last object valid since " << tagInfo().lastInterval.since << " ( " - << boost::posix_time::to_iso_extended_string( - cond::time::to_boost(tagInfo().lastInterval.since)) - << " ); from " << m_name << "::getNewObjects"; - } - - cond::Time_t lastSince = tagInfo().lastInterval.since; - if (tagInfo().isEmpty()) { - // for a new or empty tag, an empty payload should be added on top with since=1 - if (m_endFillMode) { - addEmptyPayload(1); - lastSince = 1; - } else { - addEmptyPayload(cond::time::lumiTime(1, 1)); - lastSince = cond::time::lumiTime(1, 1); - } - } else { - edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " - << m_name << "::getNewObjects"; - } - - //retrieve the data from the relational database source - cond::persistency::ConnectionPool connection; - //configure the connection - if (m_debug) { - connection.setMessageVerbosity(coral::Debug); - } else { - connection.setMessageVerbosity(coral::Error); - } - connection.setAuthenticationPath(m_authpath); - connection.configure(); - //create the sessions - cond::persistency::Session session = connection.createSession(m_connectionString, false); - cond::persistency::Session session2 = connection.createSession(m_ecalConnectionString, false); - // fetch last payload when available - if (!tagInfo().lastInterval.payloadId.empty()) { - cond::persistency::Session session3 = dbSession(); - session3.transaction().start(true); - m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); - session3.transaction().commit(); - } - - boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); - cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); - - cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); - cond::Time_t nextFillSearchTimestamp = - std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime()); - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); - - while (true) { - if (nextFillSearchTimestamp >= executionTimeIov) { - edm::LogInfo(m_name) << "Sampling ended at the time " - << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); - break; - } - boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); - boost::posix_time::ptime startSampleTime; - boost::posix_time::ptime endSampleTime; - - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - - edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); - query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) { - query->filterGE("start_time", nextFillSearchTime); - } else { - query->filterGT("start_time", nextFillSearchTime); - } - - query->filterLT("start_time", m_endTime); - if (m_endFillMode) - query->filterNotNull("end_time"); - bool foundFill = query->execute(); - if (foundFill) - foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogInfo(m_name) << "No fill found - END of job."; - break; - } - - startSampleTime = cond::time::to_boost(m_fillPayload->createTime()); - cond::Time_t startFillTime = m_fillPayload->createTime(); - cond::Time_t endFillTime = m_fillPayload->endTime(); - unsigned short lhcFill = m_fillPayload->fillNumber(); - bool ongoingFill = endFillTime == 0ULL; - if (ongoingFill) { - edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " - << cond::time::to_boost(startFillTime); - endSampleTime = executionTime; - nextFillSearchTimestamp = executionTimeIov; - } else { - edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime) - << " ending at " << cond::time::to_boost(endFillTime); - endSampleTime = cond::time::to_boost(endFillTime); - nextFillSearchTimestamp = endFillTime; - } - if (m_endFillMode || ongoingFill) { - getDipData(oms, startSampleTime, endSampleTime); - getLumiData(oms, lhcFill, startSampleTime, endSampleTime); - if (!m_tmpBuffer.empty()) { - boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); - boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); - edm::LogInfo(m_name) << "First lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; - session.transaction().start(true); - getCTPPSData(session, startSampleTime, endSampleTime); - session.transaction().commit(); - session2.transaction().start(true); - getEcalData(session2, startSampleTime, endSampleTime); - session2.transaction().commit(); - } - } - - // In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs - // before transferring the payloads from the buffer to the final collection - if (!m_endFillMode) { - convertBufferedIovsToLumiid(m_timestampToLumiid); - } - - size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload); - edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; - m_tmpBuffer.clear(); - m_timestampToLumiid.clear(); - if (m_prevPayload->fillNumber() and !ongoingFill) { - if (m_endFillMode) { - addEmptyPayload(endFillTime); - } else { - addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); - } - } - } - } - - std::string id() const override { return m_name; } - - static constexpr unsigned int kLumisectionsQueryLimit = 4000; - -private: - void addEmptyPayload(cond::Time_t iov) { - bool add = false; - if (m_iovs.empty()) { - if (!m_lastPayloadEmpty) - add = true; - } else { - auto lastAdded = m_iovs.rbegin()->second; - if (lastAdded->fillNumber() != 0) { - add = true; - } - } - if (add) { - auto newPayload = std::make_shared(); - m_iovs.insert(std::make_pair(iov, newPayload)); - m_prevPayload = newPayload; - edm::LogInfo(m_name) << "Added empty payload with IOV " << iov << " ( " - << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; - } - } - - // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map - void addPayloadToBuffer(cond::OMSServiceResultRef& row) { - auto startTime = row.get("start_time"); - auto delivLumi = row.get("delivered_lumi"); - auto recLumi = row.get("recorded_lumi"); - auto runNumber = std::stoul(row.get("run_number")); - auto lsNumber = std::stoul(row.get("lumisection_number")); - auto lumiid = cond::time::lumiTime(runNumber, lsNumber); - - LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill(); - m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo)); - if (!m_endFillMode) { - m_timestampToLumiid.insert(std::make_pair(cond::time::from_boost(startTime), lumiid)); - } - LHCInfoPerFill& payload = *thisLumiSectionInfo; - payload.setDelivLumi(delivLumi); - payload.setRecLumi(recLumi); - } - - void convertBufferedIovsToLumiid(std::map timestampToLumiid) { - for (auto& item : m_tmpBuffer) { - // Check if the lumiid IOV corresponding to the timestamp is present in the map - if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) { - throw cms::Exception("LHCInfoPerFillPopConSourceHandler") - << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n"; - } - // Update the buffer with the lumiid-type IOV - item.first = timestampToLumiid.at(item.first); - } - } - - size_t getLumiData(const cond::OMSService& oms, - unsigned short fillId, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - auto query = oms.query("lumisections"); - query->addOutputVars( - {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"}); - query->filterEQ("fill_number", fillId); - query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->filterEQ("beams_stable", "true"); - query->limit(kLumisectionsQueryLimit); - if (query->execute()) { - auto queryResult = query->result(); - edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections with STABLE BEAM during the fill " - << fillId; - - if (!queryResult.empty()) { - if (m_endFillMode) { - auto firstRow = queryResult.front(); - addPayloadToBuffer(firstRow); - } - - auto lastRow = queryResult.back(); - addPayloadToBuffer(lastRow); - } - } - return 0; - } - - void getDipData(const cond::OMSService& oms, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - // unsure how to handle this. - // the old implementation is not helping: apparently it is checking only the bunchconfiguration for the first diptime set of values... - auto query1 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam1"); - query1->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - //If all this data is needed and saved properly the limit has to be set: query1->limit(...) - if (query1->execute()) { - auto res = query1->result(); - if (!res.empty()) { - std::bitset bunchConfiguration1(0ULL); - auto row = *res.begin(); - auto vbunchConf1 = row.getArray("value"); - for (auto vb : vbunchConf1) { - if (vb != 0) { - unsigned short slot = (vb - 1) / 10 + 1; - bunchConfiguration1[slot] = true; - } - } - m_fillPayload->setBunchBitsetForBeam1(bunchConfiguration1); - } - } - auto query2 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam2"); - query2->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - if (query2->execute()) { - auto res = query2->result(); - if (!res.empty()) { - std::bitset bunchConfiguration2(0ULL); - auto row = *res.begin(); - auto vbunchConf2 = row.getArray("value"); - for (auto vb : vbunchConf2) { - if (vb != 0) { - unsigned short slot = (vb - 1) / 10 + 1; - bunchConfiguration2[slot] = true; - } - } - m_fillPayload->setBunchBitsetForBeam2(bunchConfiguration2); - } - } - - auto query3 = oms.query("diplogger/dip/CMS/LHC/LumiPerBunch"); - query3->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); - //This query is limited to 100 rows, but currently only one is used - if (query3->execute()) { - auto res = query3->result(); - if (!res.empty()) { - std::vector lumiPerBX; - auto row = *res.begin(); - auto lumiBunchInst = row.getArray("lumi_bunch_inst"); - for (auto lb : lumiBunchInst) { - if (lb != 0.) { - lumiPerBX.push_back(lb); - } - } - m_fillPayload->setLumiPerBX(lumiPerBX); - } - } - } - - bool getCTPPSData(cond::persistency::Session& session, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - //run the fifth query against the CTPPS schema - //Initializing the CMS_CTP_CTPPS_COND schema. - coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); - //execute query for CTPPS Data - std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); - //FROM clause - CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); - //SELECT clause - CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); - CTPPSDataQuery->addToOutputList(std::string("LHC_STATE")); - CTPPSDataQuery->addToOutputList(std::string("LHC_COMMENT")); - if (m_debug) { - CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); - } - //WHERE CLAUSE - coral::AttributeList CTPPSDataBindVariables; - CTPPSDataBindVariables.extend(std::string("beginFillTime")); - CTPPSDataBindVariables.extend(std::string("endFillTime")); - CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); - CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); - std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); - CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); - //ORDER BY clause - CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); - //define query output - coral::AttributeList CTPPSDataOutput; - CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); - CTPPSDataOutput.extend(std::string("LHC_STATE")); - CTPPSDataOutput.extend(std::string("LHC_COMMENT")); - if (m_debug) { - CTPPSDataOutput.extend(std::string("RUN_NUMBER")); - CTPPSDataOutput.extend(std::string("LUMI_SECTION")); - } - CTPPSDataQuery->defineOutput(CTPPSDataOutput); - //execute the query - coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); - cond::Time_t dipTime = 0; - std::string lhcState = "", lhcComment = "", ctppsStatus = ""; - - //debug informations - unsigned int lumiSection = 0; - cond::Time_t runNumber = 0; - cond::Time_t savedDipTime = 0; - unsigned int savedLumiSection = 0; - cond::Time_t savedRunNumber = 0; - - bool ret = false; - LumiSectionFilter filter(m_tmpBuffer); - while (CTPPSDataCursor.next()) { - if (m_debug) { - std::ostringstream CTPPS; - CTPPSDataCursor.currentRow().toOutputStream(CTPPS); - } - coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; - if (!dipTimeAttribute.isNull()) { - dipTime = cond::time::from_boost(dipTimeAttribute.data().time()); - if (filter.process(dipTime)) { - ret = true; - coral::Attribute const& lhcStateAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_STATE")]; - if (!lhcStateAttribute.isNull()) { - lhcState = lhcStateAttribute.data(); - } - coral::Attribute const& lhcCommentAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_COMMENT")]; - if (!lhcCommentAttribute.isNull()) { - lhcComment = lhcCommentAttribute.data(); - } - - if (m_debug) { - coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; - if (!runNumberAttribute.isNull()) { - runNumber = runNumberAttribute.data(); - } - coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; - if (!lumiSectionAttribute.isNull()) { - lumiSection = lumiSectionAttribute.data(); - } - } - - for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { - // set the current values to all of the payloads of the lumi section samples after the current since - LHCInfoPerFill& payload = *(it->second); - payload.setLhcState(lhcState); - payload.setLhcComment(lhcComment); - payload.setCtppsStatus(ctppsStatus); - - if (m_debug) { - savedDipTime = dipTime; - savedLumiSection = lumiSection; - savedRunNumber = runNumber; - } - } - } - } - } - if (m_debug) { - edm::LogInfo(m_name) << "Last assigned: " - << "DipTime: " << savedDipTime << " " - << "LumiSection: " << savedLumiSection << " " - << "RunNumber: " << savedRunNumber; - } - return ret; - } - - bool getEcalData(cond::persistency::Session& session, - const boost::posix_time::ptime& lowerTime, - const boost::posix_time::ptime& upperTime) { - //run the sixth query against the CMS_DCS_ENV_PVSS_COND schema - //Initializing the CMS_DCS_ENV_PVSS_COND schema. - coral::ISchema& ECAL = session.nominalSchema(); - //start the transaction against the fill logging schema - //execute query for ECAL Data - std::unique_ptr ECALDataQuery(ECAL.newQuery()); - //FROM clause - ECALDataQuery->addToTableList(std::string("BEAM_PHASE")); - //SELECT clause - ECALDataQuery->addToOutputList(std::string("CHANGE_DATE")); - ECALDataQuery->addToOutputList(std::string("DIP_value")); - ECALDataQuery->addToOutputList(std::string("element_nr")); - ECALDataQuery->addToOutputList(std::string("VALUE_NUMBER")); - //WHERE CLAUSE - coral::AttributeList ECALDataBindVariables; - ECALDataBindVariables.extend(std::string("lowerTime")); - ECALDataBindVariables.extend(std::string("upperTime")); - ECALDataBindVariables[std::string("lowerTime")].data() = coral::TimeStamp(lowerTime); - ECALDataBindVariables[std::string("upperTime")].data() = coral::TimeStamp(upperTime); - std::string conditionStr = std::string( - "(DIP_value LIKE '%beamPhaseMean%' OR DIP_value LIKE '%cavPhaseMean%') AND CHANGE_DATE >= :lowerTime AND " - "CHANGE_DATE < :upperTime"); - - ECALDataQuery->setCondition(conditionStr, ECALDataBindVariables); - //ORDER BY clause - ECALDataQuery->addToOrderList(std::string("CHANGE_DATE")); - ECALDataQuery->addToOrderList(std::string("DIP_value")); - ECALDataQuery->addToOrderList(std::string("element_nr")); - //define query output - coral::AttributeList ECALDataOutput; - ECALDataOutput.extend(std::string("CHANGE_DATE")); - ECALDataOutput.extend(std::string("DIP_value")); - ECALDataOutput.extend(std::string("element_nr")); - ECALDataOutput.extend(std::string("VALUE_NUMBER")); - //ECALDataQuery->limitReturnedRows( 14256 ); //3564 entries per vector. - ECALDataQuery->defineOutput(ECALDataOutput); - //execute the query - coral::ICursor& ECALDataCursor = ECALDataQuery->execute(); - cond::Time_t changeTime = 0; - cond::Time_t firstTime = 0; - std::string dipVal = ""; - unsigned int elementNr = 0; - float value = 0.; - std::set initializedVectors; - LumiSectionFilter filter(m_tmpBuffer); - bool ret = false; - if (m_prevPayload.get()) { - for (auto& lumiSlot : m_tmpBuffer) { - lumiSlot.second->setBeam1VC(m_prevPayload->beam1VC()); - lumiSlot.second->setBeam2VC(m_prevPayload->beam2VC()); - lumiSlot.second->setBeam1RF(m_prevPayload->beam1RF()); - lumiSlot.second->setBeam2RF(m_prevPayload->beam2RF()); - } - } - std::map iovMap; - if (m_tmpBuffer.empty()) { - return ret; - } - cond::Time_t lowerLumi = m_tmpBuffer.front().first; - while (ECALDataCursor.next()) { - if (m_debug) { - std::ostringstream ECAL; - ECALDataCursor.currentRow().toOutputStream(ECAL); - } - coral::Attribute const& changeDateAttribute = ECALDataCursor.currentRow()[std::string("CHANGE_DATE")]; - if (!changeDateAttribute.isNull()) { - ret = true; - boost::posix_time::ptime chTime = changeDateAttribute.data().time(); - // move the first IOV found to the start of the fill interval selected - if (changeTime == 0) { - firstTime = cond::time::from_boost(chTime); - } - changeTime = cond::time::from_boost(chTime); - cond::Time_t iovTime = changeTime; - if (changeTime == firstTime) - iovTime = lowerLumi; - coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")]; - coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")]; - coral::Attribute const& elementNrAttribute = ECALDataCursor.currentRow()[std::string("element_nr")]; - if (!dipValAttribute.isNull() and !valueNumberAttribute.isNull()) { - dipVal = dipValAttribute.data(); - elementNr = elementNrAttribute.data(); - value = valueNumberAttribute.data(); - if (std::isnan(value)) - value = 0.; - if (filter.process(iovTime)) { - iovMap.insert(std::make_pair(changeTime, filter.current()->first)); - for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { - LHCInfoPerFill& payload = *(it->second); - theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors); - } - } - } - } - } - if (m_debug) { - for (auto& im : iovMap) { - edm::LogInfo(m_name) << "Found iov=" << im.first << " (" << cond::time::to_boost(im.first) << " ) moved to " - << im.second << " ( " << cond::time::to_boost(im.second) << " )"; - } - } - return ret; - } - -private: - bool m_debug; - // starting date for sampling - boost::posix_time::ptime m_startTime; - boost::posix_time::ptime m_endTime; - bool m_endFillMode = true; - std::string m_name; - //for reading from relational database source - std::string m_connectionString, m_ecalConnectionString; - std::string m_authpath; - std::string m_omsBaseUrl; - std::unique_ptr m_fillPayload; - std::shared_ptr m_prevPayload; - std::vector>> m_tmpBuffer; - bool m_lastPayloadEmpty = false; - // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs - std::map m_timestampToLumiid; -}; diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc new file mode 100644 index 0000000000000..709362592d160 --- /dev/null +++ b/CondTools/RunInfo/plugins/LHCInfoPerLSOnlinePopConAnalyzer.cc @@ -0,0 +1,7 @@ +#include "CondCore/PopCon/interface/OnlinePopConAnalyzer.h" +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +using LHCInfoPerLSOnlinePopConAnalyzer = popcon::OnlinePopConAnalyzer; + +DEFINE_FWK_MODULE(LHCInfoPerLSOnlinePopConAnalyzer); diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc index dc5fe9b298c8e..9a1fcbaa4dfdd 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc @@ -1,589 +1,7 @@ -#include "CondCore/CondDB/interface/ConnectionPool.h" -#include "CondCore/CondDB/interface/Types.h" #include "CondCore/PopCon/interface/PopConAnalyzer.h" -#include "CondCore/PopCon/interface/PopConSourceHandler.h" -#include "CondFormats/Common/interface/TimeConversions.h" -#include "CondFormats/RunInfo/interface/LHCInfoPerLS.h" -#include "CondTools/RunInfo/interface/LHCInfoHelper.h" -#include "CondTools/RunInfo/interface/OMSAccess.h" -#include "CoralBase/Attribute.h" -#include "CoralBase/AttributeList.h" -#include "CoralBase/AttributeSpecification.h" -#include "CoralBase/TimeStamp.h" +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" #include "FWCore/Framework/interface/MakerMacros.h" -#include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ParameterSet/interface/ParameterSetfwd.h" -#include "RelationalAccess/ICursor.h" -#include "RelationalAccess/IQuery.h" -#include "RelationalAccess/ISchema.h" -#include "RelationalAccess/ISessionProxy.h" - -using std::make_pair; -using std::pair; - -class LHCInfoPerLSPopConSourceHandler; typedef popcon::PopConAnalyzer LHCInfoPerLSPopConAnalyzer; -//define this as a plug-in -DEFINE_FWK_MODULE(LHCInfoPerLSPopConAnalyzer); - -namespace theLHCInfoPerLSImpl { - bool comparePayloads(const LHCInfoPerLS& rhs, const LHCInfoPerLS& lhs) { - if (rhs.fillNumber() != lhs.fillNumber() || rhs.runNumber() != lhs.runNumber() || - rhs.crossingAngleX() != lhs.crossingAngleX() || rhs.crossingAngleY() != lhs.crossingAngleY() || - rhs.betaStarX() != lhs.betaStarX() || rhs.betaStarY() != lhs.betaStarY()) { - return false; - } - return true; - } - - size_t transferPayloads(const std::vector>>& buffer, - std::map>& iovsToTransfer, - std::shared_ptr& prevPayload, - const std::map, pair>& lsIdMap, - cond::Time_t startStableBeamTime, - cond::Time_t endStableBeamTime) { - int lsMissingInPPS = 0; - int xAngleBothZero = 0, xAngleBothNonZero = 0, xAngleNegative = 0; - int betaNegative = 0; - size_t niovs = 0; - std::stringstream condIovs; - std::stringstream missingLsList; - for (auto& iov : buffer) { - bool add = false; - auto payload = iov.second; - cond::Time_t since = iov.first; - if (iovsToTransfer.empty()) { - add = true; - } else { - LHCInfoPerLS& lastAdded = *iovsToTransfer.rbegin()->second; - if (!comparePayloads(lastAdded, *payload)) { - add = true; - } - } - auto id = make_pair(payload->runNumber(), payload->lumiSection()); - bool stableBeam = since >= startStableBeamTime && since <= endStableBeamTime; - bool isMissing = lsIdMap.find(id) != lsIdMap.end() && id != lsIdMap.at(id); - if (stableBeam && isMissing) { - missingLsList << id.first << "_" << id.second << " "; - lsMissingInPPS += isMissing; - } - if (add && !isMissing) { - niovs++; - if (stableBeam) { - if (payload->crossingAngleX() == 0 && payload->crossingAngleY() == 0) - xAngleBothZero++; - if (payload->crossingAngleX() != 0 && payload->crossingAngleY() != 0) - xAngleBothNonZero++; - if (payload->crossingAngleX() < 0 || payload->crossingAngleY() < 0) - xAngleNegative++; - if (payload->betaStarX() < 0 || payload->betaStarY() < 0) - betaNegative++; - } - - condIovs << since << " "; - iovsToTransfer.insert(make_pair(since, payload)); - prevPayload = iov.second; - } - } - unsigned short fillNumber = (!buffer.empty()) ? buffer.front().second->fillNumber() : 0; - if (lsMissingInPPS > 0) { - edm::LogWarning("transferPayloads") - << "Number of stable beam LS in OMS without corresponding record in PPS DB for fill " << fillNumber << ": " - << lsMissingInPPS; - edm::LogWarning("transferPayloads") - << "Stable beam LS in OMS without corresponding record in PPS DB (run_LS): " << missingLsList.str(); - } - if (xAngleBothZero > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with crossingAngle == 0 for both X and Y for fill " << fillNumber << ": " - << xAngleBothZero; - } - if (xAngleBothNonZero > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with crossingAngle != 0 for both X and Y for fill " << fillNumber << ": " - << xAngleBothNonZero; - } - if (xAngleNegative > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with negative crossingAngle for fill " << fillNumber << ": " << xAngleNegative; - } - if (betaNegative > 0) { - edm::LogWarning("transferPayloads") - << "Number of payloads written with negative betaSta for fill " << fillNumber << ": " << betaNegative; - } - - edm::LogInfo("transferPayloads") << "TRANSFERED COND IOVS: " << condIovs.str(); - return niovs; - } - -} // namespace theLHCInfoPerLSImpl - -class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler { -public: - LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset) - : m_debug(pset.getUntrackedParameter("debug", false)), - m_startTime(), - m_endTime(), - m_endFillMode(pset.getUntrackedParameter("endFill", true)), - m_name(pset.getUntrackedParameter("name", "LHCInfoPerLSPopConSourceHandler")), - m_connectionString(pset.getUntrackedParameter("connectionString", "")), - m_authpath(pset.getUntrackedParameter("authenticationPath", "")), - m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), - m_fillPayload(), - m_prevPayload(), - m_tmpBuffer() { - if (!pset.getUntrackedParameter("startTime").empty()) { - m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); - } - boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); - m_endTime = now; - if (!pset.getUntrackedParameter("endTime").empty()) { - m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); - if (m_endTime > now) - m_endTime = now; - } - } - - ~LHCInfoPerLSPopConSourceHandler() override = default; - - void getNewObjects() override { - //if a new tag is created, transfer fake fill from 1 to the first fill for the first time - if (tagInfo().size == 0) { - edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; - } else { - //check what is already inside the database - edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size - << ", last object valid since " << tagInfo().lastInterval.since << " ( " - << boost::posix_time::to_iso_extended_string( - cond::time::to_boost(tagInfo().lastInterval.since)) - << " ); from " << m_name << "::getNewObjects"; - } - - cond::Time_t lastSince = tagInfo().lastInterval.since; - if (tagInfo().isEmpty()) { - // for a new or empty tag, an empty payload should be added on top with since=1 - if (m_endFillMode) { - addEmptyPayload(1); - lastSince = 1; - } else { - addEmptyPayload(cond::time::lumiTime(1, 1)); - lastSince = cond::time::lumiTime(1, 1); - } - } else { - edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " - << m_name << "::getNewObjects"; - } - - //retrieve the data from the relational database source - cond::persistency::ConnectionPool connection; - //configure the connection - if (m_debug) { - connection.setMessageVerbosity(coral::Debug); - } else { - connection.setMessageVerbosity(coral::Error); - } - connection.setAuthenticationPath(m_authpath); - connection.configure(); - //create the sessions - cond::persistency::Session session = connection.createSession(m_connectionString, false); - // fetch last payload when available - if (!tagInfo().lastInterval.payloadId.empty()) { - cond::persistency::Session session3 = dbSession(); - session3.transaction().start(true); - m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); - session3.transaction().commit(); - - // find startFillTime and endFillTime of the most recent fill already saved in the tag - if (m_prevPayload->fillNumber() != 0) { - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - query->addOutputVar("end_time"); - query->addOutputVar("start_time"); - query->filterEQ("fill_number", m_prevPayload->fillNumber()); - bool foundFill = query->execute(); - if (foundFill) { - auto result = query->result(); - - if (!result.empty()) { - std::string endTimeStr = (*result.begin()).get("end_time"); - m_prevEndFillTime = - (endTimeStr == "null") - ? 0 - : cond::time::from_boost((*result.begin()).get("end_time")); - auto startFillTime = (*result.begin()).get("start_time"); - m_prevStartFillTime = cond::time::from_boost(startFillTime); - } else { - foundFill = false; - } - } - if (!foundFill) { - edm::LogError(m_name) << "Could not find end time of fill #" << m_prevPayload->fillNumber(); - } - } else { - m_prevEndFillTime = 0; - m_prevStartFillTime = 0; - } - } - - boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); - cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); - - cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); - cond::Time_t nextFillSearchTimestamp = std::max(startTimestamp, m_endFillMode ? lastSince : m_prevEndFillTime); - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); - - while (true) { - if (nextFillSearchTimestamp >= executionTimeIov) { - edm::LogInfo(m_name) << "Sampling ended at the time " - << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); - break; - } - boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); - boost::posix_time::ptime startSampleTime; - boost::posix_time::ptime endSampleTime; - - cond::OMSService oms; - oms.connect(m_omsBaseUrl); - auto query = oms.query("fills"); - - if (!m_endFillMode and m_prevPayload->fillNumber() and m_prevEndFillTime == 0ULL) { - // continue processing unfinished fill with some payloads already in the tag - edm::LogInfo(m_name) << "Searching started fill #" << m_prevPayload->fillNumber(); - query->filterEQ("fill_number", m_prevPayload->fillNumber()); - bool foundFill = query->execute(); - if (foundFill) - foundFill = makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogError(m_name) << "Could not find fill #" << m_prevPayload->fillNumber(); - break; - } - startSampleTime = cond::time::to_boost(lastSince); - } else { - edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); - query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) { - query->filterGE("start_time", nextFillSearchTime); - } else { - query->filterGT("start_time", nextFillSearchTime); - } - - query->filterLT("start_time", m_endTime); - if (m_endFillMode) - query->filterNotNull("end_time"); - bool foundFill = query->execute(); - if (foundFill) - foundFill = makeFillPayload(m_fillPayload, query->result()); - if (!foundFill) { - edm::LogInfo(m_name) << "No fill found - END of job."; - break; - } - startSampleTime = cond::time::to_boost(m_startFillTime); - } - - unsigned short lhcFill = m_fillPayload->fillNumber(); - bool ongoingFill = m_endFillTime == 0ULL; - if (ongoingFill) { - edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " - << cond::time::to_boost(m_startFillTime); - endSampleTime = executionTime; - nextFillSearchTimestamp = executionTimeIov; - } else { - edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(m_startFillTime) - << " ending at " << cond::time::to_boost(m_endFillTime); - endSampleTime = cond::time::to_boost(m_endFillTime); - nextFillSearchTimestamp = m_endFillTime; - } - - if (m_endFillMode || ongoingFill) { - getLumiData(oms, lhcFill, startSampleTime, endSampleTime); - - if (!m_tmpBuffer.empty()) { - boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); - boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); - edm::LogInfo(m_name) << "First buffered lumi starts at " << flumiStart << " last lumi starts at " - << flumiStop; - session.transaction().start(true); - getCTPPSData(session, startSampleTime, endSampleTime); - session.transaction().commit(); - } - } - - size_t niovs = theLHCInfoPerLSImpl::transferPayloads( - m_tmpBuffer, m_iovs, m_prevPayload, m_lsIdMap, m_startStableBeamTime, m_endStableBeamTime); - edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; - if (niovs) { - m_prevEndFillTime = m_endFillTime; - m_prevStartFillTime = m_startFillTime; - } - m_tmpBuffer.clear(); - m_lsIdMap.clear(); - if (m_prevPayload->fillNumber() and !ongoingFill) { - if (m_endFillMode) { - addEmptyPayload(m_endFillTime); - } else { - addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); - } - } - } - } - std::string id() const override { return m_name; } - - static constexpr unsigned int kLumisectionsQueryLimit = 4000; - -private: - void addEmptyPayload(cond::Time_t iov) { - bool add = false; - if (m_iovs.empty()) { - if (!m_lastPayloadEmpty) - add = true; - } else { - auto lastAdded = m_iovs.rbegin()->second; - if (lastAdded->fillNumber() != 0) { - add = true; - } - } - if (add) { - auto newPayload = std::make_shared(); - m_iovs.insert(make_pair(iov, newPayload)); - m_prevPayload = newPayload; - m_prevEndFillTime = 0; - m_prevStartFillTime = 0; - edm::LogInfo(m_name) << "Added empty payload with IOV" << iov << " ( " - << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; - } - } - - bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { - bool ret = false; - if (!queryResult.empty()) { - auto row = *queryResult.begin(); - auto currentFill = row.get("fill_number"); - m_startFillTime = cond::time::from_boost(row.get("start_time")); - std::string endTimeStr = row.get("end_time"); - m_endFillTime = - (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); - m_startStableBeamTime = cond::time::from_boost(row.get("start_stable_beam")); - m_endStableBeamTime = cond::time::from_boost(row.get("end_stable_beam")); - targetPayload = std::make_unique(); - targetPayload->setFillNumber(currentFill); - ret = true; - } - return ret; - } - - void addPayloadToBuffer(cond::OMSServiceResultRef& row) { - auto lumiTime = row.get("start_time"); - LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); - thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); - thisLumiSectionInfo->setRunNumber(std::stoul(row.get("run_number"))); - m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); - if (m_endFillMode) { - m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); - } else { - m_tmpBuffer.emplace_back( - make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()), - thisLumiSectionInfo)); - } - } - - size_t bufferAllLS(const cond::OMSServiceResult& queryResult) { - for (auto r : queryResult) { - addPayloadToBuffer(r); - } - return queryResult.size(); - } - - size_t bufferFirstStableBeamLS(const cond::OMSServiceResult& queryResult) { - for (auto r : queryResult) { - if (r.get("beams_stable") == "true") { - addPayloadToBuffer(r); - edm::LogInfo(m_name) << "Buffered first lumisection of stable beam: LS: " - << r.get("lumisection_number") - << " run: " << r.get("run_number"); - return 1; - } - } - return 0; - } - - size_t getLumiData(const cond::OMSService& oms, - unsigned short fillId, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - auto query = oms.query("lumisections"); - query->addOutputVars({"start_time", "run_number", "beams_stable", "lumisection_number"}); - query->filterEQ("fill_number", fillId); - query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->limit(kLumisectionsQueryLimit); - size_t nlumi = 0; - if (query->execute()) { - auto queryResult = query->result(); - if (m_endFillMode) { - nlumi = bufferAllLS(queryResult); - } else if (!queryResult.empty()) { - auto newestPayload = queryResult.back(); - if (newestPayload.get("beams_stable") == "true") { - addPayloadToBuffer(newestPayload); - nlumi = 1; - edm::LogInfo(m_name) << "Buffered most recent lumisection:" - << " LS: " << newestPayload.get("lumisection_number") - << " run: " << newestPayload.get("run_number"); - } - } - edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections during the fill " << fillId; - } else { - edm::LogInfo(m_name) << "OMS query for lumisections of fill " << fillId << "failed, status:" << query->status(); - } - return nlumi; - } - - bool getCTPPSData(cond::persistency::Session& session, - const boost::posix_time::ptime& beginFillTime, - const boost::posix_time::ptime& endFillTime) { - //run the fifth query against the CTPPS schema - //Initializing the CMS_CTP_CTPPS_COND schema. - coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); - //execute query for CTPPS Data - std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); - //FROM clause - CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); - //SELECT clause - CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); - CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); - CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("FILL_NUMBER")); - CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_X_URAD")); - CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_Y_URAD")); - CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_X_M")); - CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_Y_M")); - //WHERE CLAUSE - coral::AttributeList CTPPSDataBindVariables; - CTPPSDataBindVariables.extend(std::string("beginFillTime")); - CTPPSDataBindVariables.extend(std::string("endFillTime")); - CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); - CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); - std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); - CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); - //ORDER BY clause - CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); - //define query output - coral::AttributeList CTPPSDataOutput; - CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); - CTPPSDataOutput.extend(std::string("LUMI_SECTION")); - CTPPSDataOutput.extend(std::string("RUN_NUMBER")); - CTPPSDataOutput.extend(std::string("FILL_NUMBER")); - CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_X_URAD")); - CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_Y_URAD")); - CTPPSDataOutput.extend(std::string("BETA_STAR_P5_X_M")); - CTPPSDataOutput.extend(std::string("BETA_STAR_P5_Y_M")); - CTPPSDataQuery->defineOutput(CTPPSDataOutput); - //execute the query - coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); - unsigned int lumiSection = 0; - cond::Time_t runNumber = 0; - int fillNumber = 0; - float crossingAngleX = 0., betaStarX = 0.; - float crossingAngleY = 0., betaStarY = 0.; - - bool ret = false; - int wrongFillNumbers = 0; - std::stringstream wrongFills; - std::vector>>::iterator current = m_tmpBuffer.begin(); - while (CTPPSDataCursor.next()) { - if (m_debug) { - std::ostringstream CTPPS; - CTPPSDataCursor.currentRow().toOutputStream(CTPPS); - } - coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; - if (!dipTimeAttribute.isNull()) { - ret = true; - coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; - if (!lumiSectionAttribute.isNull()) { - lumiSection = lumiSectionAttribute.data(); - } - coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; - if (!runNumberAttribute.isNull()) { - runNumber = runNumberAttribute.data(); - } - coral::Attribute const& fillNumberAttribute = CTPPSDataCursor.currentRow()[std::string("FILL_NUMBER")]; - if (!fillNumberAttribute.isNull()) { - fillNumber = fillNumberAttribute.data(); - } - coral::Attribute const& crossingAngleXAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; - if (!crossingAngleXAttribute.isNull()) { - crossingAngleX = crossingAngleXAttribute.data(); - } - coral::Attribute const& crossingAngleYAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; - if (!crossingAngleYAttribute.isNull()) { - crossingAngleY = crossingAngleYAttribute.data(); - } - coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; - if (!betaStarXAttribute.isNull()) { - betaStarX = betaStarXAttribute.data(); - } - coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; - if (!betaStarYAttribute.isNull()) { - betaStarY = betaStarYAttribute.data(); - } - if (current != m_tmpBuffer.end() && current->second->fillNumber() != fillNumber) { - wrongFills << "( " << runNumber << "_" << lumiSection << " fill: OMS: " << current->second->fillNumber() - << " PPSdb: " << fillNumber << " ) "; - wrongFillNumbers++; - } - for (; - current != m_tmpBuffer.end() && make_pair(current->second->runNumber(), current->second->lumiSection()) <= - make_pair(runNumber, lumiSection); - current++) { - LHCInfoPerLS& payload = *(current->second); - payload.setCrossingAngleX(crossingAngleX); - payload.setCrossingAngleY(crossingAngleY); - payload.setBetaStarX(betaStarX); - payload.setBetaStarY(betaStarY); - payload.setLumiSection(lumiSection); - payload.setRunNumber(runNumber); - if (m_lsIdMap.find(make_pair(payload.runNumber(), payload.lumiSection())) != m_lsIdMap.end()) { - m_lsIdMap[make_pair(payload.runNumber(), payload.lumiSection())] = make_pair(runNumber, lumiSection); - } - } - } - } - if (wrongFillNumbers) { - edm::LogWarning("getCTPPSData") << "Number of records from PPS DB with fillNumber different from OMS: " - << wrongFillNumbers; - edm::LogWarning("getCTPPSData") << "Records from PPS DB with fillNumber different from OMS: " << wrongFills.str(); - } - return ret; - } -private: - bool m_debug; - // starting date for sampling - boost::posix_time::ptime m_startTime; - boost::posix_time::ptime m_endTime; - bool m_endFillMode = true; - std::string m_name; - //for reading from relational database source - std::string m_connectionString; - std::string m_authpath; - std::string m_omsBaseUrl; - std::unique_ptr m_fillPayload; - std::shared_ptr m_prevPayload; - cond::Time_t m_startFillTime; - cond::Time_t m_endFillTime; - cond::Time_t m_prevEndFillTime; - cond::Time_t m_prevStartFillTime; - cond::Time_t m_startStableBeamTime; - cond::Time_t m_endStableBeamTime; - std::vector>> m_tmpBuffer; - bool m_lastPayloadEmpty = false; - //mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to the IDs they've been assignd from PPS DB - //value pair(-1, -1) means lumisection corresponding to the key exists in OMS but no lumisection was matched from PPS - std::map, pair> m_lsIdMap; -}; +DEFINE_FWK_MODULE(LHCInfoPerLSPopConAnalyzer); \ No newline at end of file diff --git a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py index d71362ecebfa8..22406260f051a 100644 --- a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py +++ b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py @@ -3,10 +3,6 @@ import FWCore.ParameterSet.VarParsing as VarParsing process = cms.Process("LHCInfoPerFillPopulator") from CondCore.CondDB.CondDB_cfi import * -#process.load("CondCore.DBCommon.CondDBCommon_cfi") -#process.CondDBCommon.connect = 'sqlite_file:lhcinfoperls_pop_test.db' -#process.CondDBCommon.DBParameters.authenticationPath = '.' -#process.CondDBCommon.DBParameters.messageLevel=cms.untracked.int32(1) sourceConnection = 'oracle://cms_omds_adg/CMS_RUNINFO_R' if socket.getfqdn().find('.cms') != -1: @@ -60,6 +56,51 @@ processes only fills starting before endTime; default to empty string which sets no restriction""" ) + +options.register( 'sourceConnection' + , "oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """beam data source connection string (aka PPS db)""" + ) +options.register( 'ecalConnection' + , "oracle://cms_orcon_adg/CMS_DCS_ENV_PVSS_COND" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """ecal data source connection string""" + ) +options.register( 'oms' + , "http://vocms0184.cern.ch/agg/api/v1" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """OMS base URL""" + ) + +#duringFill mode specific: +options.register( 'lastLumiFile' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: path to file with lumiid to override the last lumisection processed by HLT. + Used for testing. Leave empty for production behaviour (getting this info from OMS)""" + ) +options.register( 'frontierKey' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: run-unique key for writing with OnlinePopCon + (used for confirming proper upload)""" + ) + + +# so far there was no need to use option, added just in case +options.register( 'authenticationPath' + , "" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """for now this option was always left empty""" + ) + options.parseArguments() if options.mode is None: raise ValueError("mode argument not provided. Supported modes are: duringFill endFill") @@ -68,6 +109,7 @@ CondDBConnection = CondDB.clone( connect = cms.string( options.destinationConnection ) ) CondDBConnection.DBParameters.messageLevel = cms.untracked.int32( options.messageLevel ) +CondDBConnection.DBParameters.authenticationPath = cms.untracked.string(options.authenticationPath) process.MessageLogger = cms.Service("MessageLogger", cout = cms.untracked.PSet(threshold = cms.untracked.string('INFO')), @@ -87,30 +129,52 @@ else: timetype = 'lumiid' -process.PoolDBOutputService = cms.Service("PoolDBOutputService", - CondDBConnection, - timetype = cms.untracked.string(timetype), - toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'), - tag = cms.string( options.tag ) - ) - ) - ) +if options.mode == "endFill": + process.PoolDBOutputService = cms.Service("PoolDBOutputService", + CondDBConnection, + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'), + tag = cms.string( options.tag ) + ) + ) + ) +else: + process.OnlineDBOutputService = cms.Service("OnlineDBOutputService", + CondDBConnection, + preLoadConnectionString = cms.untracked.string('frontier://FrontierProd/CMS_CONDITIONS' + if not options.destinationConnection.startswith('sqlite') + else options.destinationConnection ), + lastLumiFile = cms.untracked.string(options.lastLumiFile), + omsServiceUrl = cms.untracked.string('http://cmsoms-eventing.cms:9949/urn:xdaq-application:lid=100/getRunAndLumiSection' + if not options.lastLumiFile else "" ), + # runNumber = cms.untracked.uint64(384468), #not used in production, the last LS processed is set as the 1st LS of this + #run if the omsServiceUrl is empty and file specified in lastLumiFile is empty + latency = cms.untracked.uint32(2), + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet( + record = cms.string('LHCInfoPerFillRcd'), + tag = cms.string( options.tag ), + onlyAppendUpdatePolicy = cms.untracked.bool(True) + )), + frontierKey = cms.untracked.string(options.frontierKey) +) + -process.Test1 = cms.EDAnalyzer("LHCInfoPerFillPopConAnalyzer", +process.Test1 = cms.EDAnalyzer("LHCInfoPerFillPopConAnalyzer" if options.mode == "endFill" else "LHCInfoPerFillOnlinePopConAnalyzer", SinceAppendMode = cms.bool(True), record = cms.string('LHCInfoPerFillRcd'), name = cms.untracked.string('LHCInfo'), - Source = cms.PSet(fill = cms.untracked.uint32(6417), + Source = cms.PSet( startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - endFill = cms.untracked.bool(True if options.mode == "endFill" else False), + endFill = cms.untracked.bool(options.mode == "endFill"), name = cms.untracked.string("LHCInfoPerFillPopConSourceHandler"), - connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), - ecalConnectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_DCS_ENV_PVSS_COND"), - omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), - authenticationPath = cms.untracked.string(""), + connectionString = cms.untracked.string(options.sourceConnection), + ecalConnectionString = cms.untracked.string(options.ecalConnection), + omsBaseUrl = cms.untracked.string(options.oms), + authenticationPath = cms.untracked.string(options.authenticationPath), debug=cms.untracked.bool(False) - ), + ), loggingOn = cms.untracked.bool(True), IsDestDbCheckedInQueryLog = cms.untracked.bool(False) ) diff --git a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py index 603c7f514d4ee..a58598f1d3ef1 100644 --- a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py +++ b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py @@ -3,10 +3,6 @@ import FWCore.ParameterSet.VarParsing as VarParsing process = cms.Process("LHCInfoPerLSPopulator") from CondCore.CondDB.CondDB_cfi import * -#process.load("CondCore.DBCommon.CondDBCommon_cfi") -#process.CondDBCommon.connect = 'sqlite_file:lhcinfoperls_pop_test.db' -#process.CondDBCommon.DBParameters.authenticationPath = '.' -#process.CondDBCommon.DBParameters.messageLevel=cms.untracked.int32(1) sourceConnection = 'oracle://cms_omds_adg/CMS_RUNINFO_R' if socket.getfqdn().find('.cms') != -1: @@ -60,6 +56,109 @@ processes only fills starting before endTime; default to empty string which sets no restriction""" ) + +options.register( 'sourceConnection' + , "oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """beam data source connection string (aka PPS db) + It's the source of crossing angle and beta * data""" + ) +options.register( 'oms' + , "http://vocms0184.cern.ch/agg/api/v1" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """OMS base URL""" + ) + +#duringFill mode specific: +options.register( 'lastLumiFile' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: path to file with lumiid to override the last lumisection processed by HLT. + Used for testing. Leave empty for production behaviour (getting this info from OMS)""" + ) +options.register( 'frontierKey' + , '' + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """duringFill only: run-unique key for writing with OnlinePopCon + (used for confirming proper upload)""" + ) +options.register('offsetLS' + , 2 + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.int + , """duringFill only: offset between lastLumi (last LS processed by HLT or overriden by lastLumiFile) + and the IOV of the payload to be uploaded""" + ) +options.register( 'debugLogic' + , False + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.bool + , """duringFill only: Enables debug logic, meant to be used only for tests""" + ) +options.register( 'defaultXangleX' + , 160.0 # urad + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: crossingAngleX value (in urad) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultXangleY' + , 0.0 # urad + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: crossingAngleY value (in urad) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultBetaX' + , 0.3 # meters + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: betaStarX value (in meters) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) +options.register( 'defaultBetaY' + , 0.3 # meters + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.float + , """duringFill only: betaStarY value (in meters) for the default payload. + The default payload is inserted after the last processed fill has ended + and there's no ongoing stable beam yet. """ + ) + + +# it's unlikely to ever use values different from the defaults, added as a parameter just in case +options.register('minBetaStar', 0.1 + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [meters] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('maxBetaStar', 100. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [meters] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('minCrossingAngle', 10. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [urad] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") +options.register('maxCrossingAngle', 500. + , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float + , """duringFill only: [urad] min value of the range of valid values. + If the value is outside of this range the payload is not uploaded""") + +# as the previous options, so far there was no need to use option, added just in case +options.register( 'authenticationPath' + , "" + , VarParsing.VarParsing.multiplicity.singleton + , VarParsing.VarParsing.varType.string + , """for now this option was always left empty""" + ) + options.parseArguments() if options.mode is None: raise ValueError("mode argument not provided. Supported modes are: duringFill endFill") @@ -68,6 +167,7 @@ CondDBConnection = CondDB.clone( connect = cms.string( options.destinationConnection ) ) CondDBConnection.DBParameters.messageLevel = cms.untracked.int32( options.messageLevel ) +CondDBConnection.DBParameters.authenticationPath = cms.untracked.string(options.authenticationPath) process.MessageLogger = cms.Service("MessageLogger", cout = cms.untracked.PSet(threshold = cms.untracked.string('INFO')), @@ -87,31 +187,63 @@ else: timetype = 'lumiid' -process.PoolDBOutputService = cms.Service("PoolDBOutputService", - CondDBConnection, - timetype = cms.untracked.string(timetype), - toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'), - tag = cms.string( options.tag ) - ) - ) - ) +if options.mode == "endFill": + process.PoolDBOutputService = cms.Service("PoolDBOutputService", + CondDBConnection, + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'), + tag = cms.string( options.tag ) + ) + ) + ) +else: + process.OnlineDBOutputService = cms.Service("OnlineDBOutputService", + CondDBConnection, + preLoadConnectionString = cms.untracked.string('frontier://FrontierProd/CMS_CONDITIONS' + if not options.destinationConnection.startswith('sqlite') + else options.destinationConnection ), + lastLumiFile = cms.untracked.string(options.lastLumiFile), + omsServiceUrl = cms.untracked.string('http://cmsoms-eventing.cms:9949/urn:xdaq-application:lid=100/getRunAndLumiSection' + if not options.lastLumiFile else "" ), + # runNumber = cms.untracked.uint64(384468), #not used in production, the last LS processed is set as the 1st LS of this + #run if the omsServiceUrl is empty and file specified in lastLumiFile is empty + latency = cms.untracked.uint32(options.offsetLS), + timetype = cms.untracked.string(timetype), + toPut = cms.VPSet(cms.PSet( + record = cms.string('LHCInfoPerLSRcd'), + tag = cms.string( options.tag ), + onlyAppendUpdatePolicy = cms.untracked.bool(True) + )), + frontierKey = cms.untracked.string(options.frontierKey) +) + -process.Test1 = cms.EDAnalyzer("LHCInfoPerLSPopConAnalyzer", +process.Test1 = cms.EDAnalyzer(("LHCInfoPerLSPopConAnalyzer" if options.mode == "endFill" + else "LHCInfoPerLSOnlinePopConAnalyzer"), SinceAppendMode = cms.bool(True), record = cms.string('LHCInfoPerLSRcd'), name = cms.untracked.string('LHCInfo'), - Source = cms.PSet(fill = cms.untracked.uint32(6417), + Source = cms.PSet( startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - endFill = cms.untracked.bool(True if options.mode == "endFill" else False), + endFill = cms.untracked.bool(options.mode == "endFill"), name = cms.untracked.string("LHCInfoPerLSPopConSourceHandler"), - connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), - omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), - authenticationPath = cms.untracked.string(""), - debug=cms.untracked.bool(False) - ), + connectionString = cms.untracked.string(options.sourceConnection), + omsBaseUrl = cms.untracked.string(options.oms), + authenticationPath = cms.untracked.string(options.authenticationPath), + debug=cms.untracked.bool(False), # Additional logs + debugLogic=cms.untracked.bool(options.debugLogic), + defaultCrossingAngleX = cms.untracked.double(options.defaultXangleX), + defaultCrossingAngleY = cms.untracked.double(options.defaultXangleY), + defaultBetaStarX = cms.untracked.double(options.defaultBetaX), + defaultBetaStarY = cms.untracked.double(options.defaultBetaY), + minBetaStar = cms.untracked.double(options.minBetaStar), + maxBetaStar = cms.untracked.double(options.maxBetaStar), + minCrossingAngle = cms.untracked.double(options.minCrossingAngle), + maxCrossingAngle = cms.untracked.double(options.maxCrossingAngle), + ), loggingOn = cms.untracked.bool(True), IsDestDbCheckedInQueryLog = cms.untracked.bool(False) ) -process.p = cms.Path(process.Test1) +process.p = cms.Path(process.Test1) \ No newline at end of file diff --git a/CondTools/RunInfo/src/LHCInfoHelper.cc b/CondTools/RunInfo/src/LHCInfoHelper.cc index 54503068b8924..e23f4cc199e58 100644 --- a/CondTools/RunInfo/src/LHCInfoHelper.cc +++ b/CondTools/RunInfo/src/LHCInfoHelper.cc @@ -5,7 +5,9 @@ // Returns lumi-type IOV (packed using cond::time::lumiTime) from // last LS of last Run of the specified Fill //***************************************************************** -cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) { + +std::pair cond::lhcInfoHelper::getFillLastRunAndLS(const cond::OMSService& oms, + unsigned short fillId) { // Define query auto query = oms.query("lumisections"); query->addOutputVars({"lumisection_number", "run_number"}); @@ -27,5 +29,10 @@ cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms // Return the final IOV auto lastRun = queryResult.back().get("run_number"); auto lastLumi = queryResult.back().get("lumisection_number"); - return cond::time::lumiTime(lastRun, lastLumi); + return std::make_pair(lastRun, lastLumi); } + +cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) { + auto [lastRun, lastLumi] = getFillLastRunAndLS(oms, fillId); + return cond::time::lumiTime(lastRun, lastLumi); +} \ No newline at end of file diff --git a/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc new file mode 100644 index 0000000000000..553a0b6d9e2d6 --- /dev/null +++ b/CondTools/RunInfo/src/LHCInfoPerFillPopConSourceHandler.cc @@ -0,0 +1,744 @@ +#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h" +#include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondFormats/Common/interface/TimeConversions.h" +#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" +#include "CondTools/RunInfo/interface/LumiSectionFilter.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "CoralBase/Attribute.h" +#include "CoralBase/AttributeList.h" +#include "CoralBase/AttributeSpecification.h" +#include "CoralBase/TimeStamp.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "RelationalAccess/ICursor.h" +#include "RelationalAccess/IQuery.h" +#include "RelationalAccess/ISchema.h" +#include "RelationalAccess/ISessionProxy.h" + +using std::make_pair; +using std::pair; + +namespace cond { + namespace theLHCInfoPerFillPopConImpl { + + static const pair s_fillTypeMap[] = { + make_pair("PROTONS", LHCInfoPerFill::PROTONS), + make_pair("IONS", LHCInfoPerFill::IONS), + make_pair("COSMICS", LHCInfoPerFill::COSMICS), + make_pair("GAP", LHCInfoPerFill::GAP)}; + + static const pair s_particleTypeMap[] = { + make_pair("PROTON", LHCInfoPerFill::PROTON), + make_pair("PB82", LHCInfoPerFill::PB82), + make_pair("AR18", LHCInfoPerFill::AR18), + make_pair("D", LHCInfoPerFill::D), + make_pair("XE54", LHCInfoPerFill::XE54)}; + + LHCInfoPerFill::FillType fillTypeFromString(const std::string& s_fill_type) { + for (auto const& i : s_fillTypeMap) + if (s_fill_type == i.first) + return i.second; + return LHCInfoPerFill::UNKNOWN; + } + + LHCInfoPerFill::ParticleType particleTypeFromString(const std::string& s_particle_type) { + for (auto const& i : s_particleTypeMap) + if (s_particle_type == i.first) + return i.second; + return LHCInfoPerFill::NONE; + } + } // namespace theLHCInfoPerFillPopConImpl + + namespace impl { + + template <> + LHCInfoPerFill::FillType from_string(const std::string& attributeValue) { + return from_string_impl( + attributeValue, LHCInfoPerFill::UNKNOWN); + } + + template <> + LHCInfoPerFill::ParticleType from_string(const std::string& attributeValue) { + return from_string_impl( + attributeValue, LHCInfoPerFill::NONE); + } + + } // namespace impl +} // namespace cond + +namespace theLHCInfoPerFillImpl { + + bool makeFillPayload(std::unique_ptr& targetPayload, const cond::OMSServiceResult& queryResult) { + bool ret = false; + if (!queryResult.empty()) { + auto row = *queryResult.begin(); + auto currentFill = row.get("fill_number"); + auto bunches1 = row.get("bunches_beam1"); + auto bunches2 = row.get("bunches_beam2"); + auto collidingBunches = row.get("bunches_colliding"); + auto targetBunches = row.get("bunches_target"); + auto fillType = row.get("fill_type_runtime"); + auto particleType1 = row.get("fill_type_party1"); + auto particleType2 = row.get("fill_type_party2"); + auto intensityBeam1 = row.get("intensity_beam1"); + auto intensityBeam2 = row.get("intensity_beam2"); + auto energy = row.get("energy"); + auto creationTime = row.get("start_time"); + auto stableBeamStartTime = row.get("start_stable_beam"); + std::string endTimeStr = row.get("end_time"); + auto beamDumpTime = + (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); + auto injectionScheme = row.get("injection_scheme"); + targetPayload = std::make_unique(); + targetPayload->setFillNumber(currentFill); + targetPayload->setBunchesInBeam1(bunches1); + targetPayload->setBunchesInBeam2(bunches2); + targetPayload->setCollidingBunches(collidingBunches); + targetPayload->setTargetBunches(targetBunches); + targetPayload->setFillType(fillType); + targetPayload->setParticleTypeForBeam1(particleType1); + targetPayload->setParticleTypeForBeam2(particleType2); + targetPayload->setIntensityForBeam1(intensityBeam1); + targetPayload->setIntensityForBeam2(intensityBeam2); + targetPayload->setEnergy(energy); + targetPayload->setCreationTime(cond::time::from_boost(creationTime)); + targetPayload->setBeginTime(cond::time::from_boost(stableBeamStartTime)); + targetPayload->setEndTime(beamDumpTime); + targetPayload->setInjectionScheme(injectionScheme); + ret = true; + } + return ret; + } +} // namespace theLHCInfoPerFillImpl + +namespace theLHCInfoPerFillImpl { + static const std::map vecMap = { + {"Beam1/beamPhaseMean", 1}, {"Beam2/beamPhaseMean", 2}, {"Beam1/cavPhaseMean", 3}, {"Beam2/cavPhaseMean", 4}}; + void setElementData(cond::Time_t since, + const std::string& dipVal, + unsigned int elementNr, + float value, + LHCInfoPerFill& payload, + std::set& initList) { + if (initList.find(since) == initList.end()) { + payload.beam1VC().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam2VC().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam1RF().resize(LHCInfoPerFill::bunchSlots, 0.); + payload.beam2RF().resize(LHCInfoPerFill::bunchSlots, 0.); + initList.insert(since); + } + // set the current values to all of the payloads of the lumi section samples after the current since + if (elementNr < LHCInfoPerFill::bunchSlots) { + switch (vecMap.at(dipVal)) { + case 1: + payload.beam1VC()[elementNr] = value; + break; + case 2: + payload.beam2VC()[elementNr] = value; + break; + case 3: + payload.beam1RF()[elementNr] = value; + break; + case 4: + payload.beam2RF()[elementNr] = value; + break; + default: + break; + } + } + } +} // namespace theLHCInfoPerFillImpl + +namespace theLHCInfoPerFillImpl { + bool comparePayloads(const LHCInfoPerFill& rhs, const LHCInfoPerFill& lhs) { + if (rhs.fillNumber() != lhs.fillNumber() || rhs.delivLumi() != lhs.delivLumi() || rhs.recLumi() != lhs.recLumi() || + rhs.instLumi() != lhs.instLumi() || rhs.instLumiError() != lhs.instLumiError() || + rhs.lhcState() != lhs.lhcState() || rhs.lhcComment() != lhs.lhcComment() || + rhs.ctppsStatus() != lhs.ctppsStatus()) { + return false; + } + return true; + } + + size_t transferPayloads(const std::vector>>& buffer, + std::map>& iovsToTransfer, + std::shared_ptr& prevPayload) { + size_t niovs = 0; + std::stringstream condIovs; + std::stringstream formattedIovs; + for (auto& iov : buffer) { + bool add = false; + auto payload = iov.second; + cond::Time_t since = iov.first; + if (iovsToTransfer.empty()) { + add = true; + } else { + LHCInfoPerFill& lastAdded = *iovsToTransfer.rbegin()->second; + if (!comparePayloads(lastAdded, *payload)) { + add = true; + } + } + if (add) { + niovs++; + condIovs << since << " "; + formattedIovs << boost::posix_time::to_iso_extended_string(cond::time::to_boost(since)) << " "; + iovsToTransfer.insert(make_pair(since, payload)); + prevPayload = iov.second; + } + } + edm::LogInfo("transferPayloads") << "TRANSFERED IOVS: " << condIovs.str(); + edm::LogInfo("transferPayloads") << "FORMATTED TRANSFERED IOVS: " << formattedIovs.str(); + return niovs; + } + +} // namespace theLHCInfoPerFillImpl + +LHCInfoPerFillPopConSourceHandler::LHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset) + : m_debug(pset.getUntrackedParameter("debug", false)), + m_startTime(), + m_endTime(), + m_endFillMode(pset.getUntrackedParameter("endFill", true)), + m_name(pset.getUntrackedParameter("name", "LHCInfoPerFillPopConSourceHandler")), + m_connectionString(pset.getUntrackedParameter("connectionString", "")), + m_ecalConnectionString(pset.getUntrackedParameter("ecalConnectionString", "")), + m_authpath(pset.getUntrackedParameter("authenticationPath", "")), + m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), + m_fillPayload(), + m_prevPayload(), + m_tmpBuffer() { + if (!pset.getUntrackedParameter("startTime").empty()) { + m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); + } + boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); + m_endTime = now; + if (!pset.getUntrackedParameter("endTime").empty()) { + m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); + if (m_endTime > now) + m_endTime = now; + } +} + +void LHCInfoPerFillPopConSourceHandler::getNewObjects() { + //if a new tag is created, transfer fake fill from 1 to the first fill for the first time + if (tagInfo().size == 0) { + edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; + } else { + //check what is already inside the database + edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size + << ", last object valid since " << tagInfo().lastInterval.since << " ( " + << boost::posix_time::to_iso_extended_string( + cond::time::to_boost(tagInfo().lastInterval.since)) + << " ); from " << m_name << "::getNewObjects"; + } + + cond::Time_t lastSince = tagInfo().lastInterval.since; + if (tagInfo().isEmpty()) { + // for a new or empty tag in endFill mode, an empty payload should be added on top with since=1 + addEmptyPayload(1); + lastSince = 1; + if (!m_endFillMode) { + edm::LogInfo(m_name) << "Empty or new tag: uploading a default payload and ending the job"; + return; + } + } else { + edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " + << m_name << "::getNewObjects"; + } + + //retrieve the data from the relational database source + cond::persistency::ConnectionPool connection; + //configure the connection + if (m_debug) { + connection.setMessageVerbosity(coral::Debug); + } else { + connection.setMessageVerbosity(coral::Error); + } + connection.setAuthenticationPath(m_authpath); + connection.configure(); + //create the sessions + cond::persistency::Session session = connection.createSession(m_connectionString, false); + cond::persistency::Session session2 = connection.createSession(m_ecalConnectionString, false); + // fetch last payload when available + if (!tagInfo().lastInterval.payloadId.empty()) { + cond::persistency::Session session3 = dbSession(); + session3.transaction().start(true); + m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); + session3.transaction().commit(); + } + + boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); + cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); + + cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); + cond::Time_t nextFillSearchTimestamp = + std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime()); + + edm::LogInfo(m_name) << "Starting sampling at " + << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); + + while (true) { + if (nextFillSearchTimestamp >= executionTimeIov) { + edm::LogInfo(m_name) << "Sampling ended at the time " + << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); + break; + } + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); + boost::posix_time::ptime startSampleTime; + boost::posix_time::ptime endSampleTime; + + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + + edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); + query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); + if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) { + query->filterGE("start_time", nextFillSearchTime); + } else { + query->filterGT("start_time", nextFillSearchTime); + } + + query->filterLT("start_time", m_endTime); + if (m_endFillMode) + query->filterNotNull("end_time"); + else + query->filterEQ("end_time", cond::OMSServiceQuery::SNULL); + + bool foundFill = query->execute(); + if (foundFill) + foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result()); + if (!foundFill) { + edm::LogInfo(m_name) << "No fill found - END of job."; + break; + } + + startSampleTime = cond::time::to_boost(m_fillPayload->createTime()); + cond::Time_t startFillTime = m_fillPayload->createTime(); + cond::Time_t endFillTime = m_fillPayload->endTime(); + unsigned short lhcFill = m_fillPayload->fillNumber(); + bool ongoingFill = endFillTime == 0ULL; + if (ongoingFill) { + edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime); + endSampleTime = executionTime; + nextFillSearchTimestamp = executionTimeIov; + } else { + edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime) + << " ending at " << cond::time::to_boost(endFillTime); + endSampleTime = cond::time::to_boost(endFillTime); + nextFillSearchTimestamp = endFillTime; + } + if (m_endFillMode || ongoingFill) { + getDipData(oms, startSampleTime, endSampleTime); + getLumiData(oms, lhcFill, startSampleTime, endSampleTime); + if (!m_tmpBuffer.empty()) { + boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); + boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); + edm::LogInfo(m_name) << "First lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; + session.transaction().start(true); + getCTPPSData(session, startSampleTime, endSampleTime); + session.transaction().commit(); + session2.transaction().start(true); + getEcalData(session2, startSampleTime, endSampleTime); + session2.transaction().commit(); + } + } + + if (!m_endFillMode) { + if (m_tmpBuffer.size() > 1) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "More than 1 payload buffered for writing in duringFill mode.\ + In this mode only up to 1 payload can be written"; + } else if (m_tmpBuffer.size() == 1) { + if (theLHCInfoPerFillImpl::comparePayloads(*(m_tmpBuffer.begin()->second), *m_prevPayload)) { + m_tmpBuffer.clear(); + edm::LogInfo(m_name) + << "The buffered payload has the same data as the previous payload in the tag. It will not be written."; + } + } else if (m_tmpBuffer.empty()) { + addEmptyPayload( + cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); //the IOV doesn't matter when using OnlinePopCon + } + // In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs + // before transferring the payloads from the buffer to the final collection + convertBufferedIovsToLumiid(m_timestampToLumiid); + } + + size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload); + edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; + m_tmpBuffer.clear(); + m_timestampToLumiid.clear(); + + if (!m_endFillMode) { + return; + } + + // endFill mode only: + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(endFillTime); + } else { + addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); + } + } + } +} + +std::string LHCInfoPerFillPopConSourceHandler::id() const { return m_name; } + +void LHCInfoPerFillPopConSourceHandler::addEmptyPayload(cond::Time_t iov) { + bool add = false; + if (m_iovs.empty()) { + if (!m_lastPayloadEmpty) + add = true; + } else { + auto lastAdded = m_iovs.rbegin()->second; + if (lastAdded->fillNumber() != 0) { + add = true; + } + } + if (add) { + auto newPayload = std::make_shared(); + m_iovs.insert(make_pair(iov, newPayload)); + m_prevPayload = newPayload; + edm::LogInfo(m_name) << "Added empty payload with IOV " << iov << " ( " + << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; + } +} + +// Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map +void LHCInfoPerFillPopConSourceHandler::addPayloadToBuffer(cond::OMSServiceResultRef& row) { + auto startTime = row.get("start_time"); + auto delivLumi = row.get("delivered_lumi"); + auto recLumi = row.get("recorded_lumi"); + auto runNumber = std::stoul(row.get("run_number")); + auto lsNumber = std::stoul(row.get("lumisection_number")); + auto lumiid = cond::time::lumiTime(runNumber, lsNumber); + + LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill(); + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo)); + if (!m_endFillMode) { + m_timestampToLumiid.insert(make_pair(cond::time::from_boost(startTime), lumiid)); + } + LHCInfoPerFill& payload = *thisLumiSectionInfo; + payload.setDelivLumi(delivLumi); + payload.setRecLumi(recLumi); +} + +void LHCInfoPerFillPopConSourceHandler::convertBufferedIovsToLumiid( + std::map timestampToLumiid) { + for (auto& item : m_tmpBuffer) { + // Check if the lumiid IOV corresponding to the timestamp is present in the map + if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n"; + } + // Update the buffer with the lumiid-type IOV + item.first = timestampToLumiid.at(item.first); + } +} + +size_t LHCInfoPerFillPopConSourceHandler::getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + auto query = oms.query("lumisections"); + query->addOutputVars( + {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"}); + query->filterEQ("fill_number", fillId); + query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); + query->filterEQ("beams_stable", "true"); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); + if (query->execute()) { + auto queryResult = query->result(); + edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections with STABLE BEAM during the fill " + << fillId; + + if (!queryResult.empty()) { + if (m_endFillMode) { + auto firstRow = queryResult.front(); + addPayloadToBuffer(firstRow); + } + + auto lastRow = queryResult.back(); + addPayloadToBuffer(lastRow); + } + } + return 0; +} + +void LHCInfoPerFillPopConSourceHandler::getDipData(const cond::OMSService& oms, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + // unsure how to handle this. + // the old implementation is not helping: apparently it is checking only the bunchconfiguration for the first diptime set of values... + auto query1 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam1"); + query1->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + //If all this data is needed and saved properly the limit has to be set: query1->limit(...) + if (query1->execute()) { + auto res = query1->result(); + if (!res.empty()) { + std::bitset bunchConfiguration1(0ULL); + auto row = *res.begin(); + auto vbunchConf1 = row.getArray("value"); + for (auto vb : vbunchConf1) { + if (vb != 0) { + unsigned short slot = (vb - 1) / 10 + 1; + bunchConfiguration1[slot] = true; + } + } + m_fillPayload->setBunchBitsetForBeam1(bunchConfiguration1); + } + } + auto query2 = oms.query("diplogger/dip/acc/LHC/RunControl/CirculatingBunchConfig/Beam2"); + query2->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + if (query2->execute()) { + auto res = query2->result(); + if (!res.empty()) { + std::bitset bunchConfiguration2(0ULL); + auto row = *res.begin(); + auto vbunchConf2 = row.getArray("value"); + for (auto vb : vbunchConf2) { + if (vb != 0) { + unsigned short slot = (vb - 1) / 10 + 1; + bunchConfiguration2[slot] = true; + } + } + m_fillPayload->setBunchBitsetForBeam2(bunchConfiguration2); + } + } + + auto query3 = oms.query("diplogger/dip/CMS/LHC/LumiPerBunch"); + query3->filterGT("dip_time", beginFillTime).filterLT("dip_time", endFillTime); + //This query is limited to 100 rows, but currently only one is used + if (query3->execute()) { + auto res = query3->result(); + if (!res.empty()) { + std::vector lumiPerBX; + auto row = *res.begin(); + auto lumiBunchInst = row.getArray("lumi_bunch_inst"); + for (auto lb : lumiBunchInst) { + if (lb != 0.) { + lumiPerBX.push_back(lb); + } + } + m_fillPayload->setLumiPerBX(lumiPerBX); + } + } +} + +bool LHCInfoPerFillPopConSourceHandler::getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + //run the fifth query against the CTPPS schema + //Initializing the CMS_CTP_CTPPS_COND schema. + coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); + //execute query for CTPPS Data + std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); + //FROM clause + CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); + //SELECT clause + CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); + CTPPSDataQuery->addToOutputList(std::string("LHC_STATE")); + CTPPSDataQuery->addToOutputList(std::string("LHC_COMMENT")); + if (m_debug) { + CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); + } + //WHERE CLAUSE + coral::AttributeList CTPPSDataBindVariables; + CTPPSDataBindVariables.extend(std::string("beginFillTime")); + CTPPSDataBindVariables.extend(std::string("endFillTime")); + CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); + CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); + std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); + CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); + //ORDER BY clause + CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); + //define query output + coral::AttributeList CTPPSDataOutput; + CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); + CTPPSDataOutput.extend(std::string("LHC_STATE")); + CTPPSDataOutput.extend(std::string("LHC_COMMENT")); + if (m_debug) { + CTPPSDataOutput.extend(std::string("RUN_NUMBER")); + CTPPSDataOutput.extend(std::string("LUMI_SECTION")); + } + CTPPSDataQuery->defineOutput(CTPPSDataOutput); + //execute the query + coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); + cond::Time_t dipTime = 0; + std::string lhcState = "", lhcComment = "", ctppsStatus = ""; + + //debug informations + unsigned int lumiSection = 0; + cond::Time_t runNumber = 0; + cond::Time_t savedDipTime = 0; + unsigned int savedLumiSection = 0; + cond::Time_t savedRunNumber = 0; + + bool ret = false; + LumiSectionFilter filter(m_tmpBuffer); + while (CTPPSDataCursor.next()) { + if (m_debug) { + std::ostringstream CTPPS; + CTPPSDataCursor.currentRow().toOutputStream(CTPPS); + } + coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; + if (!dipTimeAttribute.isNull()) { + dipTime = cond::time::from_boost(dipTimeAttribute.data().time()); + if (filter.process(dipTime)) { + ret = true; + coral::Attribute const& lhcStateAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_STATE")]; + if (!lhcStateAttribute.isNull()) { + lhcState = lhcStateAttribute.data(); + } + coral::Attribute const& lhcCommentAttribute = CTPPSDataCursor.currentRow()[std::string("LHC_COMMENT")]; + if (!lhcCommentAttribute.isNull()) { + lhcComment = lhcCommentAttribute.data(); + } + + if (m_debug) { + coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; + if (!runNumberAttribute.isNull()) { + runNumber = runNumberAttribute.data(); + } + coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; + if (!lumiSectionAttribute.isNull()) { + lumiSection = lumiSectionAttribute.data(); + } + } + + for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { + // set the current values to all of the payloads of the lumi section samples after the current since + LHCInfoPerFill& payload = *(it->second); + payload.setLhcState(lhcState); + payload.setLhcComment(lhcComment); + payload.setCtppsStatus(ctppsStatus); + + if (m_debug) { + savedDipTime = dipTime; + savedLumiSection = lumiSection; + savedRunNumber = runNumber; + } + } + } + } + } + if (m_debug) { + edm::LogInfo(m_name) << "Last assigned: " + << "DipTime: " << savedDipTime << " " + << "LumiSection: " << savedLumiSection << " " + << "RunNumber: " << savedRunNumber; + } + return ret; +} + +bool LHCInfoPerFillPopConSourceHandler::getEcalData(cond::persistency::Session& session, + const boost::posix_time::ptime& lowerTime, + const boost::posix_time::ptime& upperTime) { + //run the sixth query against the CMS_DCS_ENV_PVSS_COND schema + //Initializing the CMS_DCS_ENV_PVSS_COND schema. + coral::ISchema& ECAL = session.nominalSchema(); + //start the transaction against the fill logging schema + //execute query for ECAL Data + std::unique_ptr ECALDataQuery(ECAL.newQuery()); + //FROM clause + ECALDataQuery->addToTableList(std::string("BEAM_PHASE")); + //SELECT clause + ECALDataQuery->addToOutputList(std::string("CHANGE_DATE")); + ECALDataQuery->addToOutputList(std::string("DIP_value")); + ECALDataQuery->addToOutputList(std::string("element_nr")); + ECALDataQuery->addToOutputList(std::string("VALUE_NUMBER")); + //WHERE CLAUSE + coral::AttributeList ECALDataBindVariables; + ECALDataBindVariables.extend(std::string("lowerTime")); + ECALDataBindVariables.extend(std::string("upperTime")); + ECALDataBindVariables[std::string("lowerTime")].data() = coral::TimeStamp(lowerTime); + ECALDataBindVariables[std::string("upperTime")].data() = coral::TimeStamp(upperTime); + std::string conditionStr = std::string( + "(DIP_value LIKE '%beamPhaseMean%' OR DIP_value LIKE '%cavPhaseMean%') AND CHANGE_DATE >= :lowerTime AND " + "CHANGE_DATE < :upperTime"); + + ECALDataQuery->setCondition(conditionStr, ECALDataBindVariables); + //ORDER BY clause + ECALDataQuery->addToOrderList(std::string("CHANGE_DATE")); + ECALDataQuery->addToOrderList(std::string("DIP_value")); + ECALDataQuery->addToOrderList(std::string("element_nr")); + //define query output + coral::AttributeList ECALDataOutput; + ECALDataOutput.extend(std::string("CHANGE_DATE")); + ECALDataOutput.extend(std::string("DIP_value")); + ECALDataOutput.extend(std::string("element_nr")); + ECALDataOutput.extend(std::string("VALUE_NUMBER")); + //ECALDataQuery->limitReturnedRows( 14256 ); //3564 entries per vector. + ECALDataQuery->defineOutput(ECALDataOutput); + //execute the query + coral::ICursor& ECALDataCursor = ECALDataQuery->execute(); + cond::Time_t changeTime = 0; + cond::Time_t firstTime = 0; + std::string dipVal = ""; + unsigned int elementNr = 0; + float value = 0.; + std::set initializedVectors; + LumiSectionFilter filter(m_tmpBuffer); + bool ret = false; + if (m_prevPayload.get()) { + for (auto& lumiSlot : m_tmpBuffer) { + lumiSlot.second->setBeam1VC(m_prevPayload->beam1VC()); + lumiSlot.second->setBeam2VC(m_prevPayload->beam2VC()); + lumiSlot.second->setBeam1RF(m_prevPayload->beam1RF()); + lumiSlot.second->setBeam2RF(m_prevPayload->beam2RF()); + } + } + std::map iovMap; + if (m_tmpBuffer.empty()) { + return ret; + } + cond::Time_t lowerLumi = m_tmpBuffer.front().first; + while (ECALDataCursor.next()) { + if (m_debug) { + std::ostringstream ECAL; + ECALDataCursor.currentRow().toOutputStream(ECAL); + } + coral::Attribute const& changeDateAttribute = ECALDataCursor.currentRow()[std::string("CHANGE_DATE")]; + if (!changeDateAttribute.isNull()) { + ret = true; + boost::posix_time::ptime chTime = changeDateAttribute.data().time(); + // move the first IOV found to the start of the fill interval selected + if (changeTime == 0) { + firstTime = cond::time::from_boost(chTime); + } + changeTime = cond::time::from_boost(chTime); + cond::Time_t iovTime = changeTime; + if (changeTime == firstTime) + iovTime = lowerLumi; + coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")]; + coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")]; + coral::Attribute const& elementNrAttribute = ECALDataCursor.currentRow()[std::string("element_nr")]; + if (!dipValAttribute.isNull() and !valueNumberAttribute.isNull()) { + dipVal = dipValAttribute.data(); + elementNr = elementNrAttribute.data(); + value = valueNumberAttribute.data(); + if (std::isnan(value)) + value = 0.; + if (filter.process(iovTime)) { + iovMap.insert(make_pair(changeTime, filter.current()->first)); + for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { + LHCInfoPerFill& payload = *(it->second); + theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors); + } + } + } + } + } + if (m_debug) { + for (auto& im : iovMap) { + edm::LogInfo(m_name) << "Found iov=" << im.first << " (" << cond::time::to_boost(im.first) << " ) moved to " + << im.second << " ( " << cond::time::to_boost(im.second) << " )"; + } + } + return ret; +} diff --git a/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc new file mode 100644 index 0000000000000..8cb01ac930cf8 --- /dev/null +++ b/CondTools/RunInfo/src/LHCInfoPerLSPopConSourceHandler.cc @@ -0,0 +1,659 @@ +#include "CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h" +#include "CondCore/CondDB/interface/ConnectionPool.h" +#include "CondCore/CondDB/interface/Types.h" +#include "CondFormats/Common/interface/TimeConversions.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" +#include "CoralBase/Attribute.h" +#include "CoralBase/AttributeList.h" +#include "CoralBase/AttributeSpecification.h" +#include "CoralBase/TimeStamp.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "RelationalAccess/ICursor.h" +#include "RelationalAccess/IQuery.h" +#include "RelationalAccess/ISchema.h" +#include "RelationalAccess/ISessionProxy.h" +#include +#include + +using std::make_pair; +using std::pair; + +namespace theLHCInfoPerLSImpl { + bool comparePayloads(const LHCInfoPerLS& rhs, const LHCInfoPerLS& lhs) { + if (rhs.fillNumber() != lhs.fillNumber() || rhs.runNumber() != lhs.runNumber() || + rhs.crossingAngleX() != lhs.crossingAngleX() || rhs.crossingAngleY() != lhs.crossingAngleY() || + rhs.betaStarX() != lhs.betaStarX() || rhs.betaStarY() != lhs.betaStarY()) { + return false; + } + return true; + } + + size_t transferPayloads(const std::vector>>& buffer, + std::map>& iovsToTransfer, + std::shared_ptr& prevPayload, + const std::map, pair>& lsIdMap, + cond::Time_t startStableBeamTime, + cond::Time_t endStableBeamTime) { + int lsMissingInPPS = 0; + int xAngleBothZero = 0, xAngleBothNonZero = 0, xAngleNegative = 0; + int betaNegative = 0; + size_t niovs = 0; + std::stringstream condIovs; + std::stringstream missingLsList; + for (auto& iov : buffer) { + bool add = false; + auto payload = iov.second; + cond::Time_t since = iov.first; + if (iovsToTransfer.empty()) { + add = true; + } else { + LHCInfoPerLS& lastAdded = *iovsToTransfer.rbegin()->second; + if (!comparePayloads(lastAdded, *payload)) { + add = true; + } + } + auto id = make_pair(payload->runNumber(), payload->lumiSection()); + bool stableBeam = since >= startStableBeamTime && since <= endStableBeamTime; + bool isMissing = lsIdMap.find(id) != lsIdMap.end() && id != lsIdMap.at(id); + if (stableBeam && isMissing) { + missingLsList << id.first << "_" << id.second << " "; + lsMissingInPPS += isMissing; + } + if (add && !isMissing) { + niovs++; + if (stableBeam) { + if (payload->crossingAngleX() == 0 && payload->crossingAngleY() == 0) + xAngleBothZero++; + if (payload->crossingAngleX() != 0 && payload->crossingAngleY() != 0) + xAngleBothNonZero++; + if (payload->crossingAngleX() < 0 || payload->crossingAngleY() < 0) + xAngleNegative++; + if (payload->betaStarX() < 0 || payload->betaStarY() < 0) + betaNegative++; + } + + condIovs << since << " "; + iovsToTransfer.insert(make_pair(since, payload)); + prevPayload = iov.second; + } + } + unsigned short fillNumber = (!buffer.empty()) ? buffer.front().second->fillNumber() : 0; + if (lsMissingInPPS > 0) { + edm::LogWarning("transferPayloads") + << "Number of stable beam LS in OMS without corresponding record in PPS DB for fill " << fillNumber << ": " + << lsMissingInPPS; + edm::LogWarning("transferPayloads") + << "Stable beam LS in OMS without corresponding record in PPS DB (run_LS): " << missingLsList.str(); + } + if (xAngleBothZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle == 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothZero; + } + if (xAngleBothNonZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle != 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothNonZero; + } + if (xAngleNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative crossingAngle for fill " << fillNumber << ": " << xAngleNegative; + } + if (betaNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative betaSta for fill " << fillNumber << ": " << betaNegative; + } + + edm::LogInfo("transferPayloads") << "TRANSFERED COND IOVS: " << condIovs.str(); + return niovs; + } + +} // namespace theLHCInfoPerLSImpl + +LHCInfoPerLSPopConSourceHandler::LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset) + : m_debug(pset.getUntrackedParameter("debug", false)), + m_startTime(), + m_endTime(), + m_endFillMode(pset.getUntrackedParameter("endFill", true)), + m_name(pset.getUntrackedParameter("name", "LHCInfoPerLSPopConSourceHandler")), + m_connectionString(pset.getUntrackedParameter("connectionString", "")), + m_authpath(pset.getUntrackedParameter("authenticationPath", "")), + m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), + m_debugLogic(pset.getUntrackedParameter("debugLogic", false)), + m_defaultCrossingAngleX(pset.getUntrackedParameter("defaultCrossingAngleX", 0)), + m_defaultCrossingAngleY(pset.getUntrackedParameter("defaultCrossingAngleY", 0)), + m_defaultBetaStarX(pset.getUntrackedParameter("defaultBetaStarX", 0)), + m_defaultBetaStarY(pset.getUntrackedParameter("defaultBetaStarY", 0)), + m_minBetaStar(pset.getUntrackedParameter("minBetaStar", 0.1)), + m_maxBetaStar(pset.getUntrackedParameter("maxBetaStar", 100.)), + m_minCrossingAngle(pset.getUntrackedParameter("minCrossingAngle", 10.)), + m_maxCrossingAngle(pset.getUntrackedParameter("maxCrossingAngle", 500.)), + m_fillPayload(), + m_prevPayload(), + m_tmpBuffer() { + if (!pset.getUntrackedParameter("startTime").empty()) { + m_startTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("startTime")); + } + boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); + m_endTime = now; + if (!pset.getUntrackedParameter("endTime").empty()) { + m_endTime = boost::posix_time::time_from_string(pset.getUntrackedParameter("endTime")); + if (m_endTime > now) + m_endTime = now; + } + if (m_debugLogic && m_endFillMode) { + throw cms::Exception("invalid argument") << "debugLogic == true not supported for endFillMode == true"; + } +} + +LHCInfoPerLSPopConSourceHandler::~LHCInfoPerLSPopConSourceHandler() = default; + +void LHCInfoPerLSPopConSourceHandler::getNewObjects() { + populateIovs(); + if (!m_endFillMode) { // duringFill mode + filterInvalidPayloads(); + } +} + +void LHCInfoPerLSPopConSourceHandler::filterInvalidPayloads() { + // note: at the moment used only in duringFill mode so the m_iovs is quaranteed to have size() <= 1 + // but iterating through the whole map is implemented just in case the way it's used changes + auto it = m_iovs.begin(); + while (it != m_iovs.end()) { + std::stringstream payloadData; + payloadData << "LS = " << it->second->lumiSection() << ", run = " << it->second->runNumber() << ", " + << "xAngleX = " << it->second->crossingAngleX() << " urad, " + << "xAngleY = " << it->second->crossingAngleY() << " urad, " + << "beta*X = " << it->second->betaStarX() << " m, " + << "beta*Y = " << it->second->betaStarY() << " m"; + if (!isPayloadValid(*(it->second))) { + edm::LogWarning(m_name) << "Skipping upload of payload with invalid values: " << payloadData.str(); + m_iovs.erase(it++); // note: post-increment necessary to avoid using invalidated iterators + } else { + edm::LogInfo(m_name) << "Payload to be uploaded: " << payloadData.str(); + ++it; + } + } +} + +bool LHCInfoPerLSPopConSourceHandler::isPayloadValid(const LHCInfoPerLS& payload) const { + if ((payload.crossingAngleX() == 0. && payload.crossingAngleY() == 0.) || + (payload.crossingAngleX() != 0. && payload.crossingAngleY() != 0.)) + return false; + auto non0CrossingAngle = payload.crossingAngleX() != 0. ? payload.crossingAngleX() : payload.crossingAngleY(); + if ((non0CrossingAngle < m_minCrossingAngle || m_maxCrossingAngle < non0CrossingAngle) || + (payload.betaStarX() < m_minBetaStar || m_maxBetaStar < payload.betaStarX()) || + (payload.betaStarX() < m_minBetaStar || m_maxBetaStar < payload.betaStarX())) + return false; + return true; +} + +void LHCInfoPerLSPopConSourceHandler::populateIovs() { + //if a new tag is created, transfer fake fill from 1 to the first fill for the first time + if (tagInfo().size == 0) { + edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects"; + } else { + //check what is already inside the database + edm::LogInfo(m_name) << "got info for tag " << tagInfo().name << ": size " << tagInfo().size + << ", last object valid since " << tagInfo().lastInterval.since << " ( " + << boost::posix_time::to_iso_extended_string( + cond::time::to_boost(tagInfo().lastInterval.since)) + << " ); from " << m_name << "::getNewObjects"; + } + + cond::Time_t lastSince = tagInfo().lastInterval.since; + if (tagInfo().isEmpty()) { + // for a new or empty tag in endFill mode, an empty payload should be added on top with since=1 + if (m_endFillMode) { + addEmptyPayload(1); + lastSince = 1; + } else { //duringFill mode + edm::LogInfo(m_name) << "Empty or new tag: uploading a default payload and ending the job"; + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + addDefaultPayload(1, 1, 1, 1); + return; + } + } else { + edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from " + << m_name << "::getNewObjects"; + } + + //retrieve the data from the relational database source + cond::persistency::ConnectionPool connection; + //configure the connection + if (m_debug) { + connection.setMessageVerbosity(coral::Debug); + } else { + connection.setMessageVerbosity(coral::Error); + } + connection.setAuthenticationPath(m_authpath); + connection.configure(); + //create the sessions + cond::persistency::Session session = connection.createSession(m_connectionString, false); + // fetch last payload when available + if (!tagInfo().lastInterval.payloadId.empty()) { + cond::persistency::Session session3 = dbSession(); + session3.transaction().start(true); + m_prevPayload = session3.fetchPayload(tagInfo().lastInterval.payloadId); + session3.transaction().commit(); + + // find startFillTime and endFillTime of the most recent fill already saved in the tag + if (m_prevPayload->fillNumber() != 0) { + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + query->addOutputVar("end_time"); + query->addOutputVar("start_time"); + query->filterEQ("fill_number", m_prevPayload->fillNumber()); + bool foundFill = query->execute(); + if (foundFill) { + auto result = query->result(); + + if (!result.empty()) { + std::string endTimeStr = (*result.begin()).get("end_time"); + m_prevEndFillTime = (endTimeStr == "null") + ? 0 + : cond::time::from_boost((*result.begin()).get("end_time")); + auto startFillTime = (*result.begin()).get("start_time"); + m_prevStartFillTime = cond::time::from_boost(startFillTime); + } else { + foundFill = false; + } + } + if (!foundFill) { + edm::LogError(m_name) << "Could not find end time of fill #" << m_prevPayload->fillNumber(); + } + } else { + m_prevEndFillTime = 0; + m_prevStartFillTime = 0; + } + } + + boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time(); + cond::Time_t executionTimeIov = cond::time::from_boost(executionTime); + + cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime); + cond::Time_t nextFillSearchTimestamp = std::max(startTimestamp, m_endFillMode ? lastSince : m_prevEndFillTime); + + edm::LogInfo(m_name) << "Starting sampling at " + << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); + + while (true) { + if (nextFillSearchTimestamp >= executionTimeIov) { + edm::LogInfo(m_name) << "Sampling ended at the time " + << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); + break; + } + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); + boost::posix_time::ptime startSampleTime; + boost::posix_time::ptime endSampleTime; + + cond::OMSService oms; + oms.connect(m_omsBaseUrl); + auto query = oms.query("fills"); + + if (m_debugLogic) + m_prevEndFillTime = 0ULL; + + if (!m_endFillMode and m_prevPayload->fillNumber() and m_prevEndFillTime == 0ULL) { + // continue processing unfinished fill with some payloads already in the tag + edm::LogInfo(m_name) << "Searching started fill #" << m_prevPayload->fillNumber(); + query->filterEQ("fill_number", m_prevPayload->fillNumber()); + bool foundFill = query->execute(); + if (foundFill) + foundFill = makeFillPayload(m_fillPayload, query->result()); + if (!foundFill) { + edm::LogError(m_name) << "Could not find fill #" << m_prevPayload->fillNumber(); + break; + } + } else { + edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime); + query->filterNotNull("start_stable_beam").filterNotNull("fill_number"); + if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) { + query->filterGE("start_time", nextFillSearchTime); + } else { + query->filterGT("start_time", nextFillSearchTime); + } + + query->filterLT("start_time", m_endTime); + if (m_endFillMode) + query->filterNotNull("end_time"); + else + query->filterEQ("end_time", cond::OMSServiceQuery::SNULL); + + bool querySuccess = query->execute(); + if (!querySuccess) { + edm::LogError(m_name) << "OMS fill query failed (http status not 200 nor 201). Request URL:\n" << query->url(); + } + bool foundFill = querySuccess ? makeFillPayload(m_fillPayload, query->result()) : false; + + if (!foundFill) { + if (m_endFillMode) { + edm::LogInfo(m_name) << "No fill found - END of job."; + } else { //duringFill mode + edm::LogInfo(m_name) << "No ongoing fill found."; + addDefaultPayload(1, m_prevPayload->fillNumber(), oms); //IOV doesn't matter here in duringFill mode + } + break; + } + } + startSampleTime = cond::time::to_boost(m_startFillTime); + + unsigned short lhcFill = m_fillPayload->fillNumber(); + bool ongoingFill = m_endFillTime == 0ULL; + if (ongoingFill) { + edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at " + << cond::time::to_boost(m_startFillTime); + endSampleTime = executionTime; + nextFillSearchTimestamp = executionTimeIov; + } else { + edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(m_startFillTime) + << " ending at " << cond::time::to_boost(m_endFillTime); + endSampleTime = cond::time::to_boost(m_endFillTime); + nextFillSearchTimestamp = m_endFillTime; + } + + if (m_endFillMode || ongoingFill) { + getLumiData(oms, lhcFill, startSampleTime, endSampleTime); + + if (!m_tmpBuffer.empty()) { + boost::posix_time::ptime flumiStart = cond::time::to_boost(m_tmpBuffer.front().first); + boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); + edm::LogInfo(m_name) << "First buffered lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; + session.transaction().start(true); + getCTPPSData(session, startSampleTime, endSampleTime); + session.transaction().commit(); + } + } + + if (!m_endFillMode) { + if (m_tmpBuffer.size() > 1) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "More than 1 payload buffered for writing in duringFill mode.\ + In this mode only up to 1 payload can be written"; + } else if (m_tmpBuffer.size() == 1) { + if (theLHCInfoPerLSImpl::comparePayloads(*(m_tmpBuffer.begin()->second), *m_prevPayload)) { + m_tmpBuffer.clear(); + edm::LogInfo(m_name) + << "The buffered payload has the same data as the previous payload in the tag. It will not be written."; + } + } else if (m_tmpBuffer.empty()) { + // note: the IOV doesn't matter when using OnlinePopCon: + addDefaultPayload(1, lhcFill, oms); + } + } + + size_t niovs = theLHCInfoPerLSImpl::transferPayloads( + m_tmpBuffer, m_iovs, m_prevPayload, m_lsIdMap, m_startStableBeamTime, m_endStableBeamTime); + edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time"; + m_tmpBuffer.clear(); + m_lsIdMap.clear(); + + if (!m_endFillMode) { + return; + } + + // endFill mode only: + if (niovs) { + m_prevEndFillTime = m_endFillTime; + m_prevStartFillTime = m_startFillTime; + } + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(m_endFillTime); + } + } + } +} + +std::string LHCInfoPerLSPopConSourceHandler::id() const { return m_name; } + +void LHCInfoPerLSPopConSourceHandler::addEmptyPayload(cond::Time_t iov) { + bool add = false; + if (m_iovs.empty()) { + if (!m_lastPayloadEmpty) + add = true; + } else { + auto lastAdded = m_iovs.rbegin()->second; + if (lastAdded->fillNumber() != 0) { + add = true; + } + } + if (add) { + auto newPayload = std::make_shared(); + m_iovs.insert(make_pair(iov, newPayload)); + m_prevPayload = newPayload; + m_prevEndFillTime = 0; + m_prevStartFillTime = 0; + edm::LogInfo(m_name) << "Added empty payload with IOV" << iov << " ( " + << boost::posix_time::to_iso_extended_string(cond::time::to_boost(iov)) << " )"; + } +} + +void LHCInfoPerLSPopConSourceHandler::addDefaultPayload(cond::Time_t iov, + unsigned short fill, + const cond::OMSService& oms) { + auto defaultPayload = std::make_shared(); + defaultPayload->setFillNumber(fill); + auto [lastRun, lastLumi] = cond::lhcInfoHelper::getFillLastRunAndLS(oms, fill); + addDefaultPayload(iov, fill, lastRun, lastLumi); +} + +void LHCInfoPerLSPopConSourceHandler::addDefaultPayload(cond::Time_t iov, + unsigned short fill, + int run, + unsigned short lumi) { + auto defaultPayload = std::make_shared(); + defaultPayload->setFillNumber(fill); + defaultPayload->setRunNumber(run); + defaultPayload->setLumiSection(lumi); + defaultPayload->setCrossingAngleX(m_defaultCrossingAngleX); + defaultPayload->setCrossingAngleY(m_defaultCrossingAngleY); + defaultPayload->setBetaStarX(m_defaultBetaStarX); + defaultPayload->setBetaStarY(m_defaultBetaStarY); + + if (m_prevPayload && theLHCInfoPerLSImpl::comparePayloads(*defaultPayload, *m_prevPayload)) { + edm::LogInfo(m_name) + << "The default payload has the same data as the previous payload in the tag. It will not be written."; + } else { + m_iovs.insert(make_pair(iov, defaultPayload)); + edm::LogInfo(m_name) << "Uploading the default payload."; + } +} + +bool LHCInfoPerLSPopConSourceHandler::makeFillPayload(std::unique_ptr& targetPayload, + const cond::OMSServiceResult& queryResult) { + bool ret = false; + if (!queryResult.empty()) { + auto row = *queryResult.begin(); + auto currentFill = row.get("fill_number"); + m_startFillTime = cond::time::from_boost(row.get("start_time")); + std::string endTimeStr = row.get("end_time"); + if (m_debugLogic) { + m_endFillTime = 0; + } else { + m_endFillTime = + (endTimeStr == "null") ? 0 : cond::time::from_boost(row.get("end_time")); + } + m_startStableBeamTime = cond::time::from_boost(row.get("start_stable_beam")); + m_endStableBeamTime = cond::time::from_boost(row.get("end_stable_beam")); + targetPayload = std::make_unique(); + targetPayload->setFillNumber(currentFill); + ret = true; + } + return ret; +} + +void LHCInfoPerLSPopConSourceHandler::addPayloadToBuffer(cond::OMSServiceResultRef& row) { + auto lumiTime = row.get("start_time"); + LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); + thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); + thisLumiSectionInfo->setRunNumber(std::stoul(row.get("run_number"))); + m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); + if (m_endFillMode) { + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + } else { + m_tmpBuffer.emplace_back( + make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()), + thisLumiSectionInfo)); + } +} + +size_t LHCInfoPerLSPopConSourceHandler::bufferAllLS(const cond::OMSServiceResult& queryResult) { + for (auto r : queryResult) { + addPayloadToBuffer(r); + } + return queryResult.size(); +} + +size_t LHCInfoPerLSPopConSourceHandler::getLumiData(const cond::OMSService& oms, + unsigned short fillId, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + auto query = oms.query("lumisections"); + query->addOutputVars({"start_time", "run_number", "beams_stable", "lumisection_number"}); + query->filterEQ("fill_number", fillId); + query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); + size_t nlumi = 0; + if (query->execute()) { + auto queryResult = query->result(); + if (m_endFillMode) { + nlumi = bufferAllLS(queryResult); + } else if (!queryResult.empty()) { + auto newestPayload = queryResult.back(); + if (newestPayload.get("beams_stable") == "true" || m_debugLogic) { + addPayloadToBuffer(newestPayload); + nlumi = 1; + edm::LogInfo(m_name) << "Buffered most recent lumisection:" + << " LS: " << newestPayload.get("lumisection_number") + << " run: " << newestPayload.get("run_number"); + } + } + edm::LogInfo(m_name) << "Found " << queryResult.size() << " lumisections during the fill " << fillId; + } else { + edm::LogInfo(m_name) << "OMS query for lumisections of fill " << fillId << "failed, status:" << query->status(); + } + return nlumi; +} +bool LHCInfoPerLSPopConSourceHandler::getCTPPSData(cond::persistency::Session& session, + const boost::posix_time::ptime& beginFillTime, + const boost::posix_time::ptime& endFillTime) { + //run the fifth query against the CTPPS schema + //Initializing the CMS_CTP_CTPPS_COND schema. + coral::ISchema& CTPPS = session.coralSession().schema("CMS_PPS_SPECT_COND"); + //execute query for CTPPS Data + std::unique_ptr CTPPSDataQuery(CTPPS.newQuery()); + //FROM clause + CTPPSDataQuery->addToTableList(std::string("PPS_LHC_MACHINE_PARAMS")); + //SELECT clause + CTPPSDataQuery->addToOutputList(std::string("DIP_UPDATE_TIME")); + CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); + CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("FILL_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_X_URAD")); + CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_Y_URAD")); + CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_X_M")); + CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_Y_M")); + //WHERE CLAUSE + coral::AttributeList CTPPSDataBindVariables; + CTPPSDataBindVariables.extend(std::string("beginFillTime")); + CTPPSDataBindVariables.extend(std::string("endFillTime")); + CTPPSDataBindVariables[std::string("beginFillTime")].data() = coral::TimeStamp(beginFillTime); + CTPPSDataBindVariables[std::string("endFillTime")].data() = coral::TimeStamp(endFillTime); + std::string conditionStr = std::string("DIP_UPDATE_TIME>= :beginFillTime and DIP_UPDATE_TIME< :endFillTime"); + CTPPSDataQuery->setCondition(conditionStr, CTPPSDataBindVariables); + //ORDER BY clause + CTPPSDataQuery->addToOrderList(std::string("DIP_UPDATE_TIME")); + //define query output + coral::AttributeList CTPPSDataOutput; + CTPPSDataOutput.extend(std::string("DIP_UPDATE_TIME")); + CTPPSDataOutput.extend(std::string("LUMI_SECTION")); + CTPPSDataOutput.extend(std::string("RUN_NUMBER")); + CTPPSDataOutput.extend(std::string("FILL_NUMBER")); + CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_X_URAD")); + CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_Y_URAD")); + CTPPSDataOutput.extend(std::string("BETA_STAR_P5_X_M")); + CTPPSDataOutput.extend(std::string("BETA_STAR_P5_Y_M")); + CTPPSDataQuery->defineOutput(CTPPSDataOutput); + //execute the query + coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); + unsigned int lumiSection = 0; + cond::Time_t runNumber = 0; + int fillNumber = 0; + float crossingAngleX = 0., betaStarX = 0.; + float crossingAngleY = 0., betaStarY = 0.; + + bool ret = false; + int wrongFillNumbers = 0; + std::stringstream wrongFills; + std::vector>>::iterator current = m_tmpBuffer.begin(); + while (CTPPSDataCursor.next()) { + if (m_debug) { + std::ostringstream CTPPS; + CTPPSDataCursor.currentRow().toOutputStream(CTPPS); + } + coral::Attribute const& dipTimeAttribute = CTPPSDataCursor.currentRow()[std::string("DIP_UPDATE_TIME")]; + if (!dipTimeAttribute.isNull()) { + ret = true; + coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; + if (!lumiSectionAttribute.isNull()) { + lumiSection = lumiSectionAttribute.data(); + } + coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; + if (!runNumberAttribute.isNull()) { + runNumber = runNumberAttribute.data(); + } + coral::Attribute const& fillNumberAttribute = CTPPSDataCursor.currentRow()[std::string("FILL_NUMBER")]; + if (!fillNumberAttribute.isNull()) { + fillNumber = fillNumberAttribute.data(); + } + coral::Attribute const& crossingAngleXAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; + if (!crossingAngleXAttribute.isNull()) { + crossingAngleX = crossingAngleXAttribute.data(); + } + coral::Attribute const& crossingAngleYAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; + if (!crossingAngleYAttribute.isNull()) { + crossingAngleY = crossingAngleYAttribute.data(); + } + coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; + if (!betaStarXAttribute.isNull()) { + betaStarX = betaStarXAttribute.data(); + } + coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; + if (!betaStarYAttribute.isNull()) { + betaStarY = betaStarYAttribute.data(); + } + if (current != m_tmpBuffer.end() && current->second->fillNumber() != fillNumber) { + wrongFills << "( " << runNumber << "_" << lumiSection << " fill: OMS: " << current->second->fillNumber() + << " PPSdb: " << fillNumber << " ) "; + wrongFillNumbers++; + } + for (; current != m_tmpBuffer.end() && make_pair(current->second->runNumber(), current->second->lumiSection()) <= + make_pair(runNumber, lumiSection); + current++) { + LHCInfoPerLS& payload = *(current->second); + payload.setCrossingAngleX(crossingAngleX); + payload.setCrossingAngleY(crossingAngleY); + payload.setBetaStarX(betaStarX); + payload.setBetaStarY(betaStarY); + payload.setLumiSection(lumiSection); + payload.setRunNumber(runNumber); + if (m_lsIdMap.find(make_pair(payload.runNumber(), payload.lumiSection())) != m_lsIdMap.end()) { + m_lsIdMap[make_pair(payload.runNumber(), payload.lumiSection())] = make_pair(runNumber, lumiSection); + } + } + } + } + if (wrongFillNumbers) { + edm::LogWarning("getCTPPSData") << "Number of records from PPS DB with fillNumber different from OMS: " + << wrongFillNumbers; + edm::LogWarning("getCTPPSData") << "Records from PPS DB with fillNumber different from OMS: " << wrongFills.str(); + } + return ret; +} diff --git a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc index 0a344bd479711..e32604bbd20b2 100644 --- a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc +++ b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc @@ -1,5 +1,6 @@ #include "CondCore/CondDB/interface/ConnectionPool.h" #include "CondFormats/Common/interface/TimeConversions.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" #include "CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h" #include "CondTools/RunInfo/interface/LumiSectionFilter.h" #include "CondTools/RunInfo/interface/OMSAccess.h" @@ -159,7 +160,7 @@ size_t LHCInfoPopConSourceHandler::getLumiData(const cond::OMSService& oms, query->addOutputVars({"start_time", "delivered_lumi", "recorded_lumi"}); query->filterEQ("fill_number", fillId); query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); - query->limit(kLumisectionsQueryLimit); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); size_t nlumi = 0; if (query->execute()) { auto res = query->result(); diff --git a/CondTools/RunInfo/test/last_lumi.txt b/CondTools/RunInfo/test/last_lumi.txt new file mode 100644 index 0000000000000..c4ed12a727c79 --- /dev/null +++ b/CondTools/RunInfo/test/last_lumi.txt @@ -0,0 +1 @@ +1663505258250241 diff --git a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh index 750e1501f07a4..8ef3751949388 100755 --- a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh +++ b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh @@ -2,15 +2,24 @@ SCRIPTS_DIR=${CMSSW_BASE}/src/CondTools/RunInfo/python -function die { echo Failure $1: status $2 ; exit $2 ; } +function die { + log_file="$3" + if [ -f "$log_file" ]; then + echo "Log output:" + cat "$log_file" + fi + echo "Failure $1: status $2" + exit $2 +} assert_equal() { expected="$1" actual="$2" message="$3" + log_file="$4" if [ "$expected" != "$actual" ]; then - die "$message: Expected $expected, but got $actual" 1 + die "$message: Expected $expected, but got $actual" 1 "$log_file" fi } @@ -20,7 +29,7 @@ function assert_found_fills { shift 2 for fill_nr in "$@"; do if ! grep -q "Found fill $fill_nr" "$log_file"; then - die "$script_name didn't find fill $fill_nr" 1 # TODO FIX + die "$script_name didn't find fill $fill_nr" 1 "$log_file" fi done } @@ -31,43 +40,45 @@ echo "testing LHCInfoPerFillPopConAnalyzer in endFill mode for startTime=\"2022- cmsRun ${SCRIPTS_DIR}/LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=fill_end_test > fill_end_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill" $? + tag=fill_end_test > fill_end_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py mode=endFill" $? "fill_end_test.log" assert_equal 7 `cat fill_end_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerFillPopConAnalyzer in EndFill mode written wrong number of payloads" + wc -l` "LHCInfoPerFillPopConAnalyzer in EndFill mode written wrong number of payloads" "fill_end_test.log" assert_found_fills fill_end_test.log "LHCInfoPerFillPopConAnalyzer in EndFill mode" 8307 8309 echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=ls_end_test > ls_end_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? + tag=ls_end_test > ls_end_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? "ls_end_test.log" assert_equal 169 `cat ls_end_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" "ls_end_test.log" assert_found_fills ls_end_test.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 8307 8309 echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-07-11 22:00:00.000\" endTime=\"2022-07-12 18:10:10.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-07-11 22:00:00.000" endTime="2022-07-12 18:10:10.000" \ - tag=ls_end_test2 > ls_end_test2.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? + tag=ls_end_test2 > ls_end_test2.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=endFill" $? "ls_end_test2.log" assert_equal 70 `cat ls_end_test2.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" "ls_end_test2.log" assert_found_fills ls_end_test2.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 7967 +echo "1663505258250241" > last_lumi.txt + echo "testing LHCInfoPerFillPopConAnalyzer in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerFillPopConAnalyzer_cfg.py mode=duringFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=fill_during_test > fill_during_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py" $? -assert_equal 1 `cat fill_during_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerFillPopConAnalyzer in DuringFill written wrong number of payloads" -assert_found_fills fill_during_test.log "LHCInfoPerFillPopConAnalyzer in DuringFill" 8307 8309 + lastLumiFile=last_lumi.txt \ + tag=fill_during_test > fill_during_test.log || die "cmsRun LHCInfoPerFillPopConAnalyzer_cfg.py" $? "fill_during_test.log" +assert_equal 1 `cat fill_during_test.log | grep -E 'uploaded with since' | \ + wc -l` "LHCInfoPerFillPopConAnalyzer in DuringFill written wrong number of payloads" "fill_during_test.log" echo "testing LHCInfoPerLSPopConAnalyzer in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ + lastLumiFile=last_lumi.txt \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ - tag=ls_during_test > ls_during_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill" $? -assert_equal 1 `cat ls_during_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzer in duringFill mode written wrong number of payloads" -assert_found_fills ls_during_test.log "LHCInfoPerLSPopConAnalyzer in duringFill mode" 8307 8309 + tag=ls_during_test > ls_during_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer_cfg.py mode=duringFill" $? "ls_during_test.log" +assert_equal 1 `cat ls_during_test.log | grep -E 'uploaded with since' | \ + wc -l` "LHCInfoPerLSPopConAnalyzer in duringFill mode written wrong number of payloads" "ls_during_test.log"