diff --git a/dlio_profiler/logger.py b/dlio_profiler/logger.py index a9c53a61..71bad527 100644 --- a/dlio_profiler/logger.py +++ b/dlio_profiler/logger.py @@ -1,8 +1,11 @@ from functools import wraps from typing import Dict import os -DLIO_PROFILER_ENABLE = True if os.getenv("DLIO_PROFILER_ENABLE", '1') == '1' else False -DLIO_PROFILER_INIT_PRELOAD = True if os.getenv("DLIO_PROFILER_INIT", 'PRELOAD') == 'PRELOAD' else False +DLIO_PROFILER_ENABLE_ENV = "DLIO_PROFILER_ENABLE" +DLIO_PROFILER_INIT_ENV = "DLIO_PROFILER_INIT" + +DLIO_PROFILER_ENABLE = True if os.getenv(DLIO_PROFILER_ENABLE_ENV, '1') == '1' else False +DLIO_PROFILER_INIT_PRELOAD = True if os.getenv(DLIO_PROFILER_INIT_ENV, 'PRELOAD') == 'PRELOAD' else False if DLIO_PROFILER_ENABLE: import dlio_profiler_py as profiler diff --git a/src/dlio_profiler/core/constants.h b/src/dlio_profiler/core/constants.h index 3e4308d9..3bf29cd4 100644 --- a/src/dlio_profiler/core/constants.h +++ b/src/dlio_profiler/core/constants.h @@ -11,4 +11,7 @@ #define DLIO_PROFILER_INIT "DLIO_PROFILER_INIT" #define DLIO_PROFILER_INIT_COUNT "DLIO_PROFILER_INIT_COUNT" #define DLIO_PROFILER_DATA_DIR "DLIO_PROFILER_DATA_DIR" +#define DLIO_PROFILER_SET_CORE_AFFINITY "DLIO_PROFILER_SET_CORE_AFFINITY" +#define DLIO_PROFILER_ERROR "DLIO_PROFILER_ERROR" +#define DLIO_PROFILER_INC_METADATA "DLIO_PROFILER_INC_METADATA" #endif //DLIO_PROFILER_CONSTANTS_H diff --git a/src/dlio_profiler/dlio_logger.h b/src/dlio_profiler/dlio_logger.h index 5cdd12ad..ee4c2c2f 100644 --- a/src/dlio_profiler/dlio_logger.h +++ b/src/dlio_profiler/dlio_logger.h @@ -19,12 +19,16 @@ typedef std::chrono::high_resolution_clock chrono; class DLIOLogger { private: TimeResolution library_start; - bool throw_error; + bool throw_error, include_metadata; std::shared_ptr writer; bool is_init; int process_id; public: - DLIOLogger(bool init_log = false):is_init(false) { + DLIOLogger(bool init_log = false):is_init(false), include_metadata(false) { + char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA); + if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) { + include_metadata = true; + } char *dlio_profiler_error = getenv("DLIO_PROFILER_ERROR"); if (dlio_profiler_error != nullptr && strcmp(dlio_profiler_error, "1") == 0) { throw_error = true; @@ -72,6 +76,9 @@ class DLIOLogger { std::unordered_map &metadata) { writer->log(event_name, category, start_time, duration, metadata, process_id); } + inline bool has_metadata() { + return this->include_metadata; + } inline void finalize() { writer->finalize(); } @@ -80,7 +87,7 @@ class DLIOLogger { dlio_profiler::Singleton::get_instance() #define DLIO_LOGGER_FINI() \ dlio_profiler::Singleton::get_instance()->finalize() -#define DLIO_LOGGER_UPDATE(value) if (trace) metadata.insert_or_assign(#value, value); +#define DLIO_LOGGER_UPDATE(value) if (trace && this->logger->has_metadata()) metadata.insert_or_assign(#value, value); #define DLIO_LOGGER_START(entity) \ auto pair = is_traced(entity, __FUNCTION__); \ bool trace = pair.first; \ diff --git a/src/dlio_profiler/writer/chrome_writer.cpp b/src/dlio_profiler/writer/chrome_writer.cpp index cb31f66e..0d4d0a8f 100644 --- a/src/dlio_profiler/writer/chrome_writer.cpp +++ b/src/dlio_profiler/writer/chrome_writer.cpp @@ -89,48 +89,51 @@ dlio_profiler::ChromeWriter::convert_json(std::string &event_name, std::string & << "\"dur\":" << std::chrono::duration_cast(duration_sec).count() << "," << R"("ph":"X",)" << R"("args":{)"; - all_stream << "\"hostname\":\"" << hostname() << "\","; - all_stream << "\"core_affinity\": ["; - auto cores = core_affinity(); - auto cores_size = cores.size(); - for(int i = 0; i < cores_size; ++i) { + if (include_metadata) { + all_stream << "\"hostname\":\"" << hostname() << "\","; + all_stream << "\"core_affinity\": ["; + auto cores = core_affinity(); + auto cores_size = cores.size(); + for (int i = 0; i < cores_size; ++i) { all_stream << cores[i]; if (i < cores_size - 1) all_stream << ","; - } + } - all_stream << "]"; - auto meta_size = metadata.size(); - if (meta_size > 0) all_stream << ","; - int i = 0; - for(auto item : metadata) { - if (item.second.type() == typeid(int)) { - all_stream << "\"" << item.first << "\":" << std::any_cast(item.second); - if (i < meta_size - 1) all_stream << ","; - } else if (item.second.type() == typeid(const char *)) { - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - } else if (item.second.type() == typeid(std::string)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - }else if (item.second.type() == typeid(size_t)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - }else if (item.second.type() == typeid(long)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - }else if (item.second.type() == typeid(ssize_t)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - }else if (item.second.type() == typeid(off_t)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; - }else if (item.second.type() == typeid(off64_t)){ - all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; - if (i < meta_size - 1) all_stream << ","; + all_stream << "]"; + auto meta_size = metadata.size(); + if (meta_size > 0) all_stream << ","; + int i = 0; + for (auto item : metadata) { + if (item.second.type() == typeid(int)) { + all_stream << "\"" << item.first << "\":" << std::any_cast(item.second); + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(const char *)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(std::string)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(size_t)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(long)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(ssize_t)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(off_t)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } else if (item.second.type() == typeid(off64_t)) { + all_stream << "\"" << item.first << "\":\"" << std::any_cast(item.second) << "\""; + if (i < meta_size - 1) all_stream << ","; + } + i++; } - i++; + all_stream << "}"; } - all_stream << "}}\n"; + all_stream << "}\n"; DLIO_PROFILER_LOGINFO("event logged %s", all_stream.str().c_str()); return all_stream.str(); } diff --git a/src/dlio_profiler/writer/chrome_writer.h b/src/dlio_profiler/writer/chrome_writer.h index ebb60349..d67b3aa2 100644 --- a/src/dlio_profiler/writer/chrome_writer.h +++ b/src/dlio_profiler/writer/chrome_writer.h @@ -11,9 +11,12 @@ #include #include #include +#include + namespace dlio_profiler { class ChromeWriter: public BaseWriter { private: + bool enable_core_affinity, include_metadata; hwloc_topology_t topology; int fd; std::string convert_json(std::string &event_name, std::string &category, TimeResolution start_time, TimeResolution duration, @@ -23,12 +26,14 @@ namespace dlio_profiler { std::unordered_map mtx_map; std::vector core_affinity() { auto cores = std::vector(); - hwloc_cpuset_t set = hwloc_bitmap_alloc(); - hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS); - for (unsigned id = hwloc_bitmap_first(set); id != -1; id = hwloc_bitmap_next(set, id)) { - cores.push_back(id); + if (enable_core_affinity) { + hwloc_cpuset_t set = hwloc_bitmap_alloc(); + hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS); + for (unsigned id = hwloc_bitmap_first(set); id != -1; id = hwloc_bitmap_next(set, id)) { + cores.push_back(id); + } + hwloc_bitmap_free(set); } - hwloc_bitmap_free(set); return cores; } std::string hostname() { @@ -38,11 +43,21 @@ namespace dlio_profiler { return std::string(hostname); } public: - ChromeWriter(int fd=-1):BaseWriter(), is_first_write(true), mtx_map(){ + ChromeWriter(int fd=-1):BaseWriter(), is_first_write(true), mtx_map(), enable_core_affinity(false), include_metadata(false){ + char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA); + if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) { + include_metadata = true; + } + char *enable_core_affinity_str = getenv(DLIO_PROFILER_SET_CORE_AFFINITY); + if (enable_core_affinity_str != nullptr && strcmp(enable_core_affinity_str, "1") == 0) { + enable_core_affinity = true; + } process_id = getpid(); this->fd = fd; - hwloc_topology_init(&topology); // initialization - hwloc_topology_load(topology); // actual detection + if (enable_core_affinity) { + hwloc_topology_init(&topology); // initialization + hwloc_topology_load(topology); // actual detection + } } void initialize(char *filename, bool throw_error) override;