Skip to content

Commit

Permalink
make metdata optional.
Browse files Browse the repository at this point in the history
  • Loading branch information
@hariharan-devarajan
hariharan-devarajan committed Oct 6, 2023
1 parent e8b4668 commit 1d28b44
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 50 deletions.
7 changes: 5 additions & 2 deletions dlio_profiler/logger.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
from functools import wraps
from typing import Dict
import os
DLIO_PROFILER_ENABLE = True if os.getenv("DLIO_PROFILER_ENABLE", '1') == '1' else False
DLIO_PROFILER_INIT_PRELOAD = True if os.getenv("DLIO_PROFILER_INIT", 'PRELOAD') == 'PRELOAD' else False
DLIO_PROFILER_ENABLE_ENV = "DLIO_PROFILER_ENABLE"
DLIO_PROFILER_INIT_ENV = "DLIO_PROFILER_INIT"

DLIO_PROFILER_ENABLE = True if os.getenv(DLIO_PROFILER_ENABLE_ENV, '1') == '1' else False
DLIO_PROFILER_INIT_PRELOAD = True if os.getenv(DLIO_PROFILER_INIT_ENV, 'PRELOAD') == 'PRELOAD' else False

if DLIO_PROFILER_ENABLE:
import dlio_profiler_py as profiler
Expand Down
3 changes: 3 additions & 0 deletions src/dlio_profiler/core/constants.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,7 @@
#define DLIO_PROFILER_INIT "DLIO_PROFILER_INIT"
#define DLIO_PROFILER_INIT_COUNT "DLIO_PROFILER_INIT_COUNT"
#define DLIO_PROFILER_DATA_DIR "DLIO_PROFILER_DATA_DIR"
#define DLIO_PROFILER_SET_CORE_AFFINITY "DLIO_PROFILER_SET_CORE_AFFINITY"
#define DLIO_PROFILER_ERROR "DLIO_PROFILER_ERROR"
#define DLIO_PROFILER_INC_METADATA "DLIO_PROFILER_INC_METADATA"
#endif //DLIO_PROFILER_CONSTANTS_H
13 changes: 10 additions & 3 deletions src/dlio_profiler/dlio_logger.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@ typedef std::chrono::high_resolution_clock chrono;
class DLIOLogger {
private:
TimeResolution library_start;
bool throw_error;
bool throw_error, include_metadata;
std::shared_ptr<dlio_profiler::BaseWriter> writer;
bool is_init;
int process_id;
public:
DLIOLogger(bool init_log = false):is_init(false) {
DLIOLogger(bool init_log = false):is_init(false), include_metadata(false) {
char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA);
if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) {
include_metadata = true;
}
char *dlio_profiler_error = getenv("DLIO_PROFILER_ERROR");
if (dlio_profiler_error != nullptr && strcmp(dlio_profiler_error, "1") == 0) {
throw_error = true;
Expand Down Expand Up @@ -72,6 +76,9 @@ class DLIOLogger {
std::unordered_map<std::string, std::any> &metadata) {
writer->log(event_name, category, start_time, duration, metadata, process_id);
}
inline bool has_metadata() {
return this->include_metadata;
}
inline void finalize() {
writer->finalize();
}
Expand All @@ -80,7 +87,7 @@ class DLIOLogger {
dlio_profiler::Singleton<DLIOLogger>::get_instance()
#define DLIO_LOGGER_FINI() \
dlio_profiler::Singleton<DLIOLogger>::get_instance()->finalize()
#define DLIO_LOGGER_UPDATE(value) if (trace) metadata.insert_or_assign(#value, value);
#define DLIO_LOGGER_UPDATE(value) if (trace && this->logger->has_metadata()) metadata.insert_or_assign(#value, value);
#define DLIO_LOGGER_START(entity) \
auto pair = is_traced(entity, __FUNCTION__); \
bool trace = pair.first; \
Expand Down
77 changes: 40 additions & 37 deletions src/dlio_profiler/writer/chrome_writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,48 +89,51 @@ dlio_profiler::ChromeWriter::convert_json(std::string &event_name, std::string &
<< "\"dur\":" << std::chrono::duration_cast<std::chrono::microseconds>(duration_sec).count() << ","
<< R"("ph":"X",)"
<< R"("args":{)";
all_stream << "\"hostname\":\"" << hostname() << "\",";
all_stream << "\"core_affinity\": [";
auto cores = core_affinity();
auto cores_size = cores.size();
for(int i = 0; i < cores_size; ++i) {
if (include_metadata) {
all_stream << "\"hostname\":\"" << hostname() << "\",";
all_stream << "\"core_affinity\": [";
auto cores = core_affinity();
auto cores_size = cores.size();
for (int i = 0; i < cores_size; ++i) {
all_stream << cores[i];
if (i < cores_size - 1) all_stream << ",";
}
}

all_stream << "]";
auto meta_size = metadata.size();
if (meta_size > 0) all_stream << ",";
int i = 0;
for(auto item : metadata) {
if (item.second.type() == typeid(int)) {
all_stream << "\"" << item.first << "\":" << std::any_cast<int>(item.second);
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(const char *)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<const char *>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(std::string)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<std::string>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}else if (item.second.type() == typeid(size_t)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<size_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}else if (item.second.type() == typeid(long)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<long>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}else if (item.second.type() == typeid(ssize_t)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<ssize_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}else if (item.second.type() == typeid(off_t)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<off_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}else if (item.second.type() == typeid(off64_t)){
all_stream << "\"" << item.first << "\":\"" << std::any_cast<off64_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
all_stream << "]";
auto meta_size = metadata.size();
if (meta_size > 0) all_stream << ",";
int i = 0;
for (auto item : metadata) {
if (item.second.type() == typeid(int)) {
all_stream << "\"" << item.first << "\":" << std::any_cast<int>(item.second);
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(const char *)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<const char *>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(std::string)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<std::string>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(size_t)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<size_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(long)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<long>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(ssize_t)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<ssize_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(off_t)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<off_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
} else if (item.second.type() == typeid(off64_t)) {
all_stream << "\"" << item.first << "\":\"" << std::any_cast<off64_t>(item.second) << "\"";
if (i < meta_size - 1) all_stream << ",";
}
i++;
}
i++;
all_stream << "}";
}
all_stream << "}}\n";
all_stream << "}\n";
DLIO_PROFILER_LOGINFO("event logged %s", all_stream.str().c_str());
return all_stream.str();
}
31 changes: 23 additions & 8 deletions src/dlio_profiler/writer/chrome_writer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,12 @@
#include <mutex>
#include <unistd.h>
#include <hwloc.h>
#include <dlio_profiler/core/constants.h>

namespace dlio_profiler {
class ChromeWriter: public BaseWriter {
private:
bool enable_core_affinity, include_metadata;
hwloc_topology_t topology;
int fd;
std::string convert_json(std::string &event_name, std::string &category, TimeResolution start_time, TimeResolution duration,
Expand All @@ -23,12 +26,14 @@ namespace dlio_profiler {
std::unordered_map<int, std::mutex> mtx_map;
std::vector<unsigned> core_affinity() {
auto cores = std::vector<unsigned>();
hwloc_cpuset_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
for (unsigned id = hwloc_bitmap_first(set); id != -1; id = hwloc_bitmap_next(set, id)) {
cores.push_back(id);
if (enable_core_affinity) {
hwloc_cpuset_t set = hwloc_bitmap_alloc();
hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
for (unsigned id = hwloc_bitmap_first(set); id != -1; id = hwloc_bitmap_next(set, id)) {
cores.push_back(id);
}
hwloc_bitmap_free(set);
}
hwloc_bitmap_free(set);
return cores;
}
std::string hostname() {
Expand All @@ -38,11 +43,21 @@ namespace dlio_profiler {
return std::string(hostname);
}
public:
ChromeWriter(int fd=-1):BaseWriter(), is_first_write(true), mtx_map(){
ChromeWriter(int fd=-1):BaseWriter(), is_first_write(true), mtx_map(), enable_core_affinity(false), include_metadata(false){
char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA);
if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) {
include_metadata = true;
}
char *enable_core_affinity_str = getenv(DLIO_PROFILER_SET_CORE_AFFINITY);
if (enable_core_affinity_str != nullptr && strcmp(enable_core_affinity_str, "1") == 0) {
enable_core_affinity = true;
}
process_id = getpid();
this->fd = fd;
hwloc_topology_init(&topology); // initialization
hwloc_topology_load(topology); // actual detection
if (enable_core_affinity) {
hwloc_topology_init(&topology); // initialization
hwloc_topology_load(topology); // actual detection
}
}
void initialize(char *filename, bool throw_error) override;

Expand Down

0 comments on commit 1d28b44

Please sign in to comment.