Skip to content

Commit

Permalink
Bugfix: fixed hash to be integers (#222)
Browse files Browse the repository at this point in the history 
* fixed hash to be integers

* optimize hash

* missed type conversion for metadata.
  • Loading branch information
@hariharan-devarajan
hariharan-devarajan authored Oct 11, 2024
1 parent 3a65307 commit 4782269
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 61 deletions.
1 change: 1 addition & 0 deletions include/dftracer/core/constants.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,6 @@ static const int EVENT_TYPE_SIZE = 128;
static const unsigned int DFT_PATH_MAX = 1024 * 4;
static const char SEPARATOR = ';';
static const int HASH_OUTPUT = 16;
#define NO_HASH_DEFAULT 0

#endif // DFTRACER_CONSTANTS_H
1 change: 1 addition & 0 deletions include/dftracer/core/typedef.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ typedef unsigned long int ThreadID;
typedef unsigned long int ProcessID;
typedef char* EventNameType;
typedef const char* ConstEventNameType;
typedef unsigned long long HashType;
#endif // DFTRACER_TYPEDEF_H
24 changes: 13 additions & 11 deletions src/dftracer/brahma/posix.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
#define DFTRACER_POSIX_H

#include <brahma/brahma.h>
#include <dftracer/core/constants.h>
#include <dftracer/core/logging.h>
#include <dftracer/core/typedef.h>
#include <dftracer/df_logger.h>
#include <dftracer/utils/md5.h>
#include <dftracer/utils/utils.h>
Expand All @@ -25,24 +27,24 @@ class POSIXDFTracer : public POSIX {
static bool stop_trace;
static std::shared_ptr<POSIXDFTracer> instance;
static const int MAX_FD = 1024;
std::string tracked_fd[MAX_FD];
HashType tracked_fd[MAX_FD];
std::shared_ptr<DFTLogger> logger;
bool trace_all_files;

inline std::string is_traced(int fd, const char *func) {
if (fd < 0) return std::string();
std::string trace = tracked_fd[fd % MAX_FD];
if (trace.empty()) {
inline HashType is_traced(int fd, const char *func) {
if (fd < 0) return NO_HASH_DEFAULT;
HashType trace = tracked_fd[fd % MAX_FD];
if (trace == NO_HASH_DEFAULT) {
DFTRACER_LOG_DEBUG(
"Calling POSIXDFTracer.is_traced for %s and"
" fd %d trace %d",
func, fd, !trace.empty());
func, fd, trace != NO_HASH_DEFAULT);
}
return trace;
}

inline std::string is_traced(const char *filename, const char *func) {
if (stop_trace) return std::string();
inline HashType is_traced(const char *filename, const char *func) {
if (stop_trace) return NO_HASH_DEFAULT;
if (trace_all_files) {
return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH);
} else {
Expand All @@ -57,7 +59,7 @@ class POSIXDFTracer : public POSIX {
}
}

inline void trace(int fd, std::string hash) {
inline void trace(int fd, HashType hash) {
DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.trace for %d and %d", fd, hash);
if (fd == -1) return;
tracked_fd[fd % MAX_FD] = hash;
Expand All @@ -66,13 +68,13 @@ class POSIXDFTracer : public POSIX {
inline void remove_trace(int fd) {
DFTRACER_LOG_DEBUG("Calling POSIXDFTracer.remove_trace for %d", fd);
if (fd == -1) return;
tracked_fd[fd % MAX_FD] = std::string();
tracked_fd[fd % MAX_FD] = NO_HASH_DEFAULT;
}

public:
POSIXDFTracer(bool trace_all) : POSIX(), trace_all_files(trace_all) {
DFTRACER_LOG_DEBUG("POSIX class intercepted", "");
for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = std::string();
for (int i = 0; i < MAX_FD; ++i) tracked_fd[i] = NO_HASH_DEFAULT;
logger = DFT_LOGGER_INIT();
}
void finalize() {
Expand Down
18 changes: 10 additions & 8 deletions src/dftracer/brahma/stdio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

#include <brahma/brahma.h>
#include <dftracer/core/logging.h>
#include <dftracer/core/typedef.h>
#include <dftracer/core/constants.h>
#include <dftracer/df_logger.h>
#include <dftracer/utils/utils.h>
#include <fcntl.h>
Expand All @@ -22,25 +24,25 @@ class STDIODFTracer : public STDIO {
private:
static bool stop_trace;
static std::shared_ptr<STDIODFTracer> instance;
std::unordered_map<FILE *, std::string> tracked_fh;
std::unordered_map<FILE *, HashType> tracked_fh;
std::shared_ptr<DFTLogger> logger;
bool trace_all_files;

inline std::string is_traced(FILE *fh, const char *func) {
inline HashType is_traced(FILE *fh, const char *func) {
DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced for %s", func);
if (stop_trace) return std::string();
if (fh == NULL) return std::string();
if (stop_trace) return NO_HASH_DEFAULT;
if (fh == NULL) return NO_HASH_DEFAULT;
auto iter = tracked_fh.find(fh);
if (iter != tracked_fh.end()) {
return iter->second;
}
return std::string();
return NO_HASH_DEFAULT;
}

inline std::string is_traced(const char *filename, const char *func) {
inline HashType is_traced(const char *filename, const char *func) {
DFTRACER_LOG_DEBUG("Calling STDIODFTracer.is_traced with filename for %s",
func);
if (stop_trace) return std::string();
if (stop_trace) return NO_HASH_DEFAULT;
if (trace_all_files)
return logger->hash_and_store(filename, METADATA_NAME_FILE_HASH);
else {
Expand All @@ -49,7 +51,7 @@ class STDIODFTracer : public STDIO {
}
}

inline void trace(FILE *fh, std::string hash) {
inline void trace(FILE *fh, HashType hash) {
DFTRACER_LOG_DEBUG("Calling STDIODFTracer.trace with hash %d", hash);
tracked_fh.insert_or_assign(fh, hash);
}
Expand Down
63 changes: 33 additions & 30 deletions src/dftracer/df_logger.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <dftracer/core/constants.h>
#include <dftracer/core/logging.h>
#include <dftracer/core/singleton.h>
#include <dftracer/core/typedef.h>
#include <dftracer/utils/configuration_manager.h>
#include <dftracer/utils/md5.h>
#include <dftracer/utils/utils.h>
Expand All @@ -22,6 +23,7 @@
#include <cstdint>
#include <cstring>
#include <dftracer/dftracer_config.hpp>
#include <string>
#include <unordered_map>

#include "core/enumeration.h"
Expand All @@ -45,7 +47,7 @@ class DFTLogger {
std::shared_ptr<dftracer::ChromeWriter> writer;
uint32_t level;
std::vector<int> index_stack;
std::unordered_map<std::string, std::string> computed_hash;
std::unordered_map<std::string, HashType> computed_hash;
std::atomic_int index;
bool has_entry;
#ifdef DFTRACER_MPI_ENABLE
Expand Down Expand Up @@ -104,15 +106,15 @@ class DFTLogger {
}
~DFTLogger() {}

inline std::string get_hash(char *name) {
inline HashType get_hash(char *name) {
uint8_t result[HASH_OUTPUT];
md5String(name, result);
std::string hash;
hash.reserve(HASH_OUTPUT + 1);
char hash_str[HASH_OUTPUT + 1];
for (int i = 0; i < HASH_OUTPUT; ++i) {
sprintf(hash.data() + i, "%02x", result[i]);
sprintf(hash_str + i, "%02x", result[i]);
}
hash.data()[HASH_OUTPUT] = '\0';
hash_str[HASH_OUTPUT] = '\0';
HashType hash = std::stoull(hash_str, nullptr, 16);
return hash;
}

Expand All @@ -125,15 +127,15 @@ class DFTLogger {
tid = df_gettid();
}
this->writer = dftracer::Singleton<dftracer::ChromeWriter>::get_instance();
std::string hostname_hash;
std::string cmd_hash;
std::string exec_hash;
HashType hostname_hash;
HashType cmd_hash;
HashType exec_hash;
if (this->writer != nullptr) {
char hostname[256];
gethostname(hostname, 256);
hostname_hash = get_hash(hostname);
this->writer->initialize(log_file.data(), this->throw_error,
hostname_hash.c_str());
hostname_hash);
hostname_hash = hash_and_store(hostname, METADATA_NAME_HOSTNAME_HASH);
char thread_name[128];
auto size = sprintf(thread_name, "%lu", this->process_id);
Expand Down Expand Up @@ -232,16 +234,16 @@ class DFTLogger {
return -1;
}

inline std::string has_hash(ConstEventNameType key) {
inline HashType has_hash(ConstEventNameType key) {
std::shared_lock<std::shared_mutex> lock(map_mtx);
auto iter = computed_hash.find(key);
if (iter != computed_hash.end()) return iter->second.c_str();
return std::string();
if (iter != computed_hash.end()) return iter->second;
return NO_HASH_DEFAULT;
}

inline void insert_hash(ConstEventNameType key, std::string hash) {
inline void insert_hash(ConstEventNameType key, HashType hash) {
std::unique_lock<std::shared_mutex> lock(map_mtx);
computed_hash.insert_or_assign(key, hash.c_str());
computed_hash.insert_or_assign(key, hash);
}

inline TimeResolution get_time() {
Expand Down Expand Up @@ -331,8 +333,8 @@ class DFTLogger {
}
}

inline std::string hash_and_store(char *filename, ConstEventNameType name) {
if (filename == NULL) return std::string();
inline HashType hash_and_store(char *filename, ConstEventNameType name) {
if (filename == NULL) return NO_HASH_DEFAULT;
char file[PATH_MAX];
strcpy(file, filename);
file[PATH_MAX - 1] = '\0';
Expand All @@ -359,30 +361,31 @@ class DFTLogger {
}
}

inline std::string hash_and_store_str(char file[PATH_MAX],
ConstEventNameType name) {
std::string hash = has_hash(file);
if (hash.empty()) {
inline HashType hash_and_store_str(char file[PATH_MAX],
ConstEventNameType name) {
HashType hash = has_hash(file);
if (hash == NO_HASH_DEFAULT) {
hash = get_hash(file);
insert_hash(file, hash.c_str());
insert_hash(file, hash);
if (this->writer != nullptr) {
ThreadID tid = 0;
if (dftracer_tid) {
tid = df_gettid();
}
fix_str(file, PATH_MAX);
int current_index = this->enter_event();
this->writer->log_metadata(current_index, file, hash.c_str(), name,
this->process_id, tid);
this->writer->log_metadata(current_index, file,
std::to_string(hash).c_str(), name,
this->process_id, tid, false);
this->exit_event();
}
}
return hash;
}

inline std::string hash_and_store(const char *filename,
ConstEventNameType name) {
if (filename == NULL) return std::string();
inline HashType hash_and_store(const char *filename,
ConstEventNameType name) {
if (filename == NULL) return NO_HASH_DEFAULT;
char file[PATH_MAX];
strcpy(file, filename);
file[PATH_MAX - 1] = '\0';
Expand Down Expand Up @@ -416,15 +419,15 @@ class DFTLogger {

#define DFT_LOGGER_UPDATE_HASH(value) \
if (trace && this->logger->include_metadata) { \
std::string value##_hash = \
HashType value##_hash = \
this->logger->hash_and_store(value, METADATA_NAME_FILE_HASH); \
DFT_LOGGER_UPDATE(value##_hash); \
}

#define DFT_LOGGER_START(entity) \
DFTRACER_LOG_DEBUG("Calling function %s", __FUNCTION__); \
std::string fhash = is_traced(entity, __FUNCTION__); \
bool trace = !fhash.empty(); \
HashType fhash = is_traced(entity, __FUNCTION__); \
bool trace = fhash != NO_HASH_DEFAULT; \
TimeResolution start_time = 0; \
std::unordered_map<std::string, std::any> *metadata = nullptr; \
if (trace) { \
Expand Down
24 changes: 14 additions & 10 deletions src/dftracer/writer/chrome_writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ template <>
bool dftracer::Singleton<dftracer::ChromeWriter>::stop_creating_instances =
false;
void dftracer::ChromeWriter::initialize(char *filename, bool throw_error,
const char *hostname_hash) {
HashType hostname_hash) {
this->hostname_hash = hostname_hash;
this->throw_error = throw_error;
this->filename = filename;
Expand Down Expand Up @@ -185,6 +185,11 @@ void dftracer::ChromeWriter::convert_json(
<< "\":" << std::any_cast<uint16_t>(item.second) << "";
if (i < meta_size - 1) meta_stream << ",";

} else if (item.second.type() == typeid(HashType)) {
meta_stream << "\"" << item.first
<< "\":" << std::any_cast<HashType>(item.second) << "";
if (i < meta_size - 1) meta_stream << ",";

} else if (item.second.type() == typeid(long)) {
meta_stream << "\"" << item.first
<< "\":" << std::any_cast<long>(item.second) << "";
Expand Down Expand Up @@ -214,10 +219,9 @@ void dftracer::ChromeWriter::convert_json(
previous_index = current_index;
auto written_size = sprintf(
buffer.data() + current_index,
R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":"%s"%s}})",
R"(%s{"id":%d,"name":"%s","cat":"%s","pid":%lu,"tid":%lu,"ts":%llu,"dur":%llu,"ph":"X","args":{"hhash":%llu%s}})",
is_first_char, index, event_name, category, process_id, thread_id,
start_time, duration, this->hostname_hash.c_str(),
all_stream.str().c_str());
start_time, duration, this->hostname_hash, all_stream.str().c_str());
current_index += written_size;
buffer[current_index] = '\n';
current_index++;
Expand Down Expand Up @@ -256,15 +260,15 @@ void dftracer::ChromeWriter::convert_json_metadata(
if (is_string) {
written_size = sprintf(
buffer.data() + current_index,
R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":"%s"}})",
is_first_char, index, ph, process_id, thread_id,
this->hostname_hash.c_str(), name, value);
R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%llu,"name":"%s","value":"%s"}})",
is_first_char, index, ph, process_id, thread_id, this->hostname_hash,
name, value);
} else {
written_size = sprintf(
buffer.data() + current_index,
R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":"%s","name":"%s","value":%s}})",
is_first_char, index, ph, process_id, thread_id,
this->hostname_hash.c_str(), name, value);
R"(%s{"id":%d,"name":"%s","cat":"dftracer","pid":%lu,"tid":%lu,"ph":"M","args":{"hhash":%llu,"name":"%s","value":%s}})",
is_first_char, index, ph, process_id, thread_id, this->hostname_hash,
name, value);
}
current_index += written_size;
buffer[current_index] = '\n';
Expand Down
4 changes: 2 additions & 2 deletions src/dftracer/writer/chrome_writer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class ChromeWriter {
bool enable_core_affinity;

FILE *fh;
std::string hostname_hash;
HashType hostname_hash;
static const int MAX_LINE_SIZE = 16 * 1024L;
size_t write_buffer_size;

Expand Down Expand Up @@ -100,7 +100,7 @@ class ChromeWriter {
}
}
~ChromeWriter() { DFTRACER_LOG_DEBUG("Destructing ChromeWriter", ""); }
void initialize(char *filename, bool throw_error, const char *hostname_hash);
void initialize(char *filename, bool throw_error, HashType hostname_hash);

void log(int index, ConstEventNameType event_name,
ConstEventNameType category, TimeResolution start_time,
Expand Down

0 comments on commit 4782269

Please sign in to comment.