From e04ef8d56ccdfb16837a470cfbf73f3ed2ed244d Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Mon, 9 Oct 2023 18:37:17 -0700 Subject: [PATCH] code reformatting --- .clang-format | 1 + .github/workflows/ci.yml | 10 +- dlio_profiler/logger.py | 11 +- .../dlio_profiler/core/dlio_profiler_main.h | 18 +- include/dlio_profiler/core/enumeration.h | 12 +- include/dlio_profiler/core/error.h | 10 +- include/dlio_profiler/core/macro.h | 2 + include/dlio_profiler/core/singleton.h | 12 +- include/dlio_profiler/dlio_profiler.h | 17 +- src/dlio_profiler/brahma/posix.cpp | 36 +-- src/dlio_profiler/brahma/posix.h | 218 ++++++++++-------- src/dlio_profiler/brahma/stdio.cpp | 11 +- src/dlio_profiler/brahma/stdio.h | 130 ++++++----- src/dlio_profiler/core/dlio_profiler_main.cpp | 67 +++--- src/dlio_profiler/dlio_logger.h | 22 +- src/dlio_profiler/dlio_profiler.cpp | 10 +- src/dlio_profiler/dlio_profiler_preload.cpp | 13 +- src/dlio_profiler/dlio_profiler_preload.h | 1 + src/dlio_profiler/dlio_profiler_py.cpp | 34 +-- src/dlio_profiler/utils/posix_internal.cpp | 3 +- src/dlio_profiler/utils/utils.h | 103 +++++---- src/dlio_profiler/writer/chrome_writer.cpp | 32 +-- src/dlio_profiler/writer/chrome_writer.h | 21 +- test/CMakeLists.txt | 8 +- test/c/test.c | 9 +- test/cpp/test.cpp | 18 +- test/dlio_benchmark/CMakeLists.txt | 4 +- test/py/test.py | 25 +- 28 files changed, 501 insertions(+), 357 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..2593ef54 --- /dev/null +++ b/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17f9abf4..4ca127a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,15 @@ jobs: sudo apt-get update sudo apt-get install gcc g++ libtool-bin openmpi-bin libopenmpi-dev sudo apt-get install python3.10 python3-pip - sudo apt-get install gcovr + sudo apt-get install gcovr clang-format + - name: Check code formatting + if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user' + run: | + git clang-format --diff HEAD~1 -q + if git clang-format --diff HEAD~1 -q | grep -m 1 '^' >/dev/null; then + echo >&2 "Failed clang-format check. Run: git clang-format HEAD~1" + exit 1 + fi - name: Install dlio benchmark if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user' run: | diff --git a/dlio_profiler/logger.py b/dlio_profiler/logger.py index a6f1f4ba..a2d8ca18 100644 --- a/dlio_profiler/logger.py +++ b/dlio_profiler/logger.py @@ -1,6 +1,7 @@ from functools import wraps from typing import Dict import os + DLIO_PROFILER_ENABLE_ENV = "DLIO_PROFILER_ENABLE" DLIO_PROFILER_INIT_ENV = "DLIO_PROFILER_INIT" @@ -12,13 +13,17 @@ from pathlib import Path import inspect -import sys,signal +import sys, signal + + def capture_signal(signal_number, frame): dlio_logger.get_instance().finalize() sys.exit(1) + signal.signal(signal.SIGABRT, capture_signal) + class dlio_logger: __instance = None @@ -38,7 +43,7 @@ def get_instance(cls, log_file=None): @staticmethod def initialize_log(logfile, data_dir, process_id): log_file = None - if logfile : + if logfile: log_file = Path(logfile) instance = dlio_logger.get_instance(log_file) if DLIO_PROFILER_ENABLE: @@ -51,7 +56,7 @@ def initialize_log(logfile, data_dir, process_id): log_file = f"{instance.logfile}" if data_dir: data_dir = f"{data_dir}" - instance.logger.initialize(log_file = log_file, data_dirs = data_dir, process_id=process_id) + instance.logger.initialize(log_file=log_file, data_dirs=data_dir, process_id=process_id) return instance def get_time(self): diff --git a/include/dlio_profiler/core/dlio_profiler_main.h b/include/dlio_profiler/core/dlio_profiler_main.h index 9bfb3192..790405e7 100644 --- a/include/dlio_profiler/core/dlio_profiler_main.h +++ b/include/dlio_profiler/core/dlio_profiler_main.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_DLIO_PROFILER_MAIN_H #define DLIO_PROFILER_DLIO_PROFILER_MAIN_H + #include #include #include @@ -31,16 +32,23 @@ namespace dlio_profiler { int process_id; bool is_initialized; bool bind; - void initlialize(bool is_init, bool _bind, const char *_log_file = nullptr, const char *_data_dirs = nullptr, const int *_process_id = nullptr); + + void initlialize(bool is_init, bool _bind, const char *_log_file = nullptr, const char *_data_dirs = nullptr, + const int *_process_id = nullptr); + public: - DLIOProfilerCore(ProfilerStage stage, ProfileType type, const char *log_file = nullptr, const char *data_dirs = nullptr, const int *process_id = nullptr); + DLIOProfilerCore(ProfilerStage stage, ProfileType type, const char *log_file = nullptr, + const char *data_dirs = nullptr, const int *process_id = nullptr); + inline bool is_active() { return is_enabled; } + TimeResolution get_time(); - void log(const char* event_name, const char* category, - TimeResolution start_time, TimeResolution duration, - std::unordered_map &metadata); + + void log(const char *event_name, const char *category, + TimeResolution start_time, TimeResolution duration, + std::unordered_map &metadata); bool finalize(); }; diff --git a/include/dlio_profiler/core/enumeration.h b/include/dlio_profiler/core/enumeration.h index 5b34bf39..3b67e017 100644 --- a/include/dlio_profiler/core/enumeration.h +++ b/include/dlio_profiler/core/enumeration.h @@ -4,7 +4,13 @@ #ifndef DLIO_PROFILER_ENUMERATION_H #define DLIO_PROFILER_ENUMERATION_H -enum WriterType : uint8_t { CHROME = 0}; -enum ProfilerStage: uint8_t { PROFILER_INIT=0, PROFILER_FINI=1, PROFILER_OTHER=2}; -enum ProfileType: uint8_t { PROFILER_PRELOAD=0, PROFILER_PY_APP=1, PROFILER_CPP_APP=2, PROFILER_C_APP=3}; +enum WriterType : uint8_t { + CHROME = 0 +}; +enum ProfilerStage : uint8_t { + PROFILER_INIT = 0, PROFILER_FINI = 1, PROFILER_OTHER = 2 +}; +enum ProfileType : uint8_t { + PROFILER_PRELOAD = 0, PROFILER_PY_APP = 1, PROFILER_CPP_APP = 2, PROFILER_C_APP = 3 +}; #endif //DLIO_PROFILER_ENUMERATION_H diff --git a/include/dlio_profiler/core/error.h b/include/dlio_profiler/core/error.h index 4b2acca7..aac83dfc 100644 --- a/include/dlio_profiler/core/error.h +++ b/include/dlio_profiler/core/error.h @@ -6,8 +6,8 @@ #define DLIO_PROFILER_ERROR_H struct ErrorCode { - const char* code; - const char* message; + const char *code; + const char *message; }; // Main error codes @@ -18,7 +18,9 @@ const ErrorCode FAILURE = {"1001", "Internal Failure"}; const ErrorCode UNKNOWN_PROFILER_TYPE = {"1002", "Code 1002: Unknown profiler type %d"}; // Invalid configurations -const ErrorCode UNDEFINED_DATA_DIR = {"2001", "Code 2001: Data dirs not defined. Please define env variable DLIO_PROFILER_DATA_DIR"}; -const ErrorCode UNDEFINED_LOG_FILE = {"2002", "Code 2002: log file not defined. Please define env variable DLIO_PROFILER_LOG_FILE"}; +const ErrorCode UNDEFINED_DATA_DIR = {"2001", + "Code 2001: Data dirs not defined. Please define env variable DLIO_PROFILER_DATA_DIR"}; +const ErrorCode UNDEFINED_LOG_FILE = {"2002", + "Code 2002: log file not defined. Please define env variable DLIO_PROFILER_LOG_FILE"}; #endif //DLIO_PROFILER_ERROR_H diff --git a/include/dlio_profiler/core/macro.h b/include/dlio_profiler/core/macro.h index 3185feee..fc79060d 100644 --- a/include/dlio_profiler/core/macro.h +++ b/include/dlio_profiler/core/macro.h @@ -4,7 +4,9 @@ #ifndef DLIO_PROFILER_MACRO_H #define DLIO_PROFILER_MACRO_H + #include + #define DLIO_PROFILER_LOGGER cpplogger::Logger::Instance("DLIO_PROFILER") #define DLIO_PROFILER_LOGINFO(format, ...) \ DLIO_PROFILER_LOGGER->log(cpplogger::LOG_INFO, format, __VA_ARGS__); diff --git a/include/dlio_profiler/core/singleton.h b/include/dlio_profiler/core/singleton.h index e1f8104a..b2a0792b 100644 --- a/include/dlio_profiler/core/singleton.h +++ b/include/dlio_profiler/core/singleton.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_SINGLETON_H #define DLIO_PROFILER_SINGLETON_H + #include #include #include @@ -13,7 +14,7 @@ * @tparam T */ namespace dlio_profiler { - template + template class Singleton { public: /** @@ -24,7 +25,7 @@ namespace dlio_profiler { * @tparam T * @return instance of T */ - template + template static std::shared_ptr get_instance(Args... args) { if (instance == nullptr) instance = std::shared_ptr(new T(std::forward(args)...)); @@ -34,19 +35,20 @@ namespace dlio_profiler { /** * Operators */ - Singleton& operator=(const Singleton) = delete; /* deleting = operatos*/ + Singleton &operator=(const Singleton) = delete; /* deleting = operatos*/ /** * Constructor */ public: - Singleton(const Singleton&) = delete; /* deleting copy constructor. */ + Singleton(const Singleton &) = delete; /* deleting copy constructor. */ protected: static std::shared_ptr instance; + Singleton() {} /* hidden default constructor. */ }; - template + template std::shared_ptr Singleton::instance = nullptr; } // namespace dlio_profiler #endif //DLIO_PROFILER_SINGLETON_H diff --git a/include/dlio_profiler/dlio_profiler.h b/include/dlio_profiler/dlio_profiler.h index e8552501..3a4eb44a 100644 --- a/include/dlio_profiler/dlio_profiler.h +++ b/include/dlio_profiler/dlio_profiler.h @@ -10,6 +10,7 @@ */ #include #include + #ifdef __cplusplus /** * CPP Only @@ -24,25 +25,28 @@ #include class DLIOProfiler { - const char* name; + const char *name; TimeResolution start_time; std::unordered_map metadata; std::shared_ptr dlio_profiler_core; public: - DLIOProfiler(const char* _name):name(_name), metadata(){ + DLIOProfiler(const char *_name) : name(_name), metadata() { dlio_profiler_core = DLIO_PROFILER_MAIN_SINGLETON(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_CPP_APP); start_time = dlio_profiler_core->get_time(); } - inline void update(const char* key, int value) { + + inline void update(const char *key, int value) { if (dlio_profiler_core->is_active()) { metadata.insert_or_assign(key, value); } } - inline void update(const char* key, const char* value) { + + inline void update(const char *key, const char *value) { if (dlio_profiler_core->is_active()) { metadata.insert_or_assign(key, value); } } + ~DLIOProfiler() { if (dlio_profiler_core->is_active()) { TimeResolution end_time = dlio_profiler_core->get_time(); @@ -50,6 +54,7 @@ class DLIOProfiler { } } }; + #define DLIO_PROFILER_CPP_INIT(log_file, data_dirs, process_id) \ DLIO_PROFILER_MAIN_SINGLETON_INIT(ProfilerStage::PROFILER_INIT, ProfileType::PROFILER_CPP_APP, log_file, data_dirs, process_id); #define DLIO_PROFILER_CPP_FINI() \ @@ -69,9 +74,9 @@ delete profiler_##name extern "C" { #endif // C APIs -void initialize(const char * log_file, const char * data_dirs, int* process_id); +void initialize(const char *log_file, const char *data_dirs, int *process_id); TimeResolution get_time(); -void log_event(const char * name, const char * cat, TimeResolution start_time, TimeResolution duration); +void log_event(const char *name, const char *cat, TimeResolution start_time, TimeResolution duration); void finalize(); #define DLIO_PROFILER_C_INIT(log_file, data_dirs, process_id) \ diff --git a/src/dlio_profiler/brahma/posix.cpp b/src/dlio_profiler/brahma/posix.cpp index fec26328..46d6f75d 100644 --- a/src/dlio_profiler/brahma/posix.cpp +++ b/src/dlio_profiler/brahma/posix.cpp @@ -7,6 +7,7 @@ #define CATEGORY "POSIX" std::shared_ptr brahma::POSIXDLIOProfiler::instance = nullptr; + int brahma::POSIXDLIOProfiler::open(const char *pathname, int flags, ...) { BRAHMA_MAP_OR_FAIL(open); DLIO_LOGGER_START(pathname); @@ -28,6 +29,7 @@ int brahma::POSIXDLIOProfiler::open(const char *pathname, int flags, ...) { if (trace) this->trace(ret); return ret; } + int brahma::POSIXDLIOProfiler::close(int fd) { BRAHMA_MAP_OR_FAIL(close); DLIO_LOGGER_START(fd); @@ -38,6 +40,7 @@ int brahma::POSIXDLIOProfiler::close(int fd) { if (trace) this->remove_trace(fd); return ret; } + ssize_t brahma::POSIXDLIOProfiler::write(int fd, const void *buf, size_t count) { BRAHMA_MAP_OR_FAIL(write); DLIO_LOGGER_START(fd); @@ -48,6 +51,7 @@ ssize_t brahma::POSIXDLIOProfiler::write(int fd, const void *buf, size_t count) DLIO_LOGGER_END(); return ret; } + ssize_t brahma::POSIXDLIOProfiler::read(int fd, void *buf, size_t count) { BRAHMA_MAP_OR_FAIL(read); DLIO_LOGGER_START(fd); @@ -58,6 +62,7 @@ ssize_t brahma::POSIXDLIOProfiler::read(int fd, void *buf, size_t count) { DLIO_LOGGER_END(); return ret; } + off_t brahma::POSIXDLIOProfiler::lseek(int fd, off_t offset, int whence) { BRAHMA_MAP_OR_FAIL(lseek); DLIO_LOGGER_START(fd); @@ -93,7 +98,7 @@ int brahma::POSIXDLIOProfiler::open64(const char *path, int flags, ...) { va_end(args); DLIO_LOGGER_UPDATE(mode) ret = __real_open64(path, flags, mode); - }else { + } else { ret = __real_open64(path, flags); } DLIO_LOGGER_UPDATE(path) @@ -211,7 +216,7 @@ void *brahma::POSIXDLIOProfiler::mmap(void *addr, size_t length, int prot, int f DLIO_LOGGER_UPDATE(flags); DLIO_LOGGER_UPDATE(offset); DLIO_LOGGER_UPDATE(fd); - void* ret = __real_mmap(addr, length, prot, flags, fd, offset); + void *ret = __real_mmap(addr, length, prot, flags, fd, offset); DLIO_LOGGER_END(); return ret; } @@ -223,7 +228,7 @@ void *brahma::POSIXDLIOProfiler::mmap64(void *addr, size_t length, int prot, int DLIO_LOGGER_UPDATE(flags); DLIO_LOGGER_UPDATE(offset); DLIO_LOGGER_UPDATE(fd); - void* ret = __real_mmap64(addr, length, prot, flags, fd, offset); + void *ret = __real_mmap64(addr, length, prot, flags, fd, offset); DLIO_LOGGER_END(); return ret; } @@ -435,18 +440,19 @@ int brahma::POSIXDLIOProfiler::utime(const char *filename, const utimbuf *buf) { return ret; } -DIR* brahma::POSIXDLIOProfiler::opendir(const char *name) { +DIR *brahma::POSIXDLIOProfiler::opendir(const char *name) { BRAHMA_MAP_OR_FAIL(opendir); DLIO_LOGGER_START(name); DLIO_LOGGER_UPDATE(name); - DIR* ret = __real_opendir(name); + DIR *ret = __real_opendir(name); DLIO_LOGGER_END(); return ret; } int brahma::POSIXDLIOProfiler::fcntl(int fd, int cmd, ...) { BRAHMA_MAP_OR_FAIL(fcntl); - if(cmd==F_DUPFD || cmd==F_DUPFD_CLOEXEC || cmd==F_SETFD || cmd==F_SETFL || cmd==F_SETOWN) { // arg: int + if (cmd == F_DUPFD || cmd == F_DUPFD_CLOEXEC || cmd == F_SETFD || cmd == F_SETFL || + cmd == F_SETOWN) { // arg: int va_list arg; va_start(arg, cmd); int val = va_arg(arg, int); @@ -457,14 +463,14 @@ int brahma::POSIXDLIOProfiler::fcntl(int fd, int cmd, ...) { int ret = __real_fcntl(fd, cmd, val); DLIO_LOGGER_END(); return ret; - } else if(cmd==F_GETFD || cmd==F_GETFL || cmd==F_GETOWN) { + } else if (cmd == F_GETFD || cmd == F_GETFL || cmd == F_GETOWN) { DLIO_LOGGER_START(fd); DLIO_LOGGER_UPDATE(fd); DLIO_LOGGER_UPDATE(cmd); int ret = __real_fcntl(fd, cmd); DLIO_LOGGER_END(); return ret; - } else if(cmd==F_SETLK || cmd==F_SETLKW || cmd==F_GETLK) { + } else if (cmd == F_SETLK || cmd == F_SETLKW || cmd == F_GETLK) { va_list arg; va_start(arg, cmd); struct flock *lk = va_arg(arg, struct flock*); @@ -512,12 +518,14 @@ int brahma::POSIXDLIOProfiler::mkfifo(const char *pathname, mode_t mode) { return ret; } -mode_t brahma::POSIXDLIOProfiler::umask(mode_t mask) { - BRAHMA_MAP_OR_FAIL(umask); - DLIO_LOGGER_START(mask); - mode_t ret = __real_umask(mask); - DLIO_LOGGER_END(); - return ret; +mode_t brahma::POSIXDLIOProfiler::umask(mode_t +mask) { +BRAHMA_MAP_OR_FAIL(umask); +DLIO_LOGGER_START(mask); +mode_t ret = __real_umask(mask); +DLIO_LOGGER_END(); +return +ret; } int brahma::POSIXDLIOProfiler::access(const char *path, int amode) { diff --git a/src/dlio_profiler/brahma/posix.h b/src/dlio_profiler/brahma/posix.h index 21cbab42..85028dbb 100644 --- a/src/dlio_profiler/brahma/posix.h +++ b/src/dlio_profiler/brahma/posix.h @@ -14,142 +14,164 @@ #include #include #include + namespace fs = std::filesystem; namespace brahma { -class POSIXDLIOProfiler : public POSIX { - private: - static std::shared_ptr instance; - std::unordered_set tracked_fd; - std::vector track_filename; - std::vector ignore_filename; - std::shared_ptr logger; - - inline std::pair is_traced(int fd, const char* func) { - if (fd == -1) return std::pair(false, ""); - auto iter = tracked_fd.find(fd); - if (iter != tracked_fd.end()) return std::pair(true, ""); - return is_traced(get_filename(fd).c_str(), func); - } - inline std::pair is_traced(const char* filename, const char* func) { - return is_traced_common(filename, func, ignore_filename, track_filename); - } - inline void trace(int fd) { - tracked_fd.insert(fd); - } - inline void remove_trace(int fd) { - tracked_fd.erase(fd); - } - public: - POSIXDLIOProfiler() : POSIX() { - DLIO_PROFILER_LOGINFO("POSIX class intercepted", ""); - logger = DLIO_LOGGER_INIT(); - } - inline void trace(const char* filename) { - char resolved_path[PATH_MAX]; - char* data = realpath(filename, resolved_path); - (void) data; - track_filename.push_back(resolved_path); - } - inline void untrace(const char* filename) { - char resolved_path[PATH_MAX]; - char* data = realpath(filename, resolved_path); - (void) data; - ignore_filename.push_back(resolved_path); - } - ~POSIXDLIOProfiler() override = default; // GCOVR_EXCL_LINE - static std::shared_ptr get_instance() { - if (instance == nullptr) { - instance = std::make_shared(); - POSIX::set_instance(instance); - } - return instance; - } - int open(const char *pathname, int flags, ...) override; - int creat64(const char *path, mode_t mode) override; - int open64(const char *path, int flags, ...) override; - int close(int fd) override; - ssize_t write(int fd, const void *buf, size_t count) override; - ssize_t read(int fd, void *buf, size_t count) override; - off_t lseek(int fd, off_t offset, int whence) override; - off64_t lseek64(int fd, off64_t offset, int whence) override; - ssize_t pread(int fd, void *buf, size_t count, off_t offset) override; - ssize_t pread64(int fd, void *buf, size_t count, off64_t offset) override; - ssize_t pwrite(int fd, const void *buf, size_t count, off64_t offset) override; - ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset) override; - int fsync(int fd) override; - int fdatasync(int fd) override; + class POSIXDLIOProfiler : public POSIX { + private: + static std::shared_ptr instance; + std::unordered_set tracked_fd; + std::vector track_filename; + std::vector ignore_filename; + std::shared_ptr logger; + + inline std::pair is_traced(int fd, const char *func) { + if (fd == -1) return std::pair(false, ""); + auto iter = tracked_fd.find(fd); + if (iter != tracked_fd.end()) return std::pair(true, ""); + return is_traced(get_filename(fd).c_str(), func); + } + + inline std::pair is_traced(const char *filename, const char *func) { + return is_traced_common(filename, func, ignore_filename, track_filename); + } + + inline void trace(int fd) { + tracked_fd.insert(fd); + } + + inline void remove_trace(int fd) { + tracked_fd.erase(fd); + } + + public: + POSIXDLIOProfiler() : POSIX() { + DLIO_PROFILER_LOGINFO("POSIX class intercepted", ""); + logger = DLIO_LOGGER_INIT(); + } + + inline void trace(const char *filename) { + char resolved_path[PATH_MAX]; + char *data = realpath(filename, resolved_path); + (void) data; + track_filename.push_back(resolved_path); + } + + inline void untrace(const char *filename) { + char resolved_path[PATH_MAX]; + char *data = realpath(filename, resolved_path); + (void) data; + ignore_filename.push_back(resolved_path); + } + + ~POSIXDLIOProfiler() override = default; // GCOVR_EXCL_LINE + static std::shared_ptr get_instance() { + if (instance == nullptr) { + instance = std::make_shared(); + POSIX::set_instance(instance); + } + return instance; + } + + int open(const char *pathname, int flags, ...) override; + + int creat64(const char *path, mode_t mode) override; + + int open64(const char *path, int flags, ...) override; + + int close(int fd) override; + + ssize_t write(int fd, const void *buf, size_t count) override; + + ssize_t read(int fd, void *buf, size_t count) override; + + off_t lseek(int fd, off_t offset, int whence) override; + + off64_t lseek64(int fd, off64_t offset, int whence) override; + + ssize_t pread(int fd, void *buf, size_t count, off_t offset) override; + + ssize_t pread64(int fd, void *buf, size_t count, off64_t offset) override; + + ssize_t pwrite(int fd, const void *buf, size_t count, off64_t offset) override; + + ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset) override; + + int fsync(int fd) override; + + int fdatasync(int fd) override; - int openat(int dirfd, const char *pathname, int flags, ...) override; + int openat(int dirfd, const char *pathname, int flags, ...) override; - void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override; + void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override; - void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off64_t offset) override; + void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off64_t offset) override; - int __xstat(int vers, const char *path, struct stat *buf) override; + int __xstat(int vers, const char *path, struct stat *buf) override; - int __xstat64(int vers, const char *path, struct stat64 *buf) override; + int __xstat64(int vers, const char *path, struct stat64 *buf) override; - int __lxstat(int vers, const char *path, struct stat *buf) override; + int __lxstat(int vers, const char *path, struct stat *buf) override; - int __lxstat64(int vers, const char *path, struct stat64 *buf) override; + int __lxstat64(int vers, const char *path, struct stat64 *buf) override; - int __fxstat(int vers, int fd, struct stat *buf) override; + int __fxstat(int vers, int fd, struct stat *buf) override; - int __fxstat64(int vers, int fd, struct stat64 *buf) override; + int __fxstat64(int vers, int fd, struct stat64 *buf) override; - int mkdir(const char *pathname, mode_t mode) override; + int mkdir(const char *pathname, mode_t mode) override; - int rmdir(const char *pathname) override; + int rmdir(const char *pathname) override; - int chdir(const char *path) override; + int chdir(const char *path) override; - int link(const char *oldpath, const char *newpath) override; + int link(const char *oldpath, const char *newpath) override; - int linkat(int fd1, const char *path1, int fd2, const char *path2, int flag) override; + int linkat(int fd1, const char *path1, int fd2, const char *path2, int flag) override; - int unlink(const char *pathname) override; + int unlink(const char *pathname) override; - int symlink(const char *path1, const char *path2) override; + int symlink(const char *path1, const char *path2) override; - int symlinkat(const char *path1, int fd, const char *path2) override; + int symlinkat(const char *path1, int fd, const char *path2) override; - ssize_t readlink(const char *path, char *buf, size_t bufsize) override; + ssize_t readlink(const char *path, char *buf, size_t bufsize) override; - ssize_t readlinkat(int fd, const char *path, char *buf, size_t bufsize) override; + ssize_t readlinkat(int fd, const char *path, char *buf, size_t bufsize) override; - int rename(const char *oldpath, const char *newpath) override; + int rename(const char *oldpath, const char *newpath) override; - int chmod(const char *path, mode_t mode) override; + int chmod(const char *path, mode_t mode) override; - int chown(const char *path, uid_t owner, gid_t group) override; + int chown(const char *path, uid_t owner, gid_t group) override; - int lchown(const char *path, uid_t owner, gid_t group) override; + int lchown(const char *path, uid_t owner, gid_t group) override; - int utime(const char *filename, const utimbuf *buf) override; + int utime(const char *filename, const utimbuf *buf) override; - DIR *opendir(const char *name) override; + DIR *opendir(const char *name) override; - int fcntl(int fd, int cmd, ...) override; + int fcntl(int fd, int cmd, ...) override; - int dup(int oldfd) override; + int dup(int oldfd) override; - int dup2(int oldfd, int newfd) override; + int dup2(int oldfd, int newfd) override; - int mkfifo(const char *pathname, mode_t mode) override; + int mkfifo(const char *pathname, mode_t mode) override; - mode_t umask(mode_t mask) override; + mode_t umask(mode_t mask) override; - int access(const char *path, int amode) override; + int access(const char *path, int amode) override; - int faccessat(int fd, const char *path, int amode, int flag) override; + int faccessat(int fd, const char *path, int amode, int flag) override; - int remove(const char *pathname) override; + int remove(const char *pathname) override; - int truncate(const char *pathname, off_t length) override; + int truncate(const char *pathname, off_t length) override; - int ftruncate(int fd, off_t length) override; -}; + int ftruncate(int fd, off_t length) override; + }; } // namespace brahma #endif // DLIO_PROFILER_POSIX_H diff --git a/src/dlio_profiler/brahma/stdio.cpp b/src/dlio_profiler/brahma/stdio.cpp index ee4b878b..4ba7c2ee 100644 --- a/src/dlio_profiler/brahma/stdio.cpp +++ b/src/dlio_profiler/brahma/stdio.cpp @@ -8,6 +8,7 @@ #define CATEGORY "STDIO" std::shared_ptr brahma::STDIODLIOProfiler::instance = nullptr; + FILE *brahma::STDIODLIOProfiler::fopen64(const char *path, const char *mode) { BRAHMA_MAP_OR_FAIL(fopen64); DLIO_LOGGER_START(path); @@ -18,6 +19,7 @@ FILE *brahma::STDIODLIOProfiler::fopen64(const char *path, const char *mode) { if (trace) this->trace(ret); return ret; } + FILE *brahma::STDIODLIOProfiler::fopen(const char *path, const char *mode) { BRAHMA_MAP_OR_FAIL(fopen); DLIO_LOGGER_START(path); @@ -28,6 +30,7 @@ FILE *brahma::STDIODLIOProfiler::fopen(const char *path, const char *mode) { if (trace) this->trace(ret); return ret; } + int brahma::STDIODLIOProfiler::fclose(FILE *fp) { BRAHMA_MAP_OR_FAIL(fclose); DLIO_LOGGER_START(fp); @@ -36,8 +39,9 @@ int brahma::STDIODLIOProfiler::fclose(FILE *fp) { if (trace) this->remove_trace(fp); return ret; } + size_t brahma::STDIODLIOProfiler::fread(void *ptr, size_t size, size_t nmemb, - FILE *fp) { + FILE *fp) { BRAHMA_MAP_OR_FAIL(fread); DLIO_LOGGER_START(fp); DLIO_LOGGER_UPDATE(size); @@ -46,8 +50,9 @@ size_t brahma::STDIODLIOProfiler::fread(void *ptr, size_t size, size_t nmemb, DLIO_LOGGER_END(); return ret; } + size_t brahma::STDIODLIOProfiler::fwrite(const void *ptr, size_t size, size_t nmemb, - FILE *fp) { + FILE *fp) { BRAHMA_MAP_OR_FAIL(fwrite); DLIO_LOGGER_START(fp); DLIO_LOGGER_UPDATE(size); @@ -56,6 +61,7 @@ size_t brahma::STDIODLIOProfiler::fwrite(const void *ptr, size_t size, size_t nm DLIO_LOGGER_END(); return ret; } + long brahma::STDIODLIOProfiler::ftell(FILE *fp) { BRAHMA_MAP_OR_FAIL(ftell); DLIO_LOGGER_START(fp); @@ -63,6 +69,7 @@ long brahma::STDIODLIOProfiler::ftell(FILE *fp) { DLIO_LOGGER_END(); return ret; } + int brahma::STDIODLIOProfiler::fseek(FILE *fp, long offset, int whence) { BRAHMA_MAP_OR_FAIL(fseek); DLIO_LOGGER_START(fp); diff --git a/src/dlio_profiler/brahma/stdio.h b/src/dlio_profiler/brahma/stdio.h index 22cfd4af..6ad07bb6 100644 --- a/src/dlio_profiler/brahma/stdio.h +++ b/src/dlio_profiler/brahma/stdio.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_STDIO_H #define DLIO_PROFILER_STDIO_H + #include #include #include @@ -12,66 +13,81 @@ #include #include #include + namespace fs = std::filesystem; namespace brahma { -class STDIODLIOProfiler : public STDIO { - private: - static std::shared_ptr instance; - std::unordered_set tracked_fh; - std::vector track_filename; - std::vector ignore_filename; - std::shared_ptr logger; - inline std::pair is_traced(FILE* fh, const char* func) { - if (fh == NULL) return std::pair(false, ""); - auto iter = tracked_fh.find(fh); - if (iter != tracked_fh.end()) return std::pair(true, ""); - return is_traced(get_filename(fileno(fh)).c_str(), func); - } - inline std::pair is_traced(const char* filename, const char* func) { - return is_traced_common(filename, func, ignore_filename, track_filename); - } - inline void trace(FILE* fh) { - tracked_fh.insert(fh); - } - inline void remove_trace(FILE* fh) { - tracked_fh.erase(fh); - } - public: - STDIODLIOProfiler() : STDIO(), tracked_fh(), track_filename() { - DLIO_PROFILER_LOGINFO("STDIO class intercepted", ""); - logger = DLIO_LOGGER_INIT(); - } - - inline void trace(const char* filename) { - char resolved_path[PATH_MAX]; - char* data = realpath(filename, resolved_path); - (void) data; - track_filename.push_back(resolved_path); - } - inline void untrace(const char* filename) { - char resolved_path[PATH_MAX]; - char* data = realpath(filename, resolved_path); - (void) data; - ignore_filename.push_back(resolved_path); - } - - ~STDIODLIOProfiler() = default; - static std::shared_ptr get_instance() { - if (instance == nullptr) { - instance = std::make_shared(); - STDIO::set_instance(instance); - } - return instance; - } - FILE *fopen(const char *path, const char *mode) override; - FILE *fopen64(const char *path, const char *mode) override; - int fclose(FILE *fp) override; - size_t fread(void *ptr, size_t size, size_t nmemb, FILE *fp) override; - size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *fp) override; - long ftell(FILE *fp) override; - int fseek(FILE *fp, long offset, int whence) override; -}; + class STDIODLIOProfiler : public STDIO { + private: + static std::shared_ptr instance; + std::unordered_set tracked_fh; + std::vector track_filename; + std::vector ignore_filename; + std::shared_ptr logger; + + inline std::pair is_traced(FILE *fh, const char *func) { + if (fh == NULL) return std::pair(false, ""); + auto iter = tracked_fh.find(fh); + if (iter != tracked_fh.end()) return std::pair(true, ""); + return is_traced(get_filename(fileno(fh)).c_str(), func); + } + + inline std::pair is_traced(const char *filename, const char *func) { + return is_traced_common(filename, func, ignore_filename, track_filename); + } + + inline void trace(FILE *fh) { + tracked_fh.insert(fh); + } + + inline void remove_trace(FILE *fh) { + tracked_fh.erase(fh); + } + + public: + STDIODLIOProfiler() : STDIO(), tracked_fh(), track_filename() { + DLIO_PROFILER_LOGINFO("STDIO class intercepted", ""); + logger = DLIO_LOGGER_INIT(); + } + + inline void trace(const char *filename) { + char resolved_path[PATH_MAX]; + char *data = realpath(filename, resolved_path); + (void) data; + track_filename.push_back(resolved_path); + } + + inline void untrace(const char *filename) { + char resolved_path[PATH_MAX]; + char *data = realpath(filename, resolved_path); + (void) data; + ignore_filename.push_back(resolved_path); + } + + ~STDIODLIOProfiler() = default; + + static std::shared_ptr get_instance() { + if (instance == nullptr) { + instance = std::make_shared(); + STDIO::set_instance(instance); + } + return instance; + } + + FILE *fopen(const char *path, const char *mode) override; + + FILE *fopen64(const char *path, const char *mode) override; + + int fclose(FILE *fp) override; + + size_t fread(void *ptr, size_t size, size_t nmemb, FILE *fp) override; + + size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *fp) override; + + long ftell(FILE *fp) override; + + int fseek(FILE *fp, long offset, int whence) override; + }; } // namespace brahma #endif // DLIO_PROFILER_STDIO_H diff --git a/src/dlio_profiler/core/dlio_profiler_main.cpp b/src/dlio_profiler/core/dlio_profiler_main.cpp index a1ce7925..fd71950c 100644 --- a/src/dlio_profiler/core/dlio_profiler_main.cpp +++ b/src/dlio_profiler/core/dlio_profiler_main.cpp @@ -9,9 +9,11 @@ #include dlio_profiler::DLIOProfilerCore::DLIOProfilerCore(ProfilerStage stage, ProfileType type, const char *log_file, - const char *data_dirs, const int *process_id): is_enabled( - false), gotcha_priority(1), logger_level(cpplogger::LoggerType::LOG_ERROR), log_file(), data_dirs(), is_initialized(false), bind(false) { - const char* user_init_type = getenv(DLIO_PROFILER_INIT); + const char *data_dirs, const int *process_id) : is_enabled( + false), gotcha_priority(1), logger_level(cpplogger::LoggerType::LOG_ERROR), log_file(), data_dirs(), + is_initialized(false), + bind(false) { + const char *user_init_type = getenv(DLIO_PROFILER_INIT); switch (type) { case ProfileType::PROFILER_PRELOAD: { if (stage == ProfilerStage::PROFILER_INIT) { @@ -46,7 +48,7 @@ dlio_profiler::DLIOProfilerCore::DLIOProfilerCore(ProfilerStage stage, ProfileTy void dlio_profiler::DLIOProfilerCore::log(const char *event_name, const char *category, TimeResolution start_time, TimeResolution duration, - std::unordered_map &metadata) { + std::unordered_map &metadata) { if (this->is_initialized && is_enabled) { dlio_profiler::Singleton::get_instance(false)-> log(event_name, category, start_time, duration, metadata); @@ -68,7 +70,7 @@ bool dlio_profiler::DLIOProfilerCore::finalize() { void dlio_profiler::DLIOProfilerCore::initlialize(bool is_init, bool _bind, const char *_log_file, const char *_data_dirs, - const int *_process_id) { + const int *_process_id) { this->bind = _bind; set_signal(); if (is_init) { @@ -108,37 +110,38 @@ dlio_profiler::DLIOProfilerCore::initlialize(bool is_init, bool _bind, const cha char proc_name[PATH_MAX], cmd[128]; sprintf(cmd, "/proc/%d/cmdline", getpid()); int fd = dlp_open(cmd, O_RDONLY); - const char* exec_file_name = nullptr; + const char *exec_file_name = nullptr; if (fd != -1) { - ssize_t read_bytes = dlp_read(fd, proc_name, PATH_MAX); - dlp_close(fd); - int index = 0, prev = 0; - char exec_name[PATH_MAX]; - while (index < read_bytes) { - if (proc_name[index] == '\0') { - strcpy(exec_name, proc_name + prev); - exec_file_name = basename(exec_name); - if (strstr(exec_file_name, "python") == nullptr) { - break; - } - prev = index + 1; - + ssize_t read_bytes = dlp_read(fd, proc_name, PATH_MAX); + dlp_close(fd); + int index = 0, prev = 0; + char exec_name[PATH_MAX]; + while (index < read_bytes) { + if (proc_name[index] == '\0') { + strcpy(exec_name, proc_name + prev); + exec_file_name = basename(exec_name); + if (strstr(exec_file_name, "python") == nullptr) { + break; } - index ++; + prev = index + 1; + } + index++; + } + } + DLIO_PROFILER_LOGINFO("Extracted process_name %s", exec_file_name); + if (dlio_profiler_log != nullptr) { + if (exec_file_name != nullptr) { + this->log_file = std::string(dlio_profiler_log) + "-" + std::string(exec_file_name) + "-" + + std::to_string(this->process_id) + ".pfw"; + } else { + this->log_file = std::string(dlio_profiler_log) + "-" + std::to_string(this->process_id) + ".pfw"; } - DLIO_PROFILER_LOGINFO("Extracted process_name %s", exec_file_name); - if (dlio_profiler_log != nullptr) { - if (exec_file_name != nullptr) { - this->log_file = std::string(dlio_profiler_log) + "-" + std::string(exec_file_name) + "-" + std::to_string(this->process_id) + ".pfw"; - } else { - this->log_file = std::string(dlio_profiler_log) + "-" + std::to_string(this->process_id) + ".pfw"; - } - } else { // GCOV_EXCL_START - DLIO_PROFILER_LOGERROR(UNDEFINED_LOG_FILE.message, ""); - throw std::runtime_error(UNDEFINED_LOG_FILE.code); - } // GCOV_EXCL_STOP + } else { // GCOV_EXCL_START + DLIO_PROFILER_LOGERROR(UNDEFINED_LOG_FILE.message, ""); + throw std::runtime_error(UNDEFINED_LOG_FILE.code); + } // GCOV_EXCL_STOP } else { this->log_file = _log_file; } @@ -174,7 +177,7 @@ dlio_profiler::DLIOProfilerCore::initlialize(bool is_init, bool _bind, const cha } } -TimeResolution dlio_profiler::DLIOProfilerCore::get_time() { +TimeResolution dlio_profiler::DLIOProfilerCore::get_time() { if (this->is_initialized && is_enabled) { return dlio_profiler::Singleton::get_instance(false)->get_time(); } diff --git a/src/dlio_profiler/dlio_logger.h b/src/dlio_profiler/dlio_logger.h index 6c2bbc00..a31faa4d 100644 --- a/src/dlio_profiler/dlio_logger.h +++ b/src/dlio_profiler/dlio_logger.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_GENERIC_LOGGER_H #define DLIO_PROFILER_GENERIC_LOGGER_H + #include #include #include @@ -16,6 +17,7 @@ #include typedef std::chrono::high_resolution_clock chrono; + class DLIOLogger { private: TimeResolution library_start; @@ -24,7 +26,7 @@ class DLIOLogger { bool is_init; int process_id; public: - DLIOLogger(bool init_log = false):is_init(false), include_metadata(false) { + DLIOLogger(bool init_log = false) : is_init(false), include_metadata(false) { char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA); if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) { include_metadata = true; @@ -33,7 +35,7 @@ class DLIOLogger { if (dlio_profiler_error != nullptr && strcmp(dlio_profiler_error, "1") == 0) { throw_error = true; // GCOVR_EXCL_LINE } - this->is_init=true; + this->is_init = true; int fd = -1; std::string log_file; if (log_file.empty()) { @@ -56,33 +58,41 @@ class DLIOLogger { } update_log_file(log_file); } - inline TimeResolution get_current_time(){ - return std::chrono::duration(std::chrono::high_resolution_clock::now().time_since_epoch()).count(); + + inline TimeResolution get_current_time() { + return std::chrono::duration( + std::chrono::high_resolution_clock::now().time_since_epoch()).count(); } + inline void update_log_file(std::string log_file, int process_id = -1) { this->process_id = process_id; writer = std::make_shared(-1); writer->initialize(log_file.data(), this->throw_error); - this->is_init=true; + this->is_init = true; library_start = get_current_time(); DLIO_PROFILER_LOGINFO("Writing trace to %s with time %f", log_file.c_str(), library_start); } + inline TimeResolution get_time() { - auto t = get_current_time() - library_start; + auto t = get_current_time() - library_start; return t; } + inline void log(std::string event_name, std::string category, TimeResolution start_time, TimeResolution duration, std::unordered_map &metadata) { writer->log(event_name, category, start_time, duration, metadata, process_id); } + inline bool has_metadata() { return this->include_metadata; } + inline void finalize() { writer->finalize(); } }; + #define DLIO_LOGGER_INIT() \ dlio_profiler::Singleton::get_instance() #define DLIO_LOGGER_FINI() \ diff --git a/src/dlio_profiler/dlio_profiler.cpp b/src/dlio_profiler/dlio_profiler.cpp index ef89b455..1995004f 100644 --- a/src/dlio_profiler/dlio_profiler.cpp +++ b/src/dlio_profiler/dlio_profiler.cpp @@ -5,17 +5,19 @@ #include #include -void initialize(const char * log_file, const char * data_dirs, int* process_id) { - DLIO_PROFILER_MAIN_SINGLETON_INIT(ProfilerStage::PROFILER_INIT, ProfileType::PROFILER_C_APP, log_file, data_dirs, process_id); +void initialize(const char *log_file, const char *data_dirs, int *process_id) { + DLIO_PROFILER_MAIN_SINGLETON_INIT(ProfilerStage::PROFILER_INIT, ProfileType::PROFILER_C_APP, log_file, data_dirs, + process_id); } TimeResolution get_time() { return DLIO_PROFILER_MAIN_SINGLETON(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_C_APP)->get_time(); } -void log_event(const char * name, const char * cat, TimeResolution start_time, TimeResolution duration) { +void log_event(const char *name, const char *cat, TimeResolution start_time, TimeResolution duration) { auto args = std::unordered_map(); - DLIO_PROFILER_MAIN_SINGLETON(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_C_APP)->log(name, cat, start_time, duration, args); + DLIO_PROFILER_MAIN_SINGLETON(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_C_APP)->log(name, cat, start_time, + duration, args); } void finalize() { diff --git a/src/dlio_profiler/dlio_profiler_preload.cpp b/src/dlio_profiler/dlio_profiler_preload.cpp index 99a4e1a4..b34d8451 100644 --- a/src/dlio_profiler/dlio_profiler_preload.cpp +++ b/src/dlio_profiler/dlio_profiler_preload.cpp @@ -11,17 +11,22 @@ namespace dlio_profiler { bool init = false; } -bool is_init() {return dlio_profiler::init;} -void set_init(bool _init) { dlio_profiler::init = _init;} +bool is_init() { return dlio_profiler::init; } + +void set_init(bool _init) { dlio_profiler::init = _init; } + void dlio_profiler_init(void) { if (!is_init()) { - dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_INIT, ProfileType::PROFILER_PRELOAD); + dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_INIT, + ProfileType::PROFILER_PRELOAD); set_init(true); } } + void dlio_profiler_fini(void) { if (is_init()) { - dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_FINI, ProfileType::PROFILER_PRELOAD)->finalize(); + dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_FINI, + ProfileType::PROFILER_PRELOAD)->finalize(); set_init(false); } diff --git a/src/dlio_profiler/dlio_profiler_preload.h b/src/dlio_profiler/dlio_profiler_preload.h index a29a7b19..16144ffe 100644 --- a/src/dlio_profiler/dlio_profiler_preload.h +++ b/src/dlio_profiler/dlio_profiler_preload.h @@ -6,6 +6,7 @@ #define DLIO_PROFILER_DLIO_PROFILER_PRELOAD_H extern void __attribute__ ((constructor)) dlio_profiler_init(void); + extern void __attribute__ ((destructor)) dlio_profiler_fini(void); #endif //DLIO_PROFILER_DLIO_PROFILER_PRELOAD_H diff --git a/src/dlio_profiler/dlio_profiler_py.cpp b/src/dlio_profiler/dlio_profiler_py.cpp index 024c932c..e41d7e1d 100644 --- a/src/dlio_profiler/dlio_profiler_py.cpp +++ b/src/dlio_profiler/dlio_profiler_py.cpp @@ -19,25 +19,33 @@ namespace py = pybind11; namespace dlio_profiler { - - void initialize(const char* log_file, const char* data_dirs, int process_id) { - dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_INIT, ProfileType::PROFILER_PY_APP, log_file, data_dirs, &process_id); + void initialize(const char *log_file, const char *data_dirs, int process_id) { + dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_INIT, + ProfileType::PROFILER_PY_APP, log_file, + data_dirs, &process_id); } + TimeResolution get_time() { - return dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_PY_APP)->get_time(); + return dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_OTHER, + ProfileType::PROFILER_PY_APP)->get_time(); } + void log_event(std::string &name, std::string &cat, TimeResolution start_time, TimeResolution duration, - std::unordered_map &int_args, - std::unordered_map &string_args, - std::unordered_map &float_args) { + std::unordered_map &int_args, + std::unordered_map &string_args, + std::unordered_map &float_args) { auto args = std::unordered_map(); for (auto item:int_args) args.insert_or_assign(item.first, item.second); for (auto item:string_args) args.insert_or_assign(item.first, item.second); for (auto item:float_args) args.insert_or_assign(item.first, item.second); - dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_OTHER, ProfileType::PROFILER_PY_APP)->log(name.c_str(), cat.c_str(), start_time, duration, args); + dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_OTHER, + ProfileType::PROFILER_PY_APP)->log( + name.c_str(), cat.c_str(), start_time, duration, args); } + void finalize() { - dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_FINI, ProfileType::PROFILER_PY_APP)->finalize(); + dlio_profiler::Singleton::get_instance(ProfilerStage::PROFILER_FINI, + ProfileType::PROFILER_PY_APP)->finalize(); } } // dlio_profiler PYBIND11_MODULE(dlio_profiler_py, m) { @@ -48,10 +56,10 @@ PYBIND11_MODULE(dlio_profiler_py, m) { py::arg("process_id") = -1); m.def("get_time", &dlio_profiler::get_time, "get time from profiler"); m.def("log_event", &dlio_profiler::log_event, "log event with args", - py::arg("name"), py::arg("cat"), py::arg("start_time"), py::arg("duration"), - py::arg("int_args") = std::unordered_map(), - py::arg("string_args") = std::unordered_map(), - py::arg("float_args") = std::unordered_map()); + py::arg("name"), py::arg("cat"), py::arg("start_time"), py::arg("duration"), + py::arg("int_args") = std::unordered_map(), + py::arg("string_args") = std::unordered_map(), + py::arg("float_args") = std::unordered_map()); m.def("finalize", &dlio_profiler::finalize, "finalize dlio profiler"); } diff --git a/src/dlio_profiler/utils/posix_internal.cpp b/src/dlio_profiler/utils/posix_internal.cpp index fa6b2ebd..2a41c3f4 100644 --- a/src/dlio_profiler/utils/posix_internal.cpp +++ b/src/dlio_profiler/utils/posix_internal.cpp @@ -12,8 +12,7 @@ int dlp_open(const char *pathname, int flags, ...) { va_start(args, flags); if (flags & O_CREAT) { mode = va_arg(args, mode_t); - } - else { + } else { mode = 0; } va_end(args); diff --git a/src/dlio_profiler/utils/utils.h b/src/dlio_profiler/utils/utils.h index 980fb7ef..5c0cda15 100644 --- a/src/dlio_profiler/utils/utils.h +++ b/src/dlio_profiler/utils/utils.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_UTILS_H #define DLIO_PROFILER_UTILS_H + #include #include #include @@ -34,6 +35,7 @@ inline std::string sh(std::string cmd) { } return result; } + inline void print_backtrace(void) { void *bt[1024]; int bt_size; @@ -58,62 +60,61 @@ inline void print_backtrace(void) { free(bt_syms); } -inline void signal_handler(int sig){ - switch(sig) { - case SIGHUP:{ - DLIO_PROFILER_LOGPRINT("hangup signal caught",0); +inline void signal_handler(int sig) { + switch (sig) { + case SIGHUP: { + DLIO_PROFILER_LOGPRINT("hangup signal caught", 0); break; } - case SIGTERM:{ - DLIO_PROFILER_LOGPRINT("terminate signal caught",0); + case SIGTERM: { + DLIO_PROFILER_LOGPRINT("terminate signal caught", 0); //MPI_Finalize(); exit(0); break; } - default:{ - DLIO_PROFILER_LOGPRINT("signal caught %d",sig); + default: { + DLIO_PROFILER_LOGPRINT("signal caught %d", sig); //print_backtrace(); void *array[20]; size_t size; void *trace[16]; - char **messages = (char **)NULL; + char **messages = (char **) NULL; int i, trace_size = 0; size = backtrace(array, 20); messages = backtrace_symbols(array, size); /* skip first stack frame (points here) */ std::stringstream myString; - myString << "[bt] Execution path with signal "< split(std::string str, char delimiter) { std::vector res; if (str.find(delimiter) == std::string::npos) { @@ -150,8 +152,9 @@ inline std::vector split(std::string str, char delimiter) { } return res; } -inline bool ignore_files(const char* filename) { - for(auto &file: ignore_filenames) { + +inline bool ignore_files(const char *filename) { + for (auto &file: ignore_filenames) { if (strstr(filename, file.c_str()) != NULL) { return true; } @@ -168,13 +171,13 @@ inline std::string get_filename(int fd) { return filename; } -inline std::pair is_traced_common(const char* filename, const char* func, - const std::vector& ignore_filename, - const std::vector& track_filename) { +inline std::pair is_traced_common(const char *filename, const char *func, + const std::vector &ignore_filename, + const std::vector &track_filename) { bool found = false; bool ignore = false; char resolved_path[PATH_MAX]; - char* data = realpath(filename, resolved_path); + char *data = realpath(filename, resolved_path); (void) data; if (ignore_files(resolved_path) || ignore_files(filename)) { DLIO_PROFILER_LOGINFO("Profiler ignoring file %s for func %s", resolved_path, func); @@ -194,7 +197,9 @@ inline std::pair is_traced_common(const char* filename, const } } } - if (!found and !ignore) DLIO_PROFILER_LOGINFO("Profiler Intercepted POSIX not tracing file %s for func %s", resolved_path, func); + if (!found and !ignore) + DLIO_PROFILER_LOGINFO("Profiler Intercepted POSIX not tracing file %s for func %s", resolved_path, func); return std::pair(found, filename); } + #endif // DLIO_PROFILER_UTILS_H diff --git a/src/dlio_profiler/writer/chrome_writer.cpp b/src/dlio_profiler/writer/chrome_writer.cpp index 89f91dd0..b9652f80 100644 --- a/src/dlio_profiler/writer/chrome_writer.cpp +++ b/src/dlio_profiler/writer/chrome_writer.cpp @@ -14,13 +14,14 @@ #define ERROR(cond, format, ...) \ DLIO_PROFILER_LOGERROR(format, __VA_ARGS__); \ if (this->throw_error) assert(cond); + void dlio_profiler::ChromeWriter::initialize(char *filename, bool throw_error) { this->throw_error = throw_error; this->filename = filename; if (fd == -1) { fd = dlp_open(filename, O_WRONLY | O_APPEND | O_CREAT, S_IRUSR | S_IWUSR); if (fd == -1) { - ERROR(fd == -1,"unable to create log file %s", filename); + ERROR(fd == -1, "unable to create log file %s", filename); } else { DLIO_PROFILER_LOGINFO("created log file %s with fd %d", filename, fd); } @@ -28,13 +29,15 @@ void dlio_profiler::ChromeWriter::initialize(char *filename, bool throw_error) { } void -dlio_profiler::ChromeWriter::log(std::string &event_name, std::string &category, TimeResolution &start_time, TimeResolution &duration, +dlio_profiler::ChromeWriter::log(std::string &event_name, std::string &category, TimeResolution &start_time, + TimeResolution &duration, std::unordered_map &metadata, int process_id) { if (fd != -1) { std::string json = convert_json(event_name, category, start_time, duration, metadata, process_id); auto written_elements = dlp_write(fd, json.c_str(), json.size()); if (written_elements != json.size()) { - ERROR(written_elements != json.size(), "unable to log write %s fd %d for a+ written only %d of %d with error %s", filename.c_str(), fd, written_elements, json.size(), strerror(errno)); + ERROR(written_elements != json.size(), "unable to log write %s fd %d for a+ written only %d of %d with error %s", + filename.c_str(), fd, written_elements, json.size(), strerror(errno)); } } is_first_write = false; @@ -42,19 +45,20 @@ dlio_profiler::ChromeWriter::log(std::string &event_name, std::string &category, void dlio_profiler::ChromeWriter::finalize() { if (fd != -1) { - DLIO_PROFILER_LOGINFO("Profiler finalizing writer %s\n", filename .c_str()); + DLIO_PROFILER_LOGINFO("Profiler finalizing writer %s\n", filename.c_str()); int status = dlp_close(fd); if (status != 0) { ERROR(status != 0, "unable to close log file %d for a+", filename.c_str()); } fd = dlp_open(this->filename.c_str(), O_WRONLY); if (fd == -1) { - ERROR(fd == -1,"unable to open log file %s with O_WRONLY", this->filename.c_str()); + ERROR(fd == -1, "unable to open log file %s with O_WRONLY", this->filename.c_str()); } std::string data = "[\n"; auto written_elements = dlp_write(fd, data.c_str(), data.size()); if (written_elements != data.size()) { - ERROR(written_elements != data.size(), "unable to finalize log write %s for r+ written only %d of %d", filename.c_str(), data.size(), written_elements); + ERROR(written_elements != data.size(), "unable to finalize log write %s for r+ written only %d of %d", + filename.c_str(), data.size(), written_elements); } status = dlp_close(fd); if (status != 0) { @@ -83,14 +87,14 @@ dlio_profiler::ChromeWriter::convert_json(std::string &event_name, std::string & auto start_sec = std::chrono::duration>(start_time); auto duration_sec = std::chrono::duration>(duration); if (is_first_write) all_stream << " "; - all_stream << R"({"name":")" << event_name << "\"," - << R"("cat":")" << category << "\"," - << "\"pid\":" << pid << "," - << "\"tid\":" << tid << "," - << "\"ts\":" << std::chrono::duration_cast(start_sec).count() << "," - << "\"dur\":" << std::chrono::duration_cast(duration_sec).count() << "," - << R"("ph":"X",)" - << R"("args":{)"; + all_stream << R"({"name":")" << event_name << "\"," + << R"("cat":")" << category << "\"," + << "\"pid\":" << pid << "," + << "\"tid\":" << tid << "," + << "\"ts\":" << std::chrono::duration_cast(start_sec).count() << "," + << "\"dur\":" << std::chrono::duration_cast(duration_sec).count() << "," + << R"("ph":"X",)" + << R"("args":{)"; if (include_metadata) { all_stream << "\"hostname\":\"" << hostname() << "\","; all_stream << "\"core_affinity\": ["; diff --git a/src/dlio_profiler/writer/chrome_writer.h b/src/dlio_profiler/writer/chrome_writer.h index d67b3aa2..fa3f50c8 100644 --- a/src/dlio_profiler/writer/chrome_writer.h +++ b/src/dlio_profiler/writer/chrome_writer.h @@ -4,6 +4,7 @@ #ifndef DLIO_PROFILER_CHROME_WRITER_H #define DLIO_PROFILER_CHROME_WRITER_H + #include #include #include @@ -14,16 +15,20 @@ #include namespace dlio_profiler { - class ChromeWriter: public BaseWriter { + class ChromeWriter : public BaseWriter { private: bool enable_core_affinity, include_metadata; hwloc_topology_t topology; int fd; - std::string convert_json(std::string &event_name, std::string &category, TimeResolution start_time, TimeResolution duration, - std::unordered_map &metadata, int process_id); + + std::string + convert_json(std::string &event_name, std::string &category, TimeResolution start_time, TimeResolution duration, + std::unordered_map &metadata, int process_id); + bool is_first_write; int process_id; std::unordered_map mtx_map; + std::vector core_affinity() { auto cores = std::vector(); if (enable_core_affinity) { @@ -36,14 +41,17 @@ namespace dlio_profiler { } return cores; } + std::string hostname() { - const int SIZE=256; + const int SIZE = 256; char hostname[SIZE]; gethostname(hostname, SIZE); return std::string(hostname); } + public: - ChromeWriter(int fd=-1):BaseWriter(), is_first_write(true), mtx_map(), enable_core_affinity(false), include_metadata(false){ + ChromeWriter(int fd = -1) + : BaseWriter(), is_first_write(true), mtx_map(), enable_core_affinity(false), include_metadata(false) { char *dlio_profiler_meta = getenv(DLIO_PROFILER_INC_METADATA); if (dlio_profiler_meta != nullptr && strcmp(dlio_profiler_meta, "1") == 0) { include_metadata = true; @@ -59,10 +67,11 @@ namespace dlio_profiler { hwloc_topology_load(topology); // actual detection } } + void initialize(char *filename, bool throw_error) override; void log(std::string &event_name, std::string &category, TimeResolution &start_time, TimeResolution &duration, - std::unordered_map &metadata, int process_id) override; + std::unordered_map &metadata, int process_id) override; void finalize() override; }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b3d5f85e..4ad25c72 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,7 +20,7 @@ add_dependencies(test_c ${PROJECT_NAME}_preload) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data) -find_program (BASH_PROGRAM bash) +find_program(BASH_PROGRAM bash) set(test_name test_cpp_basic_only) dlp_add_test(${test_name} ${CMAKE_BINARY_DIR}/bin/test_cpp ./data) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_LEVEL=INFO) @@ -166,6 +166,6 @@ set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_INC_MET dlp_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 80) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) -if(ENABLE_DLIO_BENCHMARK_TESTS) -add_subdirectory(dlio_benchmark) -endif() \ No newline at end of file +if (ENABLE_DLIO_BENCHMARK_TESTS) + add_subdirectory(dlio_benchmark) +endif () \ No newline at end of file diff --git a/test/c/test.c b/test/c/test.c index 844cc848..0158aacd 100644 --- a/test/c/test.c +++ b/test/c/test.c @@ -6,6 +6,7 @@ #include #include #include + void foo() { DLIO_PROFILER_C_FUNCTION_START(); sleep(1); @@ -18,21 +19,21 @@ void foo() { DLIO_PROFILER_C_FUNCTION_END(); } -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { int init = 0; if (argc > 2) { if (strcmp(argv[2], "1") == 0) { - DLIO_PROFILER_C_INIT(NULL,NULL,NULL); + DLIO_PROFILER_C_INIT(NULL, NULL, NULL); init = 1; } } char filename[1024]; sprintf(filename, "%s/demofile_c.txt", argv[1]); foo(); - FILE* fh = fopen(filename, "w+"); + FILE *fh = fopen(filename, "w+"); fwrite("hello", sizeof("hello"), 1, fh); fclose(fh); - if(init) { + if (init) { DLIO_PROFILER_C_FINI(); } return 0; diff --git a/test/cpp/test.cpp b/test/cpp/test.cpp index 93a4aa98..ef5dd457 100644 --- a/test/cpp/test.cpp +++ b/test/cpp/test.cpp @@ -17,11 +17,11 @@ void foo() { } } -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { int init = 0; if (argc > 2) { if (strcmp(argv[2], "1") == 0) { - DLIO_PROFILER_CPP_INIT(nullptr,nullptr,nullptr); + DLIO_PROFILER_CPP_INIT(nullptr, nullptr, nullptr); init = 1; } } @@ -30,8 +30,8 @@ int main(int argc, char* argv[]) { char filename_link[1024]; sprintf(filename_link, "%s/demofile_link.txt", argv[1]); foo(); - truncate(filename,0); - FILE* fh = fopen(filename, "w+"); + truncate(filename, 0); + FILE *fh = fopen(filename, "w+"); if (fh != NULL) { fwrite("hello", sizeof("hello"), 1, fh); fclose(fh); @@ -65,8 +65,8 @@ int main(int argc, char* argv[]) { symlinkat(filename, dd, filename_link); fstat(dd, &stat_buf); fstat64(dd, &stat_buf64); - faccessat(dd,"demofile.txt",O_RDONLY,0); - linkat(dd,"demofile.txt", dd, "demofile_link2.txt", 0); + faccessat(dd, "demofile.txt", O_RDONLY, 0); + linkat(dd, "demofile.txt", dd, "demofile_link2.txt", 0); chdir(dir); int fd = openat(dd, "demofile.txt", O_RDONLY); if (fd != -1) close(fd); @@ -75,7 +75,7 @@ int main(int argc, char* argv[]) { close(dd); char filename2[1024]; sprintf(filename, "%s/demofile2.txt", argv[1]); - fd = creat64(filename, O_RDWR) ; + fd = creat64(filename, O_RDWR); if (fd != -1) close(fd); fd = open(filename, O_RDWR); int set_offset = lseek(fd, 1, SEEK_SET); @@ -87,13 +87,13 @@ int main(int argc, char* argv[]) { fsync(fd); fdatasync(fd); readlinkat(fd, filename, buf, 1); - ftruncate(fd,0); + ftruncate(fd, 0); close(fd); remove(filename2); remove(filename); remove(filename_link); remove("demofile_link2.txt"); - if (init == 1) { + if (init == 1) { DLIO_PROFILER_CPP_FINI(); } return 0; diff --git a/test/dlio_benchmark/CMakeLists.txt b/test/dlio_benchmark/CMakeLists.txt index 741178f7..047fe7dd 100644 --- a/test/dlio_benchmark/CMakeLists.txt +++ b/test/dlio_benchmark/CMakeLists.txt @@ -12,7 +12,7 @@ set(lines 1728 192 416 544 640) list(LENGTH formats formats_length) math(EXPR formats_length "${formats_length} - 1") message(STATUS "Number of formats: ${formats_length}") -foreach(index RANGE ${formats_length}) +foreach (index RANGE ${formats_length}) list(GET formats index format) list(GET lines index line) set(test_name test_dlio_${format}) @@ -23,4 +23,4 @@ foreach(index RANGE ${formats_length}) dlp_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_SOURCE_DIR}/test/check_file.sh ${CMAKE_CURRENT_BINARY_DIR}/output_${format}/.trace*.pfw ${line}) set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name}) -endforeach() \ No newline at end of file +endforeach () \ No newline at end of file diff --git a/test/py/test.py b/test/py/test.py index 3f42a97b..1dba9119 100644 --- a/test/py/test.py +++ b/test/py/test.py @@ -1,4 +1,3 @@ - from time import sleep import os import threading @@ -9,14 +8,16 @@ from dlio_profiler.logger import dlio_logger, fn_interceptor cwd = os.getcwd() -log_file=os.getenv("LOG_FILE", f"{cwd}/test_py-app.pwf") +log_file = os.getenv("LOG_FILE", f"{cwd}/test_py-app.pwf") log_inst = dlio_logger.initialize_log(logfile=None, data_dir=None, process_id=-1) dlio_log = fn_interceptor("COMPUTE") + @dlio_log.log def log_events(index): sleep(1) + def custom_events(): args = { "epoch": 1, @@ -30,7 +31,7 @@ def custom_events(): def posix_calls(val): index, is_spawn = val - path=f"{cwd}/data/demofile{index}.txt" + path = f"{cwd}/data/demofile{index}.txt" f = open(path, "w+") f.write("Now the file has more content!") f.close() @@ -40,8 +41,9 @@ def posix_calls(val): else: print(f"Not calling spawn on {index} with pid {os.getpid()}") + def npz_calls(index): - #print(f"{cwd}/data/demofile2.npz") + # print(f"{cwd}/data/demofile2.npz") path = f"{cwd}/data/demofile{index}.npz" if os.path.exists(path): os.remove(path) @@ -49,6 +51,7 @@ def npz_calls(index): record_labels = [0] * 1024 np.savez(path, x=records, y=record_labels) + def jpeg_calls(index): records = np.random.randint(255, size=(1024, 1024), dtype=np.uint8) img = im.fromarray(records) @@ -57,13 +60,16 @@ def jpeg_calls(index): with open(out_path_spec, "rb") as f: image = im.open(f) out_records = np.asarray(image) - #image = im.open(out_path_spec) + # image = im.open(out_path_spec) + + def init(): """This function is called when new processes start.""" print(f'Initializing process {os.getpid()}') -def main(): - t1 = threading.Thread(target=posix_calls, args=(10,False)) + +def main(): + t1 = threading.Thread(target=posix_calls, args=(10, False)) custom_events() t2 = threading.Thread(target=npz_calls, args=(1,)) t3 = threading.Thread(target=jpeg_calls, args=(2,)) @@ -80,7 +86,7 @@ def main(): t4.join() index = 4 with get_context('fork').Pool(1, initializer=init) as pool: - pool.map(posix_calls, ((index,False),)) + pool.map(posix_calls, ((index, False),)) index = index + 1 with get_context('spawn').Pool(1, initializer=init) as pool: @@ -90,6 +96,5 @@ def main(): log_inst.finalize() - if __name__ == "__main__": - main() \ No newline at end of file + main()