From 434ea30273930c804c636932c00aa84a9080f91e Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 17 Jun 2024 17:38:24 -0700 Subject: [PATCH 1/8] add setup tools for python > 3.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 81e757ca..7aa8bf53 100644 --- a/setup.py +++ b/setup.py @@ -178,7 +178,7 @@ def build_extension(self, ext: CMakeExtension) -> None: "Programming Language :: Python :: 3 :: Only", ], install_requires=["pybind11"], - requires=["pybind11"], + requires=["pybind11","setuptools"], keywords="profiler, deep learning, I/O, benchmark, NPZ, pytorch benchmark, tensorflow benchmark", project_urls={ # Optional "Bug Reports": "https://github.com/hariharan-devarajan/dlio-profiler/issues", From 2bb92641d5b3d7ef6332e62fb60bbb9a45ad3720 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 17 Jun 2024 17:45:14 -0700 Subject: [PATCH 2/8] check publish code. --- .github/workflows/python-publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index ea0a0f83..3e1e6b9b 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,7 +11,9 @@ name: Upload Python Package on: release: types: [published] - + pull_request: + branches: [ main, dev ] + push: permissions: contents: read From 605ab2c2947f0f9063f89e3a0ef78a7088edad62 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 17 Jun 2024 17:46:42 -0700 Subject: [PATCH 3/8] Added setup tools --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 3e1e6b9b..31f85804 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -38,7 +38,7 @@ jobs: sudo apt-get update sudo apt-get install -y gcc g++ libtool-bin openmpi-bin libopenmpi-dev python -m pip install --upgrade pip - pip install build wheel auditwheel pybind11 patchelf + pip install build wheel auditwheel pybind11 patchelf setuptools - name: Build wheel package run: | export DLIO_PROFILER_WHEEL=1 From 03288b81a50a58e6ccb8d1e60771f3df018c1aab Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 17 Jun 2024 17:59:31 -0700 Subject: [PATCH 4/8] add reference to code. --- .github/workflows/python-publish.yml | 3 ++- src/dlio_profiler/core/dlio_profiler_main.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 31f85804..21f88d37 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -52,7 +52,8 @@ jobs: echo $LD_LIBRARY_PATH auditwheel show dist/*.whl platform=$(auditwheel show dist/*.whl | grep manylinux_ | awk -F'"' '{print $2}') - echo $platform + auditwheel show dist/*.whl | grep manylinux_ + echo "Platform selected is $platform" auditwheel repair --plat $platform dist/*.whl -w dist/ rm dist/*linux_x86_64.whl rm dist/*.tar.gz diff --git a/src/dlio_profiler/core/dlio_profiler_main.cpp b/src/dlio_profiler/core/dlio_profiler_main.cpp index 2a9e3ab8..80dc06ba 100644 --- a/src/dlio_profiler/core/dlio_profiler_main.cpp +++ b/src/dlio_profiler/core/dlio_profiler_main.cpp @@ -161,7 +161,7 @@ void dlio_profiler::DLIOProfilerCore::initialize(bool _bind, } DLIO_PROFILER_LOGDEBUG("Exec command line %s", exec_file_name); auto items = split(exec_file_name, SEPARATOR); - for (const auto item : items) { + for (const auto &item : items) { if (strstr(item.c_str(), "python") == nullptr) { exec_name = basename(item.c_str()); break; From bf8ab1bd53d94c60b91a9aa37461411193f6aacc Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 17 Jun 2024 18:10:09 -0700 Subject: [PATCH 5/8] fixed publish script --- .github/workflows/python-publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 21f88d37..9d6b84a6 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,9 +11,6 @@ name: Upload Python Package on: release: types: [published] - pull_request: - branches: [ main, dev ] - push: permissions: contents: read From e4c79ece7b6b05f2f50ff9682505ab383556a947 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Fri, 21 Jun 2024 12:30:44 -0700 Subject: [PATCH 6/8] Fix extension ignore list extensions should have . character --- src/dlio_profiler/core/dlio_profiler_main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dlio_profiler/core/dlio_profiler_main.cpp b/src/dlio_profiler/core/dlio_profiler_main.cpp index 80dc06ba..03532142 100644 --- a/src/dlio_profiler/core/dlio_profiler_main.cpp +++ b/src/dlio_profiler/core/dlio_profiler_main.cpp @@ -185,7 +185,7 @@ void dlio_profiler::DLIOProfilerCore::initialize(bool _bind, if (bind) { if (conf->io) { auto trie = dlio_profiler::Singleton::get_instance(); - const char *ignore_extensions[2] = {"pfw", "py"}; + const char *ignore_extensions[2] = {".pfw", ".py"}; const char *ignore_prefix[8] = {"/pipe", "/socket", "/proc", "/sys", "/collab", "anon_inode", "socket", "/var/tmp"}; @@ -246,4 +246,4 @@ TimeResolution dlio_profiler::DLIOProfilerCore::get_time() { ""); } return -1; -} \ No newline at end of file +} From 46b21be92d38f88c21529e1f7e8634f37c17e7ee Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Fri, 21 Jun 2024 15:42:06 -0700 Subject: [PATCH 7/8] Update CMakeLists.txt --- dependency/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency/CMakeLists.txt b/dependency/CMakeLists.txt index 51020c68..68881dd2 100644 --- a/dependency/CMakeLists.txt +++ b/dependency/CMakeLists.txt @@ -3,7 +3,7 @@ install_external_project(cpp-logger https://github.com/hariharan-devarajan/cpp-l install_external_project(gotcha https://github.com/LLNL/GOTCHA.git 1.0.7 ${CMAKE_INSTALL_PREFIX} "") install_external_project(brahma https://github.com/hariharan-devarajan/brahma.git v0.0.5 ${CMAKE_INSTALL_PREFIX} "") install_external_project(yaml-cpp https://github.com/jbeder/yaml-cpp.git yaml-cpp-0.6.3 ${CMAKE_INSTALL_PREFIX} "-DYAML_CPP_BUILD_TESTS=OFF;-DYAML_BUILD_SHARED_LIBS=ON") -install_external_project(pybind11 https://github.com/pybind/pybind11.git v2.12.0 ${CMAKE_INSTALL_PREFIX} "") +install_external_project(pybind11 https://github.com/pybind/pybind11.git v2.12.0 ${CMAKE_INSTALL_PREFIX} "-DPYBIND11_TEST=OFF") add_dependencies(yaml-cpp brahma) add_dependencies(brahma cpp-logger) add_dependencies(brahma gotcha) From f76e0f26fce6771b3864f774cd11af6aab843ba0 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Fri, 21 Jun 2024 16:41:14 -0700 Subject: [PATCH 8/8] Update dlio_profiler_main.cpp --- src/dlio_profiler/core/dlio_profiler_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dlio_profiler/core/dlio_profiler_main.cpp b/src/dlio_profiler/core/dlio_profiler_main.cpp index 03532142..44dabf33 100644 --- a/src/dlio_profiler/core/dlio_profiler_main.cpp +++ b/src/dlio_profiler/core/dlio_profiler_main.cpp @@ -185,7 +185,7 @@ void dlio_profiler::DLIOProfilerCore::initialize(bool _bind, if (bind) { if (conf->io) { auto trie = dlio_profiler::Singleton::get_instance(); - const char *ignore_extensions[2] = {".pfw", ".py"}; + const char *ignore_extensions[3] = {".pfw", ".py",".pfw.gz"}; const char *ignore_prefix[8] = {"/pipe", "/socket", "/proc", "/sys", "/collab", "anon_inode", "socket", "/var/tmp"};