Merge pull request #75 from hariharan-devarajan/dev

Release 0.0.5

CMakeLists.txt

@@ -27,6 +27,7 @@ option(BUILD_PYTHON_BINDINGS "Build python bindings." ON)
 option(DLIO_PROFILER_ENABLE_TESTS "Enable internal tests" Off)
 option(ENABLE_DLIO_BENCHMARK_TESTS "Enable dlio_benchmark tests" Off)
 option(ENABLE_PAPER_TESTS "Enable paper tests" Off)
+option(DISABLE_HWLOC "Disable HWLOC" On)
 #------------------------------------------------------------------------------
 # Setup install and output Directories
 #------------------------------------------------------------------------------
@@ -100,8 +101,16 @@ if (NOT DLIO_PROFILER_EXTERNALLY_CONFIGURED)
             )
 endif ()
 
-
-
+#------------------------------------------------------------------------------
+# Set CXX FLAGS
+#------------------------------------------------------------------------------
+if (DISABLE_HWLOC)
+    message(STATUS "[DLIO_PROFILER] disabling HWLOC")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDISABLE_HWLOC=0")
+else()
+    message(STATUS "[DLIO_PROFILER] enabling HWLOC")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDISABLE_HWLOC=1")
+endif()
 
 
 if (BUILD_PYTHON_BINDINGS)
@@ -127,7 +136,10 @@ export(EXPORT gotcha-targets
 #add_dependencies(brahma gotcha)
 link_directories(${CMAKE_BINARY_DIR}/lib)
 link_directories(${CMAKE_BINARY_DIR})
-set(DEPENDENCY_LIB stdc++fs -lhwloc)
+set(DEPENDENCY_LIB stdc++fs)
+if (NOT DISABLE_HWLOC)
+    set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -lhwloc)
+endif()
 
 if (${CPP_LOGGER_FOUND})
     include_directories(${CPP_LOGGER_INCLUDE_DIRS})

README.md

@@ -2,7 +2,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/hariharan-devarajan/dlio-profiler/badge.svg?branch=feature/apis)](https://coveralls.io/github/hariharan-devarajan/dlio-profiler?branch=dev)
 [![Documentation Status](https://readthedocs.org/projects/dlio-profiler/badge/?version=latest)](https://dlio-profiler.readthedocs.io/en/latest/?badge=latest)
 
-# DLIO Profiler v0.0.4
+# DLIO Profiler v0.0.5
 A multi-level profiler for capturing application functions and low-level system I/O calls from deep learning workloads.
 
 Requirements for profiler

dependency/cpp.requirements.txt

@@ -1,4 +1,4 @@
 cpp-logger,https://github.com/hariharan-devarajan/cpp-logger.git,v0.0.3,1,tag,
 gotcha,https://github.com/LLNL/GOTCHA.git,1.0.5,1,tag,
-brahma,https://github.com/hariharan-devarajan/brahma.git,v0.0.2,1,tag,
+brahma,https://github.com/hariharan-devarajan/brahma.git,v0.0.3,1,tag,
 yaml-cpp,https://github.com/jbeder/yaml-cpp.git,yaml-cpp-0.6.3,1,tag,"-DYAML_BUILD_SHARED_LIBS=On"

dlio_profiler/logger.py

@@ -249,3 +249,23 @@ def new_init(args, *kwargs):
                                                          duration=end - start)
 
         return new_init
+
+    def log_static(self, func):
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if DLIO_PROFILER_ENABLE:
+                start = dlio_logger.get_instance().get_time()
+            x = func(*args, **kwargs)
+            if DLIO_PROFILER_ENABLE:
+                end = dlio_logger.get_instance().get_time()
+                if len(self._arguments) > 0:
+                    dlio_logger.get_instance().log_event(name=func.__qualname__, cat=self._cat, start_time=start,
+                                                         duration=end - start,
+                                                         string_args=self._arguments)
+                else:
+                    dlio_logger.get_instance().log_event(name=func.__qualname__, cat=self._cat, start_time=start,
+                                                         duration=end - start)
+            return x
+
+        return wrapper

docs/api.rst

@@ -337,6 +337,8 @@ For logging ``__init__`` function within a class, applications can use ``log_ini
         def log_events(self, index):
             sleep(1)
 
+For logging ``@staticmethod`` function within a class, applications can use ``log_static`` function.
+
 
 Iteration/Loop Profiling
 ****************************************

docs/build.rst

@@ -98,6 +98,24 @@ Download the latest DLIO Profiler release from the Releases_ page or clone the d
 branch ('develop') from the DLIO Profiler repository
 `https://github.com/hariharan-devarajan/dlio-profiler <https://github.com/hariharan-devarajan/dlio-profiler>`_.
 
+------------------------------------------
+Build ENV Variables
+------------------------------------------
+For pip based installations we can enable HWLOC using
+.. code-block:: Bash
+
+    export DLIO_PROFILER_DISABLE_HWLOC=Off
+
+------------------------------------------
+Build CMAKE Variables
+------------------------------------------
+For Cmake based build, we can enable HWLOC using
+.. code-block:: Bash
+
+    cmake -DDISABLE_HWLOC=Off <source dir>
+
+
+
 Build DLIO Profiler Dependencies
 ********************************
 

docs/conf.py

@@ -26,7 +26,7 @@
 # The short X.Y version
 version = u'0.0'
 # The full version, including alpha/beta/rc tags
-release = u'0.0.4'
+release = u'0.0.5'
 
 
 # -- General configuration ---------------------------------------------------

setup.py

@@ -72,6 +72,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         # auxiliary "native" libs
         build_type = os.environ.get("CMAKE_BUILD_TYPE", "Release")
         cmake_args += [f"-DCMAKE_BUILD_TYPE={build_type}"]
+        enable_hwloc = os.environ.get("DLIO_PROFILER_DISABLE_HWLOC", "On")
+        cmake_args += [f"-DDISABLE_HWLOC={enable_hwloc}"]
         enable_tests = os.environ.get("DLIO_PROFILER_ENABLE_TESTS", "Off")
         cmake_args += [f"-DDLIO_PROFILER_ENABLE_TESTS={enable_tests}"]
         enable_dlio_tests = os.environ.get("ENABLE_DLIO_BENCHMARK_TESTS", "Off")
@@ -127,7 +129,7 @@ def build_extension(self, ext: CMakeExtension) -> None:
 # logic and declaration, and simpler if you include description/version in a file.
 setup(
     name="dlio_profiler_py",
-    version="0.0.4",
+    version="0.0.5",
     description="I/O profiler for deep learning python apps. Specifically for dlio_benchmark.",
     long_description=long_description,
     long_description_content_type="text/markdown",

src/dlio_profiler/writer/chrome_writer.cpp

@@ -93,7 +93,9 @@ void dlio_profiler::ChromeWriter::finalize() {
     }
   }
   if (enable_core_affinity) {
+#if DISABLE_HWLOC == 1
     hwloc_topology_destroy(topology);
+#endif
   }
   DLIO_PROFILER_LOGDEBUG("Finished writer finalization", "");
 }

src/dlio_profiler/writer/chrome_writer.h

@@ -10,7 +10,10 @@
 #include <dlio_profiler/utils/configuration_manager.h>
 #include <dlio_profiler/utils/posix_internal.h>
 #include <dlio_profiler/utils/utils.h>
+#if DISABLE_HWLOC == 1
 #include <hwloc.h>
+#endif
+#include <assert.h>
 #include <unistd.h>
 
 #include <any>
@@ -31,8 +34,12 @@ namespace dlio_profiler {
         bool throw_error;
         std::string filename;
     private:
-        bool enable_core_affinity, include_metadata, enable_compression;
+        bool include_metadata, enable_compression;
+
+        bool enable_core_affinity;
+#if DISABLE_HWLOC == 1
         hwloc_topology_t topology;
+#endif
         FILE* fh;
         std::atomic_int index;
         char hostname[256];
@@ -57,6 +64,7 @@ namespace dlio_profiler {
         std::vector<unsigned> core_affinity() {
           DLIO_PROFILER_LOGDEBUG("ChromeWriter.core_affinity","");
           auto cores = std::vector<unsigned>();
+#if DISABLE_HWLOC == 1
           if (enable_core_affinity) {
             hwloc_cpuset_t set = hwloc_bitmap_alloc();
             hwloc_get_cpubind(topology, set, HWLOC_CPUBIND_PROCESS);
@@ -65,6 +73,7 @@ namespace dlio_profiler {
             }
             hwloc_bitmap_free(set);
           }
+#endif
           return cores;
         }
 
@@ -83,8 +92,10 @@ namespace dlio_profiler {
           enable_core_affinity = conf->core_affinity;
           enable_compression = conf->compression;
           if (enable_core_affinity) {
+#if DISABLE_HWLOC == 1
             hwloc_topology_init(&topology);  // initialization
             hwloc_topology_load(topology);   // actual detection
+#endif
           }
         }
         ~ChromeWriter(){DLIO_PROFILER_LOGDEBUG("Destructing ChromeWriter","");}

test/dlp_analyzer/CMakeLists.txt

@@ -1,7 +1,7 @@
 set(test_name test_dlp_analyzer)
-dlp_add_test(${test_name}_only_pfw ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
-        ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw -d --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)
-dlp_add_test(${test_name}_only_pfw_gz ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
-        ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw.gz -v --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)
-dlp_add_test(${test_name}_both ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
-        ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw* -d --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)
+# dlp_add_test(${test_name}_only_pfw ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
+#         ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw -d --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)
+# dlp_add_test(${test_name}_only_pfw_gz ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
+#         ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw.gz -v --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)
+# dlp_add_test(${test_name}_both ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/dlp_analyzer/main.py
+#         ${PROJECT_SOURCE_DIR}/dlp_analyzer/test.pfw* -d --log-file ${CMAKE_BINARY_DIR}/test/dlp_analyzer/dlp_analyzer.log)