Merge pull request #6 from hariharan-devarajan/feature/apis

Adding new APIs

.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: Google

.github/workflows/ci.yml

@@ -19,6 +19,7 @@ jobs:
       CXX: g++
       DIR: ${{ matrix.dir }}
       PERM: ${{ matrix.perm }}
+      OS: ${{ matrix.os }}
 
     steps:
       - name: Push checkout
@@ -34,8 +35,26 @@ jobs:
           sudo apt-get update
           sudo apt-get install gcc g++ libtool-bin openmpi-bin libopenmpi-dev
           sudo apt-get install python3.10 python3-pip
+          sudo apt-get install gcovr clang-format
+      - name: Install dlio benchmark
+        if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user'
+        run: |
+          LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main
+          if [[ $PERM == 'user' ]]; then
+            export DLIO_LOGGER_USER=1
+            echo pip install ${LOC}
+            pip install ${LOC}
+          else
+             echo pip install ${LOC}
+             sudo pip install ${LOC}
+          fi
       - name: Build from ${{ matrix.dir }} on ${{ matrix.perm }}
         run: |
+          export CMAKE_BUILD_TYPE=PROFILE
+          export DLIO_PROFILER_ENABLE_TESTS=ON
+          if [ $OS == 'ubuntu-22.04' ] && [ $DIR == 'github' ] && [ $PERM == 'user' ]; then
+            export ENABLE_DLIO_BENCHMARK_TESTS=ON
+          fi
           LOC=.
           if [[ $DIR == 'github' ]]; then
             LOC=git+https://github.com/hariharan-devarajan/dlio-profiler.git@dev
@@ -48,23 +67,21 @@ jobs:
              echo pip install -v ${LOC}
              sudo pip install -v ${LOC}
           fi
-      - name: Test DLIO_Benchmark
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root'
+      - name: Test ctest
+        if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user'
         run: |
-          LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main
-          if [[ $PERM == 'user' ]]; then
-            export DLIO_LOGGER_USER=1
-            echo pip install ${LOC}
-            pip install ${LOC}
-          else
-             echo pip install ${LOC}
-             sudo pip install ${LOC}
+          pip install -r test/py/requirements.txt
+          cd build/temp*/dlio_profiler_py
+          ctest -VV
+      - name: Create Coverage
+        if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user'
+        run: |
+          mkdir coverage
+          FILE=$PWD/coverage/coverage.json
+          cd build
+          COVERALLS_REPO_TOKEN=${{ secrets.GITHUB_TOKEN }} gcovr -r ../ . --coveralls $FILE -e ../test/ -e ../src/example
+          if [ -e '$FILE' ]; then
+          sed -i'' -e 's/"service_name": "github-actions-ci"/"service_name": "github"/' '$FILE'
           fi
-          export DLIO_PROFILER_ENABLE=1
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          echo $num_posix_lines
-          if [[ "$num_posix_lines" == '0' ]]; then
-            exit 1
-          fi
+          cat $FILE
+          curl -v -F json_file=@$FILE https://coveralls.io/api/v1/jobs

.readthedocs.yaml

@@ -0,0 +1,32 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+    # You can also specify other tool versions:
+    # nodejs: "19"
+    # rust: "1.64"
+    # golang: "1.19"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#    - pdf
+#    - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+# python:
+#    install:
+#    - requirements: docs/requirements.txt

CMakeLists.txt

@@ -28,6 +28,8 @@ set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CMAKE_BINARY_DIR})
 # Build options
 #------------------------------------------------------------------------------
 option(BUILD_PYTHON_BINDINGS "Build python bindings." ON)
+option(DLIO_PROFILER_ENABLE_TESTS "Enable internal tests" Off)
+option(ENABLE_DLIO_BENCHMARK_TESTS "Enable dlio_benchmark tests" Off)
 #------------------------------------------------------------------------------
 # Setup install and output Directories
 #------------------------------------------------------------------------------
@@ -69,6 +71,21 @@ if (NOT DLIO_PROFILER_EXTERNALLY_CONFIGURED)
 endif ()
 
 
+if(DLIO_PROFILER_ENABLE_TESTS)
+    if(CMAKE_BUILD_TYPE STREQUAL "PROFILE")
+        message("-- " "[DLIO Profiler] building with CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}")
+        set(CMAKE_C_FLAGS_PROFILE --coverage)
+        set(CMAKE_CXX_FLAGS_PROFILE --coverage)
+    endif()
+    enable_testing()
+    add_subdirectory(test)
+    if(CMAKE_BUILD_TYPE STREQUAL "PROFILE")
+        get_property(COVERAGE_FILE_LIST GLOBAL PROPERTY COVERAGE_FILES)
+        add_custom_command(OUTPUT default.profdata COMMAND llvm-profdata merge -sparse ${COVERAGE_FILE_LIST} -o default.profdata VERBATIM)
+        add_custom_target(merged_coverage DEPENDS default.profdata)
+    endif()
+endif()
+
 if (BUILD_PYTHON_BINDINGS)
     find_package(pybind11 REQUIRED)
     #add_subdirectory(dependency/py11bind)
@@ -134,23 +151,27 @@ endif ()
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
-set(DLIO_PROFILER_PRELOAD_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_profiler.cpp)
-set(DLIO_PROFILER_PRELOAD_PRIVATE_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_logger.h)
+set(DLIO_PROFILER_PRELOAD_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_profiler_preload.cpp)
+set(DLIO_PROFILER_PRELOAD_PRIVATE_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_logger_preload.h)
 
-set(DLIO_PROFILER_PY_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/posix.cpp
+set(DLIO_PROFILER_CORE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/posix.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/stdio.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/writer/chrome_writer.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/utils/posix_internal.cpp)
-set(DLIO_PROFILER_PY_PUBLIC_INCLUDE
-        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/macro.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/posix.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/stdio.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/dlio_profiler.h)
-set(DLIO_PROFILER_PY_PRIVATE_INCLUDE
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/core/dlio_profiler_main.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/core/constants.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/utils/posix_internal.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_profiler.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/core/dlio_profiler_main.cpp)
+set(DLIO_PROFILER_CORE_PUBLIC_INCLUDE
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/dlio_profiler.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/core/dlio_profiler_main.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/core/constants.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/core/macro.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/core/singleton.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/dlio_profiler/core/typedef.h)
+set(DLIO_PROFILER_CORE_PRIVATE_INCLUDE
         ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/utils/posix_internal.h
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/utils/utils.h)
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/utils/utils.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/posix.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/brahma/stdio.h)
 add_library(${PROJECT_NAME} SHARED)
 target_link_libraries(${PROJECT_NAME} ${DEPENDENCY_LIB})
 add_library(${PROJECT_NAME}_preload SHARED)
@@ -160,13 +181,13 @@ target_link_libraries(${PROJECT_NAME}_preload ${DEPENDENCY_LIB} ${PROJECT_NAME})
 #add_dependencies(${PROJECT_NAME} gotcha)
 message(STATUS ${CMAKE_PREFIX_PATH})
 if (BUILD_PYTHON_BINDINGS)
-    pybind11_add_module(${PROJECT_NAME}_py ${DLIO_PROFILER_PY_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_profiler_py.cpp)
+    pybind11_add_module(${PROJECT_NAME}_py ${DLIO_PROFILER_CORE_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/dlio_profiler/dlio_profiler_py.cpp)
     target_compile_definitions(${PROJECT_NAME}_py PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})
     target_link_libraries(${PROJECT_NAME}_py PRIVATE ${DEPENDENCY_LIB})
 endif()
 target_sources(${PROJECT_NAME}
         PRIVATE
-        ${DLIO_PROFILER_PY_SRC}
+        ${DLIO_PROFILER_CORE_SRC}
         )
 target_sources(${PROJECT_NAME}_preload
         PRIVATE
@@ -233,7 +254,7 @@ include(GNUInstallDirs)
 # the variant with PUBLIC_HEADER property unfortunately does not preserve the folder structure
 #set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${public_headers}")
 # so instead we iterate through public headers and install them "manually"
-foreach (header ${DLIO_PROFILER_PY_PUBLIC_INCLUDE})
+foreach (header ${DLIO_PROFILER_CORE_PUBLIC_INCLUDE})
     file(RELATIVE_PATH header_file_path "${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}" "${header}")
     get_filename_component(header_directory_path "${header_file_path}" DIRECTORY)
     install(
@@ -318,8 +339,3 @@ install(EXPORT
         FILE
         ${DLIO_PROFILER_EXPORTED_TARGETS}.cmake
         )
-option(DLIO_PROFILER_BUILD_TEST "Build the unit tests" ON)
-if (DLIO_PROFILER_BUILD_TEST)
-    enable_testing()
-    add_subdirectory(test)
-endif ()

README.md

@@ -1,3 +1,6 @@
+[![DLIO Profiler Build and Test](https://github.com/hariharan-devarajan/dlio-profiler/actions/workflows/ci.yml/badge.svg)](https://github.com/hariharan-devarajan/dlio-profiler/actions/workflows/ci.yml)
+[![Coverage Status](https://coveralls.io/repos/github/hariharan-devarajan/dlio-profiler/badge.svg?branch=feature/apis)](https://coveralls.io/github/hariharan-devarajan/dlio-profiler?branch=dev)
+
 # dlio-profiler
 A low-level profiler for capture I/O calls from deep learning applications.
 

dlio_profiler/logger.py

@@ -1,6 +1,7 @@
 from functools import wraps
 from typing import Dict
 import os
+
 DLIO_PROFILER_ENABLE_ENV = "DLIO_PROFILER_ENABLE"
 DLIO_PROFILER_INIT_ENV = "DLIO_PROFILER_INIT"
 
@@ -12,13 +13,17 @@
 
 from pathlib import Path
 import inspect
-import sys,signal
+import sys, signal
+
+
 def capture_signal(signal_number, frame):
     dlio_logger.get_instance().finalize()
     sys.exit(1)
 
+
 signal.signal(signal.SIGABRT, capture_signal)
 
+
 class dlio_logger:
     __instance = None
 
@@ -32,22 +37,26 @@ def __init__(self, logfile):
     def get_instance(cls, log_file=None):
         """ Static access method. """
         if dlio_logger.__instance is None:
-            if log_file:
-                dlio_logger(log_file)
-            else:
-                raise Exception("log_file needs to be passed for first initialization.")
+            dlio_logger(log_file)
         return dlio_logger.__instance
 
     @staticmethod
     def initialize_log(logfile, data_dir, process_id):
-        log_file = Path(logfile)
+        log_file = None
+        if logfile:
+            log_file = Path(logfile)
         instance = dlio_logger.get_instance(log_file)
         if DLIO_PROFILER_ENABLE:
-            os.makedirs(log_file.parent, exist_ok=True)
-            if os.path.isfile(log_file):
-                os.remove(log_file)
+            if log_file:
+                os.makedirs(log_file.parent, exist_ok=True)
+                if os.path.isfile(log_file):
+                    os.remove(log_file)
             instance.logger = profiler
-            instance.logger.initialize(f"{instance.logfile}", f"{data_dir}", process_id=process_id)
+            if log_file:
+                log_file = f"{instance.logfile}"
+            if data_dir:
+                data_dir = f"{data_dir}"
+            instance.logger.initialize(log_file=log_file, data_dirs=data_dir, process_id=process_id)
         return instance
 
     def get_time(self):

docs/.clang-format

@@ -0,0 +1,2 @@
+DisableFormat: true
+SortIncludes: Never

docs/.gitignore

@@ -0,0 +1 @@
+_build