added dlio_benchmark to ctest

hariharan-devarajan · Oct 9, 2023 · b61eef5 · b61eef5
1 parent d790dbb
commit b61eef5
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 73 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ jobs:
       CXX: g++
       DIR: ${{ matrix.dir }}
       PERM: ${{ matrix.perm }}
+      OS: ${{ matrix.os }}
 
     steps:
       - name: Push checkout
@@ -36,7 +37,7 @@ jobs:
           sudo apt-get install python3.10 python3-pip
           sudo apt-get install gcovr
       - name: Install dlio benchmark
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
+        if: matrix.os == 'ubuntu-22.04' && matrix.dir == 'clone' && matrix.perm == 'user'
         run: |
           LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main
           if [[ $PERM == 'user' ]]; then
@@ -51,6 +52,9 @@ jobs:
         run: |
           export CMAKE_BUILD_TYPE=PROFILE
           export DLIO_PROFILER_ENABLE_TESTS=ON
+          if [ $OS == 'ubuntu-22.04' ] && [ $DIR == 'github' ] && [ $PERM == 'user' ]; then
+            export ENABLE_DLIO_BENCHMARK_TESTS=ON
+          fi
           LOC=.
           if [[ $DIR == 'github' ]]; then
             LOC=git+https://github.com/hariharan-devarajan/dlio-profiler.git@dev
@@ -80,75 +84,4 @@ jobs:
           sed -i'' -e 's/"service_name": "github-actions-ci"/"service_name": "github"/' '$FILE'
           fi
           cat $FILE
-          curl -v -F json_file=@$FILE https://coveralls.io/api/v1/jobs
-      - name: Test dlio benchmark npz
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
-        run: |
-          export DLIO_PROFILER_ENABLE=1
-          export DLIO_PROFILER_LOG_LEVEL=ERROR
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 \
-                        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow ++workload.dataset.format=npz
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
-          echo $num_posix_lines of $total_lines
-          rm -rf $PWD/output
-          if [[ "$num_posix_lines" -ne "1728" ]]; then
-            exit 1
-          fi
-      - name: Test dlio benchmark tfrecord
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
-        run: |
-          export DLIO_PROFILER_ENABLE=1
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 \
-                        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow ++workload.dataset.format=tfrecord
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
-          echo $num_posix_lines of $total_lines
-          rm -rf $PWD/output
-          if [[ "$num_posix_lines" -ne "192" ]]; then
-            exit 1
-          fi
-      - name: Test dlio benchmark jpeg
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
-        run: |
-          export DLIO_PROFILER_ENABLE=1
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 \
-                        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow ++workload.dataset.format=jpeg
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
-          echo $num_posix_lines of $total_lines
-          rm -rf $PWD/output
-          if [[ "$num_posix_lines" -ne "416" ]]; then
-            exit 1
-          fi
-      - name: Test dlio benchmark png
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
-        run: |
-          export DLIO_PROFILER_ENABLE=1
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 \
-                        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow ++workload.dataset.format=png
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
-          echo $num_posix_lines of $total_lines
-          rm -rf $PWD/output
-          if [[ "$num_posix_lines" -ne "544" ]]; then
-            exit 1
-          fi
-      - name: Test dlio benchmark hdf5
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
-        run: |
-          export DLIO_PROFILER_ENABLE=1
-          dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 \
-                        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow ++workload.dataset.format=hdf5
-          ls -a $PWD/output
-          num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
-          echo $num_posix_lines of $total_lines
-          rm -rf $PWD/output
-          if [[ "$num_posix_lines" -ne "640" ]]; then
-            exit 1
-          fi
+          curl -v -F json_file=@$FILE https://coveralls.io/api/v1/jobs
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -29,6 +29,7 @@ set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CMAKE_BINARY_DIR})
 #------------------------------------------------------------------------------
 option(BUILD_PYTHON_BINDINGS "Build python bindings." ON)
 option(DLIO_PROFILER_ENABLE_TESTS "Enable internal tests" Off)
+option(ENABLE_DLIO_BENCHMARK_TESTS "Enable dlio_benchmark tests" Off)
 #------------------------------------------------------------------------------
 # Setup install and output Directories
 #------------------------------------------------------------------------------

diff --git a/setup.py b/setup.py
@@ -70,6 +70,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         cmake_args += [f"-DCMAKE_BUILD_TYPE={build_type}"]
         enable_tests = os.environ.get("DLIO_PROFILER_ENABLE_TESTS", "Off")
         cmake_args += [f"-DDLIO_PROFILER_ENABLE_TESTS={enable_tests}"]
+        enable_dlio_tests = os.environ.get("ENABLE_DLIO_BENCHMARK_TESTS", "Off")
+        cmake_args += [f"-DENABLE_DLIO_BENCHMARK_TESTS={enable_tests}"]
 
         # CMake lets you override the generator - we need to check this.
         # Can be set with Conda-Build, for example.

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -165,3 +165,7 @@ set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_INC_MET
 
 dlp_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_CURRENT_SOURCE_DIR}/check_file.sh ${CMAKE_CURRENT_BINARY_DIR}/${test_name}* 80)
 set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name})
+
+if(ENABLE_DLIO_BENCHMARK_TESTS)
+add_subdirectory(dlio_benchmark)
+endif()
diff --git a/test/dlio_benchmark/CMakeLists.txt b/test/dlio_benchmark/CMakeLists.txt
@@ -0,0 +1,26 @@
+find_program(DLIO_BENCHMARK_EXE dlio_benchmark)
+message("-- Found dlio_benchmark at location " ${DLIO_BENCHMARK_EXE})
+
+set(COMMON_ARGS workload=unet3d ++workload.train.computation_time=0.05
+        ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True
+        ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16
+        ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2
+        ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0
+        ++workload.framework=tensorflow ++workload.reader.data_loader=tensorflow)
+set(formats npz hdf5 tfrecord png jpeg)
+set(lines 1728 192 416 544 640)
+list(LENGTH formats formats_length)
+math(EXPR formats_length "${formats_length} - 1")
+message(STATUS "Number of formats: ${formats_length}")
+foreach(index RANGE ${formats_length})
+    list(GET formats index format)
+    list(GET lines index line)
+    set(test_name test_dlio_${format})
+    dlp_add_test(${test_name} ${DLIO_BENCHMARK_EXE} ++workload.dataset.format=${format}
+            ++workload.output.folder=${CMAKE_CURRENT_BINARY_DIR}/output_${format})
+    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_ENABLE=1)
+    set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_LEVEL=ERROR)
+
+    dlp_add_test(check_file_exists_${test_name} ${BASH_PROGRAM} ${CMAKE_SOURCE_DIR}/test/check_file.sh ${CMAKE_CURRENT_BINARY_DIR}/output_${format}/.trace*.pfw ${line})
+    set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_name})
+endforeach()