fix ci

hariharan-devarajan · Oct 8, 2023 · 3dd1ec0 · 3dd1ec0
1 parent 99da852
commit 3dd1ec0
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 24 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,6 +34,18 @@ jobs:
           sudo apt-get update
           sudo apt-get install gcc g++ libtool-bin openmpi-bin libopenmpi-dev
           sudo apt-get install python3.10 python3-pip
+      - name: Install dlio benchmark
+        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
+        run: |
+          LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main
+          if [[ $PERM == 'user' ]]; then
+            export DLIO_LOGGER_USER=1
+            echo pip install ${LOC}
+            pip install ${LOC}
+          else
+             echo pip install ${LOC}
+             sudo pip install ${LOC}
+          fi
       - name: Build from ${{ matrix.dir }} on ${{ matrix.perm }}
         run: |
           LOC=.
@@ -55,22 +67,14 @@ jobs:
           cd build/temp*/dlio_profiler_py
           ctest -VV
       - name: Test DLIO_Benchmark
-        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root'
+        if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github'
         run: |
-          LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main
-          if [[ $PERM == 'user' ]]; then
-            export DLIO_LOGGER_USER=1
-            echo pip install ${LOC}
-            pip install ${LOC}
-          else
-             echo pip install ${LOC}
-             sudo pip install ${LOC}
-          fi
           export DLIO_PROFILER_ENABLE=1
           dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0
           ls -a $PWD/output
           num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l`
-          echo $num_posix_lines
+          total_lines=`cat $PWD/output/.trace*.pfw | wc -l`
+          echo $num_posix_lines of $total_lines
           if [[ "$num_posix_lines" == '0' ]]; then
             exit 1
           fi
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -70,9 +70,10 @@ set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_na
 
 find_program(PYTHON_EXE python)
 message("-- Found python at location " ${PYTHON_EXE})
-set(test_name test_py)
+set(test_name test_py_only)
 add_test(${test_name} ${PYTHON_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/py/test.py)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_SOURCE_DIR}/venv/lib)
+set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}_app)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}/dependency/.spack-env/view/lib64)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_ENABLE=1)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_LEVEL=INFO)
@@ -86,6 +87,7 @@ set(test_name test_py_both)
 add_test(${test_name} ${PYTHON_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/py/test.py)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_SOURCE_DIR}/venv/lib)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}/dependency/.spack-env/view/lib64)
+set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}_app)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BINARY_DIR}/lib/libdlio_profiler_preload.so)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_DATA_DIR=${CMAKE_CURRENT_BINARY_DIR}/data)
 set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name})

diff --git a/test/py/test.py b/test/py/test.py
@@ -1,13 +1,20 @@
-from dlio_profiler.logger import dlio_logger, fn_interceptor
+
 from time import sleep
 import os
 import threading
+from multiprocessing import get_context
+import numpy as np
+import PIL.Image as im
+
+from dlio_profiler.logger import dlio_logger, fn_interceptor
+
 cwd = os.getcwd()
-log_inst = dlio_logger.initialize_log(f"{cwd}/test_py-app.pwf", f"{cwd}/data", -1)
+log_file=os.getenv("LOG_FILE", f"{cwd}/test_py-app.pwf")
+log_inst = dlio_logger.initialize_log(log_file, f"{cwd}/data", -1)
 dlio_log = fn_interceptor("COMPUTE")
 
 @dlio_log.log
-def custom_events2():
+def log_events():
     sleep(1)
 
 def custom_events():
@@ -21,15 +28,13 @@ def custom_events():
     log_inst.log_event("test", "cat2", start, end - start, int_args=args)
 
 
-def posix_calls1(index):
+def posix_calls(index):
     path=f"{cwd}/data/demofile{index}.txt"
     f = open(path, "w+")
     f.write("Now the file has more content!")
     f.close()
 
-import numpy as np
-import PIL.Image as im
-def posix_calls2(index):
+def npz_calls(index):
     #print(f"{cwd}/data/demofile2.npz")
     path = f"{cwd}/data/demofile{index}.npz"
     if os.path.exists(path):
@@ -38,7 +43,7 @@ def posix_calls2(index):
     record_labels = [0] * 1024
     np.savez(path, x=records, y=record_labels)
 
-def write_read_jpeg(index):
+def jpeg_calls(index):
     records = np.random.randint(255, size=(1024, 1024), dtype=np.uint8)
     img = im.fromarray(records)
     out_path_spec = f"{cwd}/data/test.jpeg"
@@ -47,13 +52,15 @@ def write_read_jpeg(index):
         image = im.open(f)
         out_records = np.asarray(image)
     #image = im.open(out_path_spec)
-
+def init():
+    """This function is called when new processes start."""
+    print(f'Initializing process {os.getpid()}')
 def main():
 
-    t1 = threading.Thread(target=posix_calls1, args=(10,))
+    t1 = threading.Thread(target=posix_calls, args=(10,))
     custom_events()
-    t2 = threading.Thread(target=posix_calls2, args=(1,))
-    t3 = threading.Thread(target=write_read_jpeg, args=(2,))
+    t2 = threading.Thread(target=npz_calls, args=(1,))
+    t3 = threading.Thread(target=jpeg_calls, args=(2,))
     # starting thread 1
     t1.start()
     t2.start()
@@ -62,8 +69,15 @@ def main():
     t1.join()
     t2.join()
     t3.join()
+    index = 3
+    for context in ('fork', 'spawn'):
+        with get_context(context).Pool(2, initializer=init) as pool:
+            pool.map(posix_calls, (index, index + 1, index + 2))
+        index = index + 3
 
     log_inst.finalize()
 
+
+
 if __name__ == "__main__":
     main()
diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt