diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f60ec02..528cb250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,18 @@ jobs: sudo apt-get update sudo apt-get install gcc g++ libtool-bin openmpi-bin libopenmpi-dev sudo apt-get install python3.10 python3-pip + - name: Install dlio benchmark + if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github' + run: | + LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main + if [[ $PERM == 'user' ]]; then + export DLIO_LOGGER_USER=1 + echo pip install ${LOC} + pip install ${LOC} + else + echo pip install ${LOC} + sudo pip install ${LOC} + fi - name: Build from ${{ matrix.dir }} on ${{ matrix.perm }} run: | LOC=. @@ -55,22 +67,14 @@ jobs: cd build/temp*/dlio_profiler_py ctest -VV - name: Test DLIO_Benchmark - if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' + if: matrix.os == 'ubuntu-22.04' && matrix.perm == 'root' && matrix.dir == 'github' run: | - LOC=git+https://github.com/argonne-lcf/dlio_benchmark.git@main - if [[ $PERM == 'user' ]]; then - export DLIO_LOGGER_USER=1 - echo pip install ${LOC} - pip install ${LOC} - else - echo pip install ${LOC} - sudo pip install ${LOC} - fi export DLIO_PROFILER_ENABLE=1 dlio_benchmark workload=unet3d ++workload.output.folder=$PWD/output ++workload.train.computation_time=0.05 ++workload.evaluation.eval_time=0.01 ++workload.train.epochs=2 ++workload.workflow.train=True ++workload.workflow.generate_data=True ++workload.dataset.num_files_train=16 ++workload.dataset.num_files_eval=16 ++workload.reader.read_threads=2 ++workload.dataset.record_length=4096 ++workload.dataset.record_length_stdev=0 ls -a $PWD/output num_posix_lines=`cat $PWD/output/.trace*.pfw | grep POSIX | wc -l` - echo $num_posix_lines + total_lines=`cat $PWD/output/.trace*.pfw | wc -l` + echo $num_posix_lines of $total_lines if [[ "$num_posix_lines" == '0' ]]; then exit 1 fi \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3f629ac8..990650b1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -70,9 +70,10 @@ set_tests_properties(check_file_exists_${test_name} PROPERTIES DEPENDS ${test_na find_program(PYTHON_EXE python) message("-- Found python at location " ${PYTHON_EXE}) -set(test_name test_py) +set(test_name test_py_only) add_test(${test_name} ${PYTHON_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/py/test.py) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_SOURCE_DIR}/venv/lib) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}_app) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}/dependency/.spack-env/view/lib64) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_ENABLE=1) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_LEVEL=INFO) @@ -86,6 +87,7 @@ set(test_name test_py_both) add_test(${test_name} ${PYTHON_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/py/test.py) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT PYTHONPATH=$ENV{PYTHONPATH}:${CMAKE_SOURCE_DIR}/venv/lib) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}/dependency/.spack-env/view/lib64) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}_app) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT LD_PRELOAD=${CMAKE_BINARY_DIR}/lib/libdlio_profiler_preload.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_DATA_DIR=${CMAKE_CURRENT_BINARY_DIR}/data) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DLIO_PROFILER_LOG_FILE=${CMAKE_CURRENT_BINARY_DIR}/${test_name}) diff --git a/test/py/test.py b/test/py/test.py index 3bd688af..bc7bcf0c 100644 --- a/test/py/test.py +++ b/test/py/test.py @@ -1,13 +1,20 @@ -from dlio_profiler.logger import dlio_logger, fn_interceptor + from time import sleep import os import threading +from multiprocessing import get_context +import numpy as np +import PIL.Image as im + +from dlio_profiler.logger import dlio_logger, fn_interceptor + cwd = os.getcwd() -log_inst = dlio_logger.initialize_log(f"{cwd}/test_py-app.pwf", f"{cwd}/data", -1) +log_file=os.getenv("LOG_FILE", f"{cwd}/test_py-app.pwf") +log_inst = dlio_logger.initialize_log(log_file, f"{cwd}/data", -1) dlio_log = fn_interceptor("COMPUTE") @dlio_log.log -def custom_events2(): +def log_events(): sleep(1) def custom_events(): @@ -21,15 +28,13 @@ def custom_events(): log_inst.log_event("test", "cat2", start, end - start, int_args=args) -def posix_calls1(index): +def posix_calls(index): path=f"{cwd}/data/demofile{index}.txt" f = open(path, "w+") f.write("Now the file has more content!") f.close() -import numpy as np -import PIL.Image as im -def posix_calls2(index): +def npz_calls(index): #print(f"{cwd}/data/demofile2.npz") path = f"{cwd}/data/demofile{index}.npz" if os.path.exists(path): @@ -38,7 +43,7 @@ def posix_calls2(index): record_labels = [0] * 1024 np.savez(path, x=records, y=record_labels) -def write_read_jpeg(index): +def jpeg_calls(index): records = np.random.randint(255, size=(1024, 1024), dtype=np.uint8) img = im.fromarray(records) out_path_spec = f"{cwd}/data/test.jpeg" @@ -47,13 +52,15 @@ def write_read_jpeg(index): image = im.open(f) out_records = np.asarray(image) #image = im.open(out_path_spec) - +def init(): + """This function is called when new processes start.""" + print(f'Initializing process {os.getpid()}') def main(): - t1 = threading.Thread(target=posix_calls1, args=(10,)) + t1 = threading.Thread(target=posix_calls, args=(10,)) custom_events() - t2 = threading.Thread(target=posix_calls2, args=(1,)) - t3 = threading.Thread(target=write_read_jpeg, args=(2,)) + t2 = threading.Thread(target=npz_calls, args=(1,)) + t3 = threading.Thread(target=jpeg_calls, args=(2,)) # starting thread 1 t1.start() t2.start() @@ -62,8 +69,15 @@ def main(): t1.join() t2.join() t3.join() + index = 3 + for context in ('fork', 'spawn'): + with get_context(context).Pool(2, initializer=init) as pool: + pool.map(posix_calls, (index, index + 1, index + 2)) + index = index + 3 log_inst.finalize() + + if __name__ == "__main__": main() \ No newline at end of file diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt new file mode 100644 index 00000000..e69de29b