From e5f5ac49ab56178fdc3942ab8b20ac2f1a28e5ff Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Wed, 21 Jun 2023 06:30:32 -0700 Subject: [PATCH] added async write flag for ADIOS2 (#1460) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added async flag * Update src/IO/ADIOS/ADIOS2IOHandler.cpp Co-authored-by: Franz Pöschel * Update doc with the new flag OPENPMD_ADIOS2_ASYNC_WRITE * typo fix * Set default flush target to buffer when using this option --------- Co-authored-by: Junmin Gu Co-authored-by: Franz Pöschel --- docs/source/backends/adios2.rst | 3 ++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 23 ++++++++++++++++++++--- test/SerialIOTest.cpp | 6 +++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 367f44abf3..c49def1fe7 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -84,6 +84,7 @@ environment variable default description ``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ ``OPENPMD2_ADIOS2_SCHEMA`` ``0`` ADIOS2 schema version (see below) ``OPENPMD_ADIOS2_STATS_LEVEL`` ``0`` whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no). +``OPENPMD_ADIOS2_ASYNC_WRITE`` ``0`` ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``). ``OPENPMD_ADIOS2_BP5_BufferChunkMB`` ``0`` ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation ``OPENPMD_ADIOS2_BP5_MaxShmMB`` ``0`` ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation ``OPENPMD_ADIOS2_BP5_NumSubFiles`` ``0`` ADIOS2 BP5 engine: num of subfiles @@ -267,7 +268,7 @@ Rather than by reallocation as in BP4, this is done by appending a new chunk, le .. figure:: https://user-images.githubusercontent.com/14241876/181477384-ce4ea8ab-3bde-4210-991b-2e627dfcc7c9.png :alt: Memory usage of BP5 when flushing to the engine buffer -The default is to flush to disk, but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level. +The default is to flush to disk (except when specifying ``OPENPMD_ADIOS2_ASYNC_WRITE=1``), but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level. diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 266063b722..ab063b5a91 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2863,6 +2863,7 @@ namespace detail // set engine parameters std::set alreadyConfigured; std::optional userSpecifiedUsesteps; + bool wasTheFlushTargetSpecifiedViaJSON = false; auto engineConfig = impl.config(ADIOS2Defaults::str_engine); if (!engineConfig.json().is_null()) { @@ -2909,9 +2910,8 @@ namespace detail "Flush target must be either 'disk' or 'buffer', but " "was non-literal type."); } - overrideFlushTarget( - m_impl->m_flushTarget, - flushTargetFromString(target.value())); + m_impl->m_flushTarget = flushTargetFromString(target.value()); + wasTheFlushTargetSpecifiedViaJSON = true; } } @@ -2995,6 +2995,23 @@ namespace detail m_IO.SetParameter("Profile", "Off"); } } + if (notYetConfigured("AsyncWrite")) + { + if (1 == auxiliary::getEnvNum("OPENPMD_ADIOS2_ASYNC_WRITE", 0) && + notYetConfigured("AsyncWrite")) + { + m_IO.SetParameter("AsyncWrite", "On"); + if (!wasTheFlushTargetSpecifiedViaJSON) + { + m_impl->m_flushTarget = FlushTarget::Buffer; + } + } + else + { + m_IO.SetParameter("AsyncWrite", "Off"); + } + } + #if openPMD_HAVE_MPI { auto num_substreams = diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 49e070e312..758db19c09 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4378,7 +4378,11 @@ BufferChunkSize = 2147483646 # 2^31 - 2 )"; adios2_bp5_flush( - cfg3, /* flushDuringStep = */ FlushDuringStep::Default_Yes); + cfg3, + /* flushDuringStep = */ + auxiliary::getEnvNum("OPENPMD_ADIOS2_ASYNC_WRITE", 0) == 0 + ? FlushDuringStep::Default_Yes + : FlushDuringStep::Default_No); std::string cfg4 = R"( [adios2]