added async write flag for ADIOS2 (#1460)

* added async flag * Update src/IO/ADIOS/ADIOS2IOHandler.cpp Co-authored-by: Franz Pöschel <[email protected]> * Update doc with the new flag OPENPMD_ADIOS2_ASYNC_WRITE * typo fix * Set default flush target to buffer when using this option --------- Co-authored-by: Junmin Gu <[email protected]> Co-authored-by: Franz Pöschel <[email protected]>
openPMD · Jun 21, 2023 · e5f5ac4 · e5f5ac4
1 parent f799aab
commit e5f5ac4
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 5 deletions.
diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst
@@ -84,6 +84,7 @@ environment variable                  default    description
 ``OPENPMD_ADIOS2_ENGINE``             ``File``   `ADIOS2 engine <https://adios2.readthedocs.io/en/latest/engines/engines.html>`_
 ``OPENPMD2_ADIOS2_SCHEMA``            ``0``      ADIOS2 schema version (see below)
 ``OPENPMD_ADIOS2_STATS_LEVEL``        ``0``      whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no).
+``OPENPMD_ADIOS2_ASYNC_WRITE``        ``0``      ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``).
 ``OPENPMD_ADIOS2_BP5_BufferChunkMB``  ``0``      ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation
 ``OPENPMD_ADIOS2_BP5_MaxShmMB``       ``0``      ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation
 ``OPENPMD_ADIOS2_BP5_NumSubFiles``    ``0``      ADIOS2 BP5 engine: num of subfiles
@@ -267,7 +268,7 @@ Rather than by reallocation as in BP4, this is done by appending a new chunk, le
 .. figure:: https://user-images.githubusercontent.com/14241876/181477384-ce4ea8ab-3bde-4210-991b-2e627dfcc7c9.png
   :alt: Memory usage of BP5 when flushing to the engine buffer
 
-The default is to flush to disk, but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level.
+The default is to flush to disk (except when specifying ``OPENPMD_ADIOS2_ASYNC_WRITE=1``), but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level.
 
 
 

diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp
@@ -2863,6 +2863,7 @@ namespace detail
         // set engine parameters
         std::set<std::string> alreadyConfigured;
         std::optional<bool> userSpecifiedUsesteps;
+        bool wasTheFlushTargetSpecifiedViaJSON = false;
         auto engineConfig = impl.config(ADIOS2Defaults::str_engine);
         if (!engineConfig.json().is_null())
         {
@@ -2909,9 +2910,8 @@ namespace detail
                         "Flush target must be either 'disk' or 'buffer', but "
                         "was non-literal type.");
                 }
-                overrideFlushTarget(
-                    m_impl->m_flushTarget,
-                    flushTargetFromString(target.value()));
+                m_impl->m_flushTarget = flushTargetFromString(target.value());
+                wasTheFlushTargetSpecifiedViaJSON = true;
             }
         }
 
@@ -2995,6 +2995,23 @@ namespace detail
                 m_IO.SetParameter("Profile", "Off");
             }
         }
+        if (notYetConfigured("AsyncWrite"))
+        {
+            if (1 == auxiliary::getEnvNum("OPENPMD_ADIOS2_ASYNC_WRITE", 0) &&
+                notYetConfigured("AsyncWrite"))
+            {
+                m_IO.SetParameter("AsyncWrite", "On");
+                if (!wasTheFlushTargetSpecifiedViaJSON)
+                {
+                    m_impl->m_flushTarget = FlushTarget::Buffer;
+                }
+            }
+            else
+            {
+                m_IO.SetParameter("AsyncWrite", "Off");
+            }
+        }
+
 #if openPMD_HAVE_MPI
         {
             auto num_substreams =

diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp
@@ -4378,7 +4378,11 @@ BufferChunkSize = 2147483646 # 2^31 - 2
 )";
 
     adios2_bp5_flush(
-        cfg3, /* flushDuringStep = */ FlushDuringStep::Default_Yes);
+        cfg3,
+        /* flushDuringStep = */
+        auxiliary::getEnvNum("OPENPMD_ADIOS2_ASYNC_WRITE", 0) == 0
+            ? FlushDuringStep::Default_Yes
+            : FlushDuringStep::Default_No);
 
     std::string cfg4 = R"(
 [adios2]