Merge branch 'master' into fix-static-blosc2-build

.github/workflows/everything.yml

@@ -188,7 +188,7 @@ jobs:
       - name: Build
         run: gha/scripts/ci/gh-actions/run.sh build
       - name: Print ccache statistics
-        run: ccache -s
+        run: ccache -s | tee $GITHUB_STEP_SUMMARY
       - name: Save cache
         uses: actions/cache/save@v3
         if: ${{ github.ref_name == 'master' && steps.restore-cache.outputs.cache-hit != 'true' }}

bindings/C/adios2/c/adios2_c_variable.cpp

@@ -79,11 +79,7 @@ adios2_error adios2_set_shape(adios2_variable *variable, const size_t ndims, con
 
 adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem)
 {
-#ifdef ADIOS2_HAVE_GPU_SUPPORT
     adios2::MemorySpace mem = adios2::MemorySpace::Detect;
-#else
-    adios2::MemorySpace mem = adios2::MemorySpace::Host;
-#endif
     switch (Cmem)
     {
 
@@ -104,11 +100,7 @@ adios2::MemorySpace adios2_ToMemorySpace(const adios2_memory_space Cmem)
 
 adios2_memory_space adios2_FromMemorySpace(const adios2::MemorySpace mem)
 {
-#ifdef ADIOS2_HAVE_GPU_SUPPORT
     adios2_memory_space Cmem = adios2_memory_space_detect;
-#else
-    adios2_memory_space Cmem = adios2_memory_space_host;
-#endif
     switch (mem)
     {
 

cmake/ADIOSBisonFlexSub.cmake

@@ -0,0 +1,15 @@
+FUNCTION (SETUP_ADIOS_BISON_FLEX_SUB)
+
+set (BISON_FLEX_PRECOMPILE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/toolkit/derived/parser/pregen-source")
+
+ADD_CUSTOM_COMMAND(OUTPUT parser.cpp
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.cpp ${CMAKE_CURRENT_BINARY_DIR}
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/parser.h ${CMAKE_CURRENT_BINARY_DIR}
+        COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/location.hh ${CMAKE_CURRENT_BINARY_DIR}
+	COMMENT "Using pre-generated Bison Output from ${BISON_FLEX_PRECOMPILE_DIR}")
+ADD_CUSTOM_COMMAND(OUTPUT lexer.cpp
+	COMMAND ${CMAKE_COMMAND} -E copy ${BISON_FLEX_PRECOMPILE_DIR}/lexer.cpp ${CMAKE_CURRENT_BINARY_DIR}
+	COMMENT "Using pre-generated Flex Output from ${BISON_FLEX_PRECOMPILE_DIR}")
+
+set (BISON_Parser_OUTPUT_SOURCE parser.cpp PARENT_SCOPE)
+ENDFUNCTION()

docs/user_guide/source/api_full/fortran.rst

@@ -746,11 +746,11 @@ ADIOS2 Fortran bindings handlers are mapped 1-to-1 to the ADIOS2 components desc
       integer, intent(out) :: ierr
       
 
-* :f90:`subroutine adios2_set_steps_selection` set a list of steps by specifying the starting step and the step count
+* :f90:`subroutine adios2_set_step_selection` set a list of steps by specifying the starting step and the step count
 
    .. code-block:: fortran
    
-      subroutine adios2_set_selection(variable, step_start, step_count, ierr)
+      subroutine adios2_set_step_selection(variable, step_start, step_count, ierr)
       
       ! WHERE
       

scripts/ci/scripts/run-clang-format.sh

@@ -12,7 +12,7 @@ then
 fi
 
 # Check C and C++ code with clang-format
-find source plugins testing examples bindings -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)'  | xargs clang-format -i
+find source plugins testing examples bindings  -regextype posix-extended -iregex '.*\.(h|c|cpp|tcc|cu)' ! -path "source/adios2/toolkit/derived/parser/pregen-source/*" | xargs clang-format -i
 DIFF="$(git diff)"
 if [ -n "${DIFF}" ]
 then

source/adios2/CMakeLists.txt

@@ -135,12 +135,51 @@ if (ADIOS2_HAVE_Derived_Variable)
       toolkit/derived/Expression.cpp
       toolkit/derived/Function.cpp toolkit/derived/Function.tcc
       toolkit/derived/ExprHelper.h)
+  set_target_properties(adios2_core PROPERTIES
+  				   INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>")
+  find_package(BISON "3.8.2")
+  find_package(FLEX)
+
+  if(NOT BISON_FOUND OR NOT FLEX_FOUND)
+    include(ADIOSBisonFlexSub)
+    SETUP_ADIOS_BISON_FLEX_SUB()
+  else()
+    BISON_TARGET(MyParser
+      toolkit/derived/parser/parser.y
+      ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
+      COMPILE_FLAGS "-o parser.cpp --header=parser.h"
+      DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/parser.h)
+    FLEX_TARGET(MyScanner
+      toolkit/derived/parser/lexer.l
+      COMPILE_FLAGS "-o lexer.cpp --header-file=lexer.h" 
+      ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
+      DEFINES_FILE ${CMAKE_CURRENT_BINARY_DIR}/lexer.h)
+    ADD_FLEX_BISON_DEPENDENCY(MyScanner MyParser)
+  endif()
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp PROPERTIES COMPILE_FLAGS -Wno-sign-compare)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+    SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/parser.cpp PROPERTIES COMPILE_FLAGS -Wno-unused-but-set-variable)
+  elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    SET_SOURCE_FILES_PROPERTIES(toolkit/derived/Expression.cpp toolkit/derived/Function.cpp PROPERTIES COMPILE_FLAGS "/wd4005 /wd4065 /wd4267 -DYY_NO_UNISTD_H")
+  endif()
   add_library(adios2_core_derived
-      toolkit/derived/parser/lexer.cpp
-      toolkit/derived/parser/parser.cpp
+      ${CMAKE_CURRENT_BINARY_DIR}/lexer.cpp
+      ${CMAKE_CURRENT_BINARY_DIR}/parser.cpp
+      toolkit/derived/parser/ASTDriver.cpp
       toolkit/derived/parser/ASTNode.cpp)
+  set_target_properties(adios2_core_derived PROPERTIES
+  					   VISIBILITY_INLINES_HIDDEN ON
+					   INCLUDE_DIRECTORIES "$<BUILD_INTERFACE:${ADIOS2_SOURCE_DIR}/source/adios2/toolkit/derived/parser>;$<BUILD_INTERFACE:${ADIOS2_BINARY_DIR}/source/adios2>"
+					   EXPORT_NAME core_derived
+					   OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_derived)
   target_link_libraries(adios2_core PRIVATE adios2_core_derived)
   set(maybe_adios2_core_derived adios2_core_derived)
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    set_target_properties(adios2_core_derived PROPERTIES COMPILE_FLAGS "/wd4005 /wd4267 /wd4065 -DYY_NO_UNISTD_H")
+  endif()
 endif()
 
 set(maybe_adios2_core_cuda)

source/adios2/common/ADIOSTypes.cpp

@@ -18,6 +18,23 @@
 namespace adios2
 {
 
+std::string ToString(MemorySpace value)
+{
+    switch (value)
+    {
+    case MemorySpace::Detect:
+        return "MemorySpace::Detect";
+    case MemorySpace::Host:
+        return "MemorySpace::Host";
+#ifdef ADIOS2_HAVE_GPU_SUPPORT
+    case MemorySpace::GPU:
+        return "MemorySpace::GPU";
+#endif
+    default:
+        return "ToString: Unknown MemorySpace";
+    }
+}
+
 std::string ToString(ShapeID value)
 {
     switch (value)

source/adios2/common/ADIOSTypes.h

@@ -355,6 +355,7 @@ std::string ToString(SelectionType value);
 std::string ToString(DataType type);
 std::string ToString(const Dims &dims);
 std::string ToString(const Box<Dims> &box);
+std::string ToString(const MemorySpace value);
 
 /**
  * os << [adios2_type] enables output of adios2 enums/classes directly

source/adios2/core/VariableBase.h

@@ -52,11 +52,7 @@ class VariableBase
     const size_t m_ElementSize;
 
     /* User requested memory space */
-#ifdef ADIOS2_HAVE_GPU_SUPPORT
     MemorySpace m_MemSpace = MemorySpace::Detect;
-#else
-    MemorySpace m_MemSpace = MemorySpace::Host;
-#endif
 #if defined(ADIOS2_HAVE_KOKKOS) || defined(ADIOS2_HAVE_GPU_SUPPORT)
     ArrayOrdering m_BaseLayout;
     ArrayOrdering m_ArrayLayout = ArrayOrdering::Auto;

source/adios2/core/VariableDerived.cpp

@@ -11,7 +11,7 @@ VariableDerived::VariableDerived(const std::string &name, adios2::derived::Expre
                                  const DerivedVarType varType)
 : VariableBase(name, exprType, helper::GetDataTypeSize(exprType), expr.GetShape(), expr.GetStart(),
                expr.GetCount(), isConstant),
-  m_Expr(expr), m_DerivedType(varType)
+  m_DerivedType(varType), m_Expr(expr)
 {
 }
 
@@ -27,15 +27,15 @@ void VariableDerived::UpdateExprDim(std::map<std::string, std::tuple<Dims, Dims,
 }
 
 std::vector<std::tuple<void *, Dims, Dims>>
-VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
+VariableDerived::ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &NameToMVI)
 {
     size_t numBlocks = 0;
     // check that all variables have the same number of blocks
-    for (auto variable : NameToMVI)
+    for (const auto &variable : NameToMVI)
     {
         if (numBlocks == 0)
-            numBlocks = variable.second.BlocksInfo.size();
-        if (numBlocks != variable.second.BlocksInfo.size())
+            numBlocks = variable.second->BlocksInfo.size();
+        if (numBlocks != variable.second->BlocksInfo.size())
             helper::Throw<std::invalid_argument>("Core", "VariableDerived", "ApplyExpression",
                                                  " variables do not have the same number of blocks "
                                                  " in computing the derived variable " +
@@ -44,21 +44,21 @@ VariableDerived::ApplyExpression(std::map<std::string, MinVarInfo> NameToMVI)
 
     std::map<std::string, std::vector<adios2::derived::DerivedData>> inputData;
     // create the map between variable name and DerivedData object
-    for (auto variable : NameToMVI)
+    for (const auto &variable : NameToMVI)
     {
         // add the dimensions of all blocks into a vector
         std::vector<adios2::derived::DerivedData> varData;
         for (size_t i = 0; i < numBlocks; i++)
         {
             Dims start;
             Dims count;
-            for (size_t d = 0; d < variable.second.Dims; d++)
+            for (int d = 0; d < variable.second->Dims; d++)
             {
-                start.push_back(variable.second.BlocksInfo[i].Start[d]);
-                count.push_back(variable.second.BlocksInfo[i].Count[d]);
+                start.push_back(variable.second->BlocksInfo[i].Start[d]);
+                count.push_back(variable.second->BlocksInfo[i].Count[d]);
             }
             varData.push_back(adios2::derived::DerivedData(
-                {variable.second.BlocksInfo[i].BufferP, start, count}));
+                {variable.second->BlocksInfo[i].BufferP, start, count}));
         }
         inputData.insert({variable.first, varData});
     }

source/adios2/core/VariableDerived.h

@@ -32,7 +32,7 @@ class VariableDerived : public VariableBase
     ApplyExpression(std::map<std::string, std::vector<void *>> NameToData,
                     std::map<std::string, std::tuple<Dims, Dims, Dims>> NameToDims);
     std::vector<std::tuple<void *, Dims, Dims>>
-    ApplyExpression(std::map<std::string, MinVarInfo> mvi);
+    ApplyExpression(std::map<std::string, std::unique_ptr<MinVarInfo>> &mvi);
 };
 
 } // end namespace core

source/adios2/engine/bp5/BP5Writer.cpp

@@ -516,7 +516,7 @@ void BP5Writer::ComputeDerivedVariables()
         auto derivedVar = dynamic_cast<core::VariableDerived *>((*it).second.get());
         std::vector<std::string> varList = derivedVar->VariableNameList();
         // to create a mapping between variable name and the varInfo (dim and data pointer)
-        std::map<std::string, MinVarInfo> nameToVarInfo;
+        std::map<std::string, std::unique_ptr<MinVarInfo>> nameToVarInfo;
         bool computeDerived = true;
         for (auto varName : varList)
         {
@@ -536,7 +536,7 @@ void BP5Writer::ComputeDerivedVariables()
                 std::cout << " .. skip derived variable " << (*it).second->m_Name << std::endl;
                 break;
             }
-            nameToVarInfo.insert({varName, *mvi});
+            nameToVarInfo.insert({varName, std::unique_ptr<MinVarInfo>(mvi)});
         }
         // skip computing derived variables if it contains variables that are not written this step
         if (!computeDerived)

source/adios2/engine/dataman/DataManReader.tcc

@@ -31,12 +31,13 @@ void DataManReader::GetSyncCommon(Variable<T> &variable, T *data)
 template <class T>
 void DataManReader::GetDeferredCommon(Variable<T> &variable, T *data)
 {
+    auto varMemSpace = variable.GetMemorySpace(data);
     if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor)
     {
         while (true)
         {
             int ret = m_Serializer.GetData(data, variable.m_Name, variable.m_Start,
-                                           variable.m_Count, m_CurrentStep, variable.m_MemSpace,
+                                           variable.m_Count, m_CurrentStep, varMemSpace,
                                            variable.m_MemoryStart, variable.m_MemoryCount);
             if (ret == 0)
             {
@@ -57,7 +58,7 @@ void DataManReader::GetDeferredCommon(Variable<T> &variable, T *data)
         while (true)
         {
             int ret = m_Serializer.GetData(data, variable.m_Name, start, count, m_CurrentStep,
-                                           variable.m_MemSpace, memstart, memcount);
+                                           varMemSpace, memstart, memcount);
             if (ret == 0)
             {
                 break;

source/adios2/engine/dataman/DataManWriter.tcc

@@ -31,10 +31,11 @@ void DataManWriter::PutSyncCommon(Variable<T> &variable, const T *values)
 template <class T>
 void DataManWriter::PutDeferredCommon(Variable<T> &variable, const T *values)
 {
+    auto varMemSpace = variable.GetMemorySpace(values);
     variable.SetData(values);
     if (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor)
     {
-        m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, "");
+        m_Serializer.PutData(variable, m_Name, CurrentStep(), m_MpiRank, varMemSpace, "");
     }
     else
     {
@@ -49,7 +50,7 @@ void DataManWriter::PutDeferredCommon(Variable<T> &variable, const T *values)
         std::reverse(memstart.begin(), memstart.end());
         std::reverse(memcount.begin(), memcount.end());
         m_Serializer.PutData(variable.m_Data, variable.m_Name, shape, start, count, memstart,
-                             memcount, variable.m_MemSpace, m_Name, CurrentStep(), m_MpiRank, "",
+                             memcount, varMemSpace, m_Name, CurrentStep(), m_MpiRank, "",
                              variable.m_Operations);
     }
 

source/adios2/toolkit/derived/ExprHelper.h

@@ -57,4 +57,4 @@ inline ExpressionOperator get_op(std::string op) { return string_to_op.at(op); }
 
 }
 }
-#endif
+#endif

source/adios2/toolkit/derived/Expression.cpp

@@ -2,19 +2,35 @@
 #define ADIOS2_DERIVED_Expression_CPP_
 
 #include "Expression.h"
-#include "parser/ASTNode.h"
-#include "parser/parser.h"
+#include "parser/ASTDriver.h"
 
 namespace adios2
 {
 namespace detail
 {
+// helper function
+adios2::detail::ExpressionOperator convert_op(std::string opname)
+{
+    adios2::detail::ExpressionOperator op;
+    try
+    {
+        op = adios2::detail::get_op(opname);
+    }
+    catch (std::out_of_range &e)
+    {
+        (void)e; // use e
+        helper::Throw<std::invalid_argument>("Derived", "ExprHelper", "get_op",
+                                             "Parser cannot recognize operator '" + opname + "'.");
+    }
+    return op;
+};
+
 adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNode *node)
 {
-    adios2::derived::ExpressionTree exprTree_node(node->operation);
-    for (adios2::detail::ASTNode *e : node->sub_expr)
+    adios2::derived::ExpressionTree exprTree_node(convert_op(node->get_opname()));
+    for (adios2::detail::ASTNode *e : node->get_subexprs())
     {
-        switch (e->operation)
+        switch (convert_op(e->get_opname()))
         {
         case adios2::detail::ExpressionOperator::OP_ALIAS: // add variable given by alias
             // add an index operation in the chain if the variable contains indeces
@@ -25,19 +41,19 @@ adios2::derived::ExpressionTree ASTNode_to_ExpressionTree(adios2::detail::ASTNod
                 index_expr.add_child(e->lookup_var_path(e->alias));
                 expTree_node->add_child(expr);
             }*/
-            exprTree_node.add_child(e->lookup_var_path(e->alias));
+            exprTree_node.add_child(e->get_varname());
             break;
         case adios2::detail::ExpressionOperator::OP_PATH: // add variable name
-            exprTree_node.add_child(e->alias);
+            exprTree_node.add_child(e->get_varname());
             break;
         case adios2::detail::ExpressionOperator::OP_NUM: // set the base value for the operation
-            exprTree_node.set_base(e->value);
+            exprTree_node.set_base(e->get_value());
             break;
         default: // if the children nodes are other expressions, convert them to expressions
             auto temp_node = ASTNode_to_ExpressionTree(e);
             // move from a binary to a multinary tree if the child has the same operation
-            if (e->operation == node->operation &&
-                adios2::detail::op_property.at(e->operation).is_associative)
+            if (convert_op(e->get_opname()) == convert_op(node->get_opname()) &&
+                adios2::detail::op_property.at(convert_op(e->get_opname())).is_associative)
             {
                 // concatenate exprTree with temp_node
                 for (std::tuple<adios2::derived::ExpressionTree, std::string, bool> childTree :
@@ -63,10 +79,10 @@ namespace derived
 {
 
 Expression::Expression(std::string string_exp)
-: ExprString(string_exp), m_Shape({0}), m_Start({0}), m_Count({0})
+: m_Shape({0}), m_Start({0}), m_Count({0}), ExprString(string_exp)
 {
-    adios2::detail::ASTNode *root_node = adios2::detail::parse_expression(string_exp);
-    m_Expr = adios2::detail::ASTNode_to_ExpressionTree(root_node);
+    adios2::detail::ASTDriver drv(string_exp);
+    m_Expr = adios2::detail::ASTNode_to_ExpressionTree(drv.getAST());
 }
 
 std::vector<std::string> Expression::VariableNameList() { return m_Expr.VariableNameList(); }

source/adios2/toolkit/derived/Expression.h

@@ -34,7 +34,7 @@ class ExpressionTree
     std::vector<std::tuple<ExpressionTree, std::string, bool>> sub_exprs;
     OpInfo detail;
 
-    ExpressionTree(){};
+    ExpressionTree() : detail({adios2::detail::ExpressionOperator::OP_NULL, {}, 0}) {}
     ExpressionTree(adios2::detail::ExpressionOperator o) : detail({o, {}, 0}) {}
     ExpressionTree(adios2::detail::ExpressionOperator o, double c) : detail({o, {}, 0}) {}
     ExpressionTree(std::vector<std::tuple<size_t, size_t, size_t>> indices)