Testing refactor

.github/workflows/capgen_unit_tests.yaml

@@ -15,8 +15,38 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: update repos and install dependencies
-      run: sudo apt-get update && sudo apt-get install -y build-essential ${{matrix.fortran-compiler}} cmake python3 git libxml2-utils
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          build-essential \
+          libopenmpi-dev \
+          ${{matrix.fortran-compiler}} \
+          cmake \
+          python3 \
+          git \
+          libxml2-utils
+        pip install --user pytest
+
+    - name: Build the framework
+      run: |
+        cmake -S. -B./build -DCCPP_FRAMEWORK_ENABLE_TESTS=ON
+        cd build
+        make
+
     - name: Run unit tests
-      run: cd test && ./run_fortran_tests.sh
+      run: |
+        cd build
+        ctest --rerun-failed --output-on-failure . --verbose
+
+    - name: Run python tests
+      run: |
+        BUILD_DIR=./build \
+        PYTHONPATH=test/:scripts/ \
+            pytest \
+              test/capgen_test/capgen_test_reports.py \
+              test/advection_test/advection_test_reports.py \
+              test/ddthost_test/ddthost_test_reports.py \
+              test/var_compatibility_test/var_compatibility_test_reports.py
+
     - name: Run Fortran to metadata test
       run: cd test && ./test_fortran_to_metadata.sh

CMakeLists.txt

@@ -4,46 +4,108 @@ project(ccpp_framework
         VERSION 5.0.0
         LANGUAGES Fortran)
 
+include(cmake/ccpp_capgen.cmake)
+
 #------------------------------------------------------------------------------
 # Set package definitions
 set(PACKAGE "ccpp-framework")
 set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Dustin Swales" "Courtney Peverley")
 string(TIMESTAMP YEAR "%Y")
 
+option(CCPP_FRAMEWORK_BUILD_DOCUMENTATION
+       "Create and install the HTML documentation (requires Doxygen)" OFF)
+option(OPENMP "Enable OpenMP support for the framework" OFF)
+option(CCPP_FRAMEWORK_ENABLE_TESTS "Enable building/running CCPP regression tests" OFF)
+option(CCPP_RUN_ADVECTION_TEST          "Enable advection regression test" OFF)
+option(CCPP_RUN_CAPGEN_TEST             "Enable capgen regression test" OFF)
+option(CCPP_RUN_DDT_HOST_TEST           "Enable ddt host regression test" OFF)
+option(CCPP_RUN_VAR_COMPATIBILITY_TEST  "Enable variable compatibility regression test" OFF)
+option(BUILD_SHARED_LIBS "Build a static library" OFF)
+set(CCPP_VERBOSITY "0" CACHE STRING "Verbosity level of output (default: 0)")
+
+# Warn user on conflicting test options
+if(CCPP_RUN_ADVECTION_TEST OR
+   CCPP_RUN_CAPGEN_TEST    OR
+   CCPP_RUN_DDT_HOST_TEST  OR
+   CCPP_RUN_VAR_COMPATIBILITY_TEST)
+   set(CCPP_MANUALLY_DECLARED_TEST ON BOOL)
+endif()
+if(CCPP_MANUALLY_DECLARED_TEST AND CCPP_FRAMEWORK_ENABLE_TESTS)
+  message(WARNING "Detected a manual test flag and the flag to run all tests.  If only expected to run a single test, please unset CCPP_FRAMEWORK_ENABLE_TESTS option.")
+endif()
+set(CCPP_RUNNING_TESTS CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_MANUALLY_DECLARED_TEST)
+
+# If running tests, set appropriate flags to help with debugging test issues.
+if(CCPP_RUNNING_TESTS)
+  if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
+    ADD_COMPILE_OPTIONS(-fcheck=all)
+    ADD_COMPILE_OPTIONS(-fbacktrace)
+    ADD_COMPILE_OPTIONS(-ffpe-trap=zero)
+    ADD_COMPILE_OPTIONS(-finit-real=nan)
+    ADD_COMPILE_OPTIONS(-ggdb)
+    ADD_COMPILE_OPTIONS(-ffree-line-length-none)
+    ADD_COMPILE_OPTIONS(-cpp)
+  elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
+    ADD_COMPILE_OPTIONS(-fpe0)
+    ADD_COMPILE_OPTIONS(-warn)
+    ADD_COMPILE_OPTIONS(-traceback)
+    ADD_COMPILE_OPTIONS(-debug extended)
+    ADD_COMPILE_OPTIONS(-fpp)
+    ADD_COMPILE_OPTIONS(-diag-disable=10448)
+  elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "IntelLLVM")
+    ADD_COMPILE_OPTIONS(-fpe0)
+    ADD_COMPILE_OPTIONS(-warn)
+    ADD_COMPILE_OPTIONS(-traceback)
+    ADD_COMPILE_OPTIONS(-debug full)
+    ADD_COMPILE_OPTIONS(-fpp)
+  elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVIDIA" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC")
+    ADD_COMPILE_OPTIONS(-Mnoipa)
+    ADD_COMPILE_OPTIONS(-traceback)
+    ADD_COMPILE_OPTIONS(-Mfree)
+    ADD_COMPILE_OPTIONS(-Ktrap=fp)
+    ADD_COMPILE_OPTIONS(-Mpreprocess)
+  else()
+    message (WARNING "This program may not be able to be compiled with compiler :${CMAKE_Fortran_COMPILER_ID}")
+  endif()
+endif()
+
+# Use rpaths on MacOSX
+set(CMAKE_MACOSX_RPATH 1)
+
 #------------------------------------------------------------------------------
 # Set MPI flags for Fortran with MPI F08 interface
-find_package(MPI REQUIRED Fortran)
+find_package(MPI COMPONENTS Fortran REQUIRED)
 if(NOT MPI_Fortran_HAVE_F08_MODULE)
   message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface")
 endif()
 
 #------------------------------------------------------------------------------
 # Set OpenMP flags for C/C++/Fortran
-if (OPENMP)
+if(OPENMP)
   find_package(OpenMP REQUIRED)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
   set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}")
-endif (OPENMP)
+endif()
 
 #------------------------------------------------------------------------------
 # Set a default build type if none was specified
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     message(STATUS "Setting build type to 'Release' as none was specified.")
-    set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE)
-    # Set the possible values of build type for cmake-gui
-    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "Coverage")
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
 endif()
 
 #------------------------------------------------------------------------------
-# Pass debug/release flag to Fortran files for preprocessor
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  add_definitions(-DDEBUG)
+# Add the sub-directories
+add_subdirectory(src)
+
+if(CCPP_RUNNING_TESTS)
+  enable_testing()
+  add_subdirectory(test)
 endif()
 
-#------------------------------------------------------------------------------
-# Request a static build
-option(BUILD_SHARED_LIBS "Build a static library" OFF)
+if (CCPP_FRAMEWORK_BUILD_DOCUMENTATION)
+  find_package(Doxygen REQUIRED)
+  add_subdirectory(doc)
+endif()
 
-#------------------------------------------------------------------------------
-# Add the sub-directories
-add_subdirectory(src)
-add_subdirectory(doc)

cmake/ccpp_capgen.cmake

@@ -0,0 +1,129 @@
+# CMake wrapper for ccpp_capgen.py
+# Currently meant to be a CMake API needed for generating caps for regression tests.
+#
+# CAPGEN_DEBUG              - ON/OFF (Default: OFF) - Enables debug capability through ccpp_capgen.py
+# CAPGEN_EXPECT_THROW_ERROR - ON/OFF (Default: OFF) - Scans ccpp_capgen.py log for error string and errors if not found.
+# HOST_NAME                 - String name of host
+# OUTPUT_ROOT               - String path to put generated caps
+# VERBOSITY                 - Number of --verbose flags to pass to capgen
+# HOSTFILES                 - CMake list of host metadata filenames
+# SCHEMEFILES               - CMake list of scheme metadata files
+# SUITES                    - CMake list of suite xml files
+function(ccpp_capgen)
+  set(optionalArgs CAPGEN_DEBUG CAPGEN_EXPECT_THROW_ERROR)
+  set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY)
+  set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES)
+
+  cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN})
+
+  # Error if script file not found.
+  set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/scripts/ccpp_capgen.py")
+  if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST})
+    message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen.py.  Looked for ${CCPP_CAPGEN_CMD_LIST}.")
+  endif()
+
+  # Interpret parsed arguments
+  if(DEFINED arg_CAPGEN_DEBUG)
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--debug")
+  endif()
+  if(DEFINED arg_HOSTFILES)
+    list(JOIN arg_HOSTFILES "," HOSTFILES_SEPARATED)
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--host-files" "${HOSTFILES_SEPARATED}")
+  endif()
+  if(DEFINED arg_SCHEMEFILES)
+    list(JOIN arg_SCHEMEFILES "," SCHEMEFILES_SEPARATED)
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--scheme-files" "${SCHEMEFILES_SEPARATED}")
+  endif()
+  if(DEFINED arg_SUITES)
+    list(JOIN arg_SUITES "," SUITES_SEPARATED)
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}")
+  endif()
+  if(DEFINED arg_HOST_NAME)
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}")
+  endif()
+  if(DEFINED arg_OUTPUT_ROOT)
+    message(STATUS "Creating output directory: ${arg_OUTPUT_ROOT}")
+    file(MAKE_DIRECTORY "${arg_OUTPUT_ROOT}")
+    list(APPEND CCPP_CAPGEN_CMD_LIST "--output-root" "${arg_OUTPUT_ROOT}")
+  endif()
+  if(DEFINED arg_VERBOSITY)
+    string(REPEAT "--verbose" ${arg_VERBOSITY} VERBOSE_PARAMS_SEPERATED)
+    separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPERATED}")
+    list(APPEND CCPP_CAPGEN_CMD_LIST ${VERBOSE_PARAMS})
+  endif()
+
+  message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}")
+
+  unset(CAPGEN_OUT) # Unset CAPGEN_OUT to prevent incorrect output on subsequent calls.
+  execute_process(COMMAND ${CCPP_CAPGEN_CMD_LIST}
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+                  OUTPUT_VARIABLE CAPGEN_OUT
+                  ERROR_VARIABLE CAPGEN_OUT
+                  RESULT_VARIABLE RES
+                  COMMAND_ECHO STDOUT)
+
+  message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}")
+
+  if(arg_CAPGEN_EXPECT_THROW_ERROR)
+    # Determine if the process succeeded but had an expected string in the process log.
+    string(FIND "${CAPGEN_OUT}" "Variables of type ccpp_constituent_properties_t only allowed in register phase" ERROR_INDEX)
+
+    if (ERROR_INDEX GREATER -1)
+      message(STATUS "Capgen build produces expected error message.")
+    else()
+      message(FATAL_ERROR "CCPP cap generation did not generate expected error. Expected 'Variables of type constituent_properties_t only allowed in register phase.")
+    endif()
+  else()
+    if(RES EQUAL 0)
+      message(STATUS "ccpp-capgen completed successfully")
+    else()
+      message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}")
+    endif()
+  endif()
+endfunction()
+
+# CMake wrapper for ccpp_datafile.py
+# Currently meant to be a CMake API needed for generating caps for regression tests.
+#
+# DATATABLE   - Path to generated datatable.xml file
+# REPORT_NAME - String report name to get list of generated files form capgen (typically --ccpp-files)
+function(ccpp_datafile)
+  set(oneValueArgs DATATABLE REPORT_NAME)
+  cmake_parse_arguments(arg "" "${oneValueArgs}" "" ${ARGN})
+
+  set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/scripts/ccpp_datafile.py")
+
+  if(NOT EXISTS ${CCPP_DATAFILE_CMD})
+    message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py.  Looked for ${CCPP_DATAFILE_CMD}.")
+  endif()
+
+  if(NOT DEFINED arg_REPORT_NAME)
+    message(FATAL_ERROR "function(ccpp_datafile): REPORT_NAME not set.  Must specify the report to generate to run cpp_datafile.py")
+  endif()
+  list(APPEND CCPP_DATAFILE_CMD "${arg_REPORT_NAME}")
+
+  if(NOT DEFINED arg_DATATABLE)
+    message(FATAL_ERROR "function(ccpp_datafile): DATATABLE not set.  A datatable file must be configured to call ccpp_datafile.")
+  endif()
+  list(APPEND CCPP_DATAFILE_CMD "${arg_DATATABLE}")
+
+  message(STATUS "Running ccpp_datafile from ${CMAKE_CURRENT_SOURCE_DIR}")
+
+  unset(CCPP_CAPS)
+  execute_process(COMMAND ${CCPP_DATAFILE_CMD}
+                  WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+                  OUTPUT_VARIABLE CCPP_CAPS
+                  RESULT_VARIABLE RES
+                  OUTPUT_STRIP_TRAILING_WHITESPACE
+                  ERROR_STRIP_TRAILING_WHITESPACE
+                  COMMAND_ECHO STDOUT)
+  message(STATUS "CCPP_CAPS = ${CCPP_CAPS}")
+  if(RES EQUAL 0)
+    message(STATUS "CCPP cap files retrieved")
+  else()
+    message(FATAL_ERROR "CCPP cap file retrieval FAILED: result = ${RES}")
+  endif()
+  string(REPLACE "," ";" CCPP_CAPS_LIST ${CCPP_CAPS}) # Convert "," separated list from python back to ";" separated list for CMake.
+  set(CCPP_CAPS_LIST "${CCPP_CAPS_LIST}" PARENT_SCOPE)
+endfunction()
+

doc/CMakeLists.txt

@@ -2,28 +2,16 @@
 # Doxygen rules
 #
 # Add a target to generate API documentation with Doxygen
-find_package(Doxygen)
-option(BUILD_DOCUMENTATION
-       "Create and install the HTML documentation (requires Doxygen)"
-       ${DOXYGEN_FOUND})
+set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
+set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
 
-#
-if(BUILD_DOCUMENTATION)
-    if(NOT DOXYGEN_FOUND)
-        message(FATAL_ERROR "Doxygen is needed to build the documentation.")
-    endif()
-
-    set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
-    set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
-
-    configure_file(${doxyfile_in} ${doxyfile} @ONLY)
+configure_file(${doxyfile_in} ${doxyfile} @ONLY)
 
-    add_custom_target(doc
-        COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile}
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-        COMMENT "Generating API documentation with Doxygen"
-        VERBATIM)
-endif()
+add_custom_target(doc
+    COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generating API documentation with Doxygen"
+    VERBATIM)
 
 set(gmtb_sty_in ${CMAKE_CURRENT_SOURCE_DIR}/DevelopersGuide/gmtb.sty)
 set(gmtb_sty ${CMAKE_CURRENT_BINARY_DIR}/DevelopersGuide/gmtb.sty)

doc/README.md

@@ -0,0 +1,11 @@
+# Building the documentation
+
+Similar to building the source code, building the documentation can be done by:
+
+```bash
+$ cmake -S<project_root_source_directory> -B<build_directory> -DCCPP_FRAMEWORK_BUILD_DOCUMENTATION=ON
+cd build_directory
+make doc
+```
+
+and the compiled documentation should be in `<build_directory>/doc/html`.

src/CMakeLists.txt

@@ -12,9 +12,10 @@ set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_I
 # Define the executable and what to link
 add_library(ccpp_framework STATIC ${SOURCES_F90})
 target_link_libraries(ccpp_framework PUBLIC MPI::MPI_Fortran)
-set_target_properties(ccpp_framework PROPERTIES VERSION ${PROJECT_VERSION}
+set_target_properties(ccpp_framework PROPERTIES
+                                      VERSION ${PROJECT_VERSION}
                                       SOVERSION ${PROJECT_VERSION_MAJOR}
-                                      LINK_FLAGS ${CMAKE_Fortran_FLAGS})
+                                      LINK_FLAGS "${CMAKE_Fortran_FLAGS}")
 
 #------------------------------------------------------------------------------
 # Installation