From e3c0d257e13641f59f9798d2bc13ef96e730c68f Mon Sep 17 00:00:00 2001 From: Viktor Yastrebov Date: Thu, 9 May 2024 10:55:59 +0300 Subject: [PATCH 01/25] development based clang transpiler integration --- .gitmodules | 3 + CMakeLists.txt | 23 + README.md | 9 + cmake/CPM_0.38.6.cmake | 1155 ++++++++++++++++++ deps/occa-transpiler | 1 + include/occa/dtype/builtins.hpp | 1 + src/dtype/builtins.cpp | 3 +- src/dtype/dtype.cpp | 1 + src/occa/internal/bin/occa.cpp | 302 +++-- src/occa/internal/core/launchedDevice.cpp | 162 ++- src/occa/internal/core/launchedDevice.hpp | 10 + src/occa/internal/modes/openmp/device.cpp | 62 + src/occa/internal/modes/openmp/device.hpp | 7 + src/occa/internal/modes/serial/device.cpp | 89 +- src/occa/internal/modes/serial/device.hpp | 8 + src/occa/internal/utils/transpiler_utils.cpp | 73 ++ src/occa/internal/utils/transpiler_utils.h | 17 + tests/src/internal/bin/occa.cpp | 4 +- 18 files changed, 1826 insertions(+), 104 deletions(-) create mode 100644 .gitmodules create mode 100644 cmake/CPM_0.38.6.cmake create mode 160000 deps/occa-transpiler create mode 100644 src/occa/internal/utils/transpiler_utils.cpp create mode 100644 src/occa/internal/utils/transpiler_utils.h diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..c68eb4cc2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/occa-transpiler"] + path = deps/occa-transpiler + url = https://github.com/libocca/occa-transpiler.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 83cd38601..41f678f77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(OCCA_ENABLE_DPCPP "Build with SYCL/DPCPP if available" ON) option(OCCA_ENABLE_TESTS "Build tests" OFF) option(OCCA_ENABLE_EXAMPLES "Build simple examples" OFF) option(OCCA_ENABLE_FORTRAN "Enable Fortran interface" OFF) +option(CLANG_BASED_TRANSPILER "Build with occa-transpiler dependecy" OFF) if(OCCA_ENABLE_FORTRAN) enable_language(Fortran) @@ -67,6 +68,20 @@ else() set(OCCA_OS "OCCA_WINDOWS_OS") endif() +# INFO: order is important, deps should not apply compiler flags +if (CLANG_BASED_TRANSPILER) + include(cmake/CPM_0.38.6.cmake) + include(cmake/GitSubmodules.cmake) + + init_submodules(${PROJECT_SOURCE_DIR}) + + CPMAddPackage( + NAME occa-transpiler + SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler + OPTIONS "TRANSPILER_TESTS OFF" + ) +endif() + include(SetCompilerFlags) include(CheckCXXCompilerFlag) @@ -113,6 +128,14 @@ target_include_directories(libocca PRIVATE $) target_compile_definitions(libocca PRIVATE -DUSE_CMAKE) +if (CLANG_BASED_TRANSPILER) + target_link_libraries(libocca + PRIVATE + occa-transpiler + ) + target_compile_definitions(libocca PRIVATE -DBUILD_WITH_OCCA_TRANSPILER) +endif() + #======================================= #---[ OpenMP ]-------------------------- diff --git a/README.md b/README.md index d309a6bbf..5f21f5e5d 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ Mission critical computational science and engineering applications from the pub - SYCL 2020 or later - OpenCL 2.0 or later - OpenMP 4.0 or later + - C++ support with clang based occa-transpiler ## Build, Test, Install @@ -67,6 +68,14 @@ $ cmake --install build --prefix install If dependencies are installed in a non-standard location, set the corresponding [environment variable](INSTALL.md#dependency-paths) to this path. +For building with C++ support/occa-transpiler: +```shell +$ mkdir build +$ cd build +$ cmake -DCMAKE_BUILD_TYPE=Release -DCLANG_BASED_TRANSPILER=ON .. +$ cmake --build . --parallel +$ cmake --install . --prefix install +``` ## Use diff --git a/cmake/CPM_0.38.6.cmake b/cmake/CPM_0.38.6.cmake new file mode 100644 index 000000000..7ffb16b60 --- /dev/null +++ b/cmake/CPM_0.38.6.cmake @@ -0,0 +1,1155 @@ +# CPM.cmake - CMake's missing package manager +# =========================================== +# See https://github.com/cpm-cmake/CPM.cmake for usage and update instructions. +# +# MIT License +# ----------- +#[[ + Copyright (c) 2019-2023 Lars Melchior and contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +]] + +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) + +# Initialize logging prefix +if(NOT CPM_INDENT) + set(CPM_INDENT + "CPM:" + CACHE INTERNAL "" + ) +endif() + +if(NOT COMMAND cpm_message) + function(cpm_message) + message(${ARGV}) + endfunction() +endif() + +set(CURRENT_CPM_VERSION 0.38.6) + +get_filename_component(CPM_CURRENT_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +if(CPM_DIRECTORY) + if(NOT CPM_DIRECTORY STREQUAL CPM_CURRENT_DIRECTORY) + if(CPM_VERSION VERSION_LESS CURRENT_CPM_VERSION) + message( + AUTHOR_WARNING + "${CPM_INDENT} \ +A dependency is using a more recent CPM version (${CURRENT_CPM_VERSION}) than the current project (${CPM_VERSION}). \ +It is recommended to upgrade CPM to the most recent version. \ +See https://github.com/cpm-cmake/CPM.cmake for more information." + ) + endif() + if(${CMAKE_VERSION} VERSION_LESS "3.17.0") + include(FetchContent) + endif() + return() + endif() + + get_property( + CPM_INITIALIZED GLOBAL "" + PROPERTY CPM_INITIALIZED + SET + ) + if(CPM_INITIALIZED) + return() + endif() +endif() + +if(CURRENT_CPM_VERSION MATCHES "development-version") + message( + WARNING "${CPM_INDENT} Your project is using an unstable development version of CPM.cmake. \ +Please update to a recent release if possible. \ +See https://github.com/cpm-cmake/CPM.cmake for details." + ) +endif() + +set_property(GLOBAL PROPERTY CPM_INITIALIZED true) + +macro(cpm_set_policies) + # the policy allows us to change options without caching + cmake_policy(SET CMP0077 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) + + # the policy allows us to change set(CACHE) without caching + if(POLICY CMP0126) + cmake_policy(SET CMP0126 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0126 NEW) + endif() + + # The policy uses the download time for timestamp, instead of the timestamp in the archive. This + # allows for proper rebuilds when a projects url changes + if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) + endif() +endmacro() +cpm_set_policies() + +option(CPM_USE_LOCAL_PACKAGES "Always try to use `find_package` to get dependencies" + $ENV{CPM_USE_LOCAL_PACKAGES} +) +option(CPM_LOCAL_PACKAGES_ONLY "Only use `find_package` to get dependencies" + $ENV{CPM_LOCAL_PACKAGES_ONLY} +) +option(CPM_DOWNLOAD_ALL "Always download dependencies from source" $ENV{CPM_DOWNLOAD_ALL}) +option(CPM_DONT_UPDATE_MODULE_PATH "Don't update the module path to allow using find_package" + $ENV{CPM_DONT_UPDATE_MODULE_PATH} +) +option(CPM_DONT_CREATE_PACKAGE_LOCK "Don't create a package lock file in the binary path" + $ENV{CPM_DONT_CREATE_PACKAGE_LOCK} +) +option(CPM_INCLUDE_ALL_IN_PACKAGE_LOCK + "Add all packages added through CPM.cmake to the package lock" + $ENV{CPM_INCLUDE_ALL_IN_PACKAGE_LOCK} +) +option(CPM_USE_NAMED_CACHE_DIRECTORIES + "Use additional directory of package name in cache on the most nested level." + $ENV{CPM_USE_NAMED_CACHE_DIRECTORIES} +) + +set(CPM_VERSION + ${CURRENT_CPM_VERSION} + CACHE INTERNAL "" +) +set(CPM_DIRECTORY + ${CPM_CURRENT_DIRECTORY} + CACHE INTERNAL "" +) +set(CPM_FILE + ${CMAKE_CURRENT_LIST_FILE} + CACHE INTERNAL "" +) +set(CPM_PACKAGES + "" + CACHE INTERNAL "" +) +set(CPM_DRY_RUN + OFF + CACHE INTERNAL "Don't download or configure dependencies (for testing)" +) + +if(DEFINED ENV{CPM_SOURCE_CACHE}) + set(CPM_SOURCE_CACHE_DEFAULT $ENV{CPM_SOURCE_CACHE}) +else() + set(CPM_SOURCE_CACHE_DEFAULT OFF) +endif() + +set(CPM_SOURCE_CACHE + ${CPM_SOURCE_CACHE_DEFAULT} + CACHE PATH "Directory to download CPM dependencies" +) + +if(NOT CPM_DONT_UPDATE_MODULE_PATH) + set(CPM_MODULE_PATH + "${CMAKE_BINARY_DIR}/CPM_modules" + CACHE INTERNAL "" + ) + # remove old modules + file(REMOVE_RECURSE ${CPM_MODULE_PATH}) + file(MAKE_DIRECTORY ${CPM_MODULE_PATH}) + # locally added CPM modules should override global packages + set(CMAKE_MODULE_PATH "${CPM_MODULE_PATH};${CMAKE_MODULE_PATH}") +endif() + +if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + set(CPM_PACKAGE_LOCK_FILE + "${CMAKE_BINARY_DIR}/cpm-package-lock.cmake" + CACHE INTERNAL "" + ) + file(WRITE ${CPM_PACKAGE_LOCK_FILE} + "# CPM Package Lock\n# This file should be committed to version control\n\n" + ) +endif() + +include(FetchContent) + +# Try to infer package name from git repository uri (path or url) +function(cpm_package_name_from_git_uri URI RESULT) + if("${URI}" MATCHES "([^/:]+)/?.git/?$") + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + else() + unset(${RESULT} PARENT_SCOPE) + endif() +endfunction() + +# Try to infer package name and version from a url +function(cpm_package_name_and_ver_from_url url outName outVer) + if(url MATCHES "[/\\?]([a-zA-Z0-9_\\.-]+)\\.(tar|tar\\.gz|tar\\.bz2|zip|ZIP)(\\?|/|$)") + # We matched an archive + set(filename "${CMAKE_MATCH_1}") + + if(filename MATCHES "([a-zA-Z0-9_\\.-]+)[_-]v?(([0-9]+\\.)*[0-9]+[a-zA-Z0-9]*)") + # We matched - (ie foo-1.2.3) + set(${outName} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + set(${outVer} + "${CMAKE_MATCH_2}" + PARENT_SCOPE + ) + elseif(filename MATCHES "(([0-9]+\\.)+[0-9]+[a-zA-Z0-9]*)") + # We couldn't find a name, but we found a version + # + # In many cases (which we don't handle here) the url would look something like + # `irrelevant/ACTUAL_PACKAGE_NAME/irrelevant/1.2.3.zip`. In such a case we can't possibly + # distinguish the package name from the irrelevant bits. Moreover if we try to match the + # package name from the filename, we'd get bogus at best. + unset(${outName} PARENT_SCOPE) + set(${outVer} + "${CMAKE_MATCH_1}" + PARENT_SCOPE + ) + else() + # Boldly assume that the file name is the package name. + # + # Yes, something like `irrelevant/ACTUAL_NAME/irrelevant/download.zip` will ruin our day, but + # such cases should be quite rare. No popular service does this... we think. + set(${outName} + "${filename}" + PARENT_SCOPE + ) + unset(${outVer} PARENT_SCOPE) + endif() + else() + # No ideas yet what to do with non-archives + unset(${outName} PARENT_SCOPE) + unset(${outVer} PARENT_SCOPE) + endif() +endfunction() + +function(cpm_find_package NAME VERSION) + string(REPLACE " " ";" EXTRA_ARGS "${ARGN}") + find_package(${NAME} ${VERSION} ${EXTRA_ARGS} QUIET) + if(${CPM_ARGS_NAME}_FOUND) + if(DEFINED ${CPM_ARGS_NAME}_VERSION) + set(VERSION ${${CPM_ARGS_NAME}_VERSION}) + endif() + cpm_message(STATUS "${CPM_INDENT} Using local package ${CPM_ARGS_NAME}@${VERSION}") + CPMRegisterPackage(${CPM_ARGS_NAME} "${VERSION}") + set(CPM_PACKAGE_FOUND + YES + PARENT_SCOPE + ) + else() + set(CPM_PACKAGE_FOUND + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Create a custom FindXXX.cmake module for a CPM package This prevents `find_package(NAME)` from +# finding the system library +function(cpm_create_module_file Name) + if(NOT CPM_DONT_UPDATE_MODULE_PATH) + # erase any previous modules + file(WRITE ${CPM_MODULE_PATH}/Find${Name}.cmake + "include(\"${CPM_FILE}\")\n${ARGN}\nset(${Name}_FOUND TRUE)" + ) + endif() +endfunction() + +# Find a package locally or fallback to CPMAddPackage +function(CPMFindPackage) + set(oneValueArgs NAME VERSION GIT_TAG FIND_PACKAGE_ARGUMENTS) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "" ${ARGN}) + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + set(downloadPackage ${CPM_DOWNLOAD_ALL}) + if(DEFINED CPM_DOWNLOAD_${CPM_ARGS_NAME}) + set(downloadPackage ${CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + elseif(DEFINED ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + set(downloadPackage $ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) + endif() + if(downloadPackage) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + if(CPM_PACKAGE_ALREADY_ADDED) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(NOT CPM_PACKAGE_FOUND) + CPMAddPackage(${ARGN}) + cpm_export_variables(${CPM_ARGS_NAME}) + endif() + +endfunction() + +# checks if a package has been added before +function(cpm_check_if_package_already_added CPM_ARGS_NAME CPM_ARGS_VERSION) + if("${CPM_ARGS_NAME}" IN_LIST CPM_PACKAGES) + CPMGetPackageVersion(${CPM_ARGS_NAME} CPM_PACKAGE_VERSION) + if("${CPM_PACKAGE_VERSION}" VERSION_LESS "${CPM_ARGS_VERSION}") + message( + WARNING + "${CPM_INDENT} Requires a newer version of ${CPM_ARGS_NAME} (${CPM_ARGS_VERSION}) than currently included (${CPM_PACKAGE_VERSION})." + ) + endif() + cpm_get_fetch_properties(${CPM_ARGS_NAME}) + set(${CPM_ARGS_NAME}_ADDED NO) + set(CPM_PACKAGE_ALREADY_ADDED + YES + PARENT_SCOPE + ) + cpm_export_variables(${CPM_ARGS_NAME}) + else() + set(CPM_PACKAGE_ALREADY_ADDED + NO + PARENT_SCOPE + ) + endif() +endfunction() + +# Parse the argument of CPMAddPackage in case a single one was provided and convert it to a list of +# arguments which can then be parsed idiomatically. For example gh:foo/bar@1.2.3 will be converted +# to: GITHUB_REPOSITORY;foo/bar;VERSION;1.2.3 +function(cpm_parse_add_package_single_arg arg outArgs) + # Look for a scheme + if("${arg}" MATCHES "^([a-zA-Z]+):(.+)$") + string(TOLOWER "${CMAKE_MATCH_1}" scheme) + set(uri "${CMAKE_MATCH_2}") + + # Check for CPM-specific schemes + if(scheme STREQUAL "gh") + set(out "GITHUB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "gl") + set(out "GITLAB_REPOSITORY;${uri}") + set(packageType "git") + elseif(scheme STREQUAL "bb") + set(out "BITBUCKET_REPOSITORY;${uri}") + set(packageType "git") + # A CPM-specific scheme was not found. Looks like this is a generic URL so try to determine + # type + elseif(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Fall back to a URL + set(out "URL;${arg}") + set(packageType "archive") + + # We could also check for SVN since FetchContent supports it, but SVN is so rare these days. + # We just won't bother with the additional complexity it will induce in this function. SVN is + # done by multi-arg + endif() + else() + if(arg MATCHES ".git/?(@|#|$)") + set(out "GIT_REPOSITORY;${arg}") + set(packageType "git") + else() + # Give up + message(FATAL_ERROR "${CPM_INDENT} Can't determine package type of '${arg}'") + endif() + endif() + + # For all packages we interpret @... as version. Only replace the last occurrence. Thus URIs + # containing '@' can be used + string(REGEX REPLACE "@([^@]+)$" ";VERSION;\\1" out "${out}") + + # Parse the rest according to package type + if(packageType STREQUAL "git") + # For git repos we interpret #... as a tag or branch or commit hash + string(REGEX REPLACE "#([^#]+)$" ";GIT_TAG;\\1" out "${out}") + elseif(packageType STREQUAL "archive") + # For archives we interpret #... as a URL hash. + string(REGEX REPLACE "#([^#]+)$" ";URL_HASH;\\1" out "${out}") + # We don't try to parse the version if it's not provided explicitly. cpm_get_version_from_url + # should do this at a later point + else() + # We should never get here. This is an assertion and hitting it means there's a bug in the code + # above. A packageType was set, but not handled by this if-else. + message(FATAL_ERROR "${CPM_INDENT} Unsupported package type '${packageType}' of '${arg}'") + endif() + + set(${outArgs} + ${out} + PARENT_SCOPE + ) +endfunction() + +# Check that the working directory for a git repo is clean +function(cpm_check_git_working_dir_is_clean repoPath gitTag isClean) + + find_package(Git REQUIRED) + + if(NOT GIT_EXECUTABLE) + # No git executable, assume directory is clean + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + # check for uncommitted changes + execute_process( + COMMAND ${GIT_EXECUTABLE} status --porcelain + RESULT_VARIABLE resultGitStatus + OUTPUT_VARIABLE repoStatus + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET + WORKING_DIRECTORY ${repoPath} + ) + if(resultGitStatus) + # not supposed to happen, assume clean anyway + message(WARNING "${CPM_INDENT} Calling git status on folder ${repoPath} failed") + set(${isClean} + TRUE + PARENT_SCOPE + ) + return() + endif() + + if(NOT "${repoStatus}" STREQUAL "") + set(${isClean} + FALSE + PARENT_SCOPE + ) + return() + endif() + + # check for committed changes + execute_process( + COMMAND ${GIT_EXECUTABLE} diff -s --exit-code ${gitTag} + RESULT_VARIABLE resultGitDiff + OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_QUIET + WORKING_DIRECTORY ${repoPath} + ) + + if(${resultGitDiff} EQUAL 0) + set(${isClean} + TRUE + PARENT_SCOPE + ) + else() + set(${isClean} + FALSE + PARENT_SCOPE + ) + endif() + +endfunction() + +# method to overwrite internal FetchContent properties, to allow using CPM.cmake to overload +# FetchContent calls. As these are internal cmake properties, this method should be used carefully +# and may need modification in future CMake versions. Source: +# https://github.com/Kitware/CMake/blob/dc3d0b5a0a7d26d43d6cfeb511e224533b5d188f/Modules/FetchContent.cmake#L1152 +function(cpm_override_fetchcontent contentName) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "SOURCE_DIR;BINARY_DIR" "") + if(NOT "${arg_UNPARSED_ARGUMENTS}" STREQUAL "") + message(FATAL_ERROR "${CPM_INDENT} Unsupported arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + + string(TOLOWER ${contentName} contentNameLower) + set(prefix "_FetchContent_${contentNameLower}") + + set(propertyName "${prefix}_sourceDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_SOURCE_DIR}") + + set(propertyName "${prefix}_binaryDir") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} "${arg_BINARY_DIR}") + + set(propertyName "${prefix}_populated") + define_property( + GLOBAL + PROPERTY ${propertyName} + BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" + FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" + ) + set_property(GLOBAL PROPERTY ${propertyName} TRUE) +endfunction() + +# Download and add a package from source +function(CPMAddPackage) + cpm_set_policies() + + list(LENGTH ARGN argnLength) + if(argnLength EQUAL 1) + cpm_parse_add_package_single_arg("${ARGN}" ARGN) + + # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM + set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;") + endif() + + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + ) + + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) + + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + + # Set default values for arguments + + if(NOT DEFINED CPM_ARGS_VERSION) + if(DEFINED CPM_ARGS_GIT_TAG) + cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) + endif() + endif() + + if(CPM_ARGS_DOWNLOAD_ONLY) + set(DOWNLOAD_ONLY ${CPM_ARGS_DOWNLOAD_ONLY}) + else() + set(DOWNLOAD_ONLY NO) + endif() + + if(DEFINED CPM_ARGS_GITHUB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://github.com/${CPM_ARGS_GITHUB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_GITLAB_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://gitlab.com/${CPM_ARGS_GITLAB_REPOSITORY}.git") + elseif(DEFINED CPM_ARGS_BITBUCKET_REPOSITORY) + set(CPM_ARGS_GIT_REPOSITORY "https://bitbucket.org/${CPM_ARGS_BITBUCKET_REPOSITORY}.git") + endif() + + if(DEFINED CPM_ARGS_GIT_REPOSITORY) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_REPOSITORY ${CPM_ARGS_GIT_REPOSITORY}) + if(NOT DEFINED CPM_ARGS_GIT_TAG) + set(CPM_ARGS_GIT_TAG v${CPM_ARGS_VERSION}) + endif() + + # If a name wasn't provided, try to infer it from the git repo + if(NOT DEFINED CPM_ARGS_NAME) + cpm_package_name_from_git_uri(${CPM_ARGS_GIT_REPOSITORY} CPM_ARGS_NAME) + endif() + endif() + + set(CPM_SKIP_FETCH FALSE) + + if(DEFINED CPM_ARGS_GIT_TAG) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_TAG ${CPM_ARGS_GIT_TAG}) + # If GIT_SHALLOW is explicitly specified, honor the value. + if(DEFINED CPM_ARGS_GIT_SHALLOW) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW ${CPM_ARGS_GIT_SHALLOW}) + endif() + endif() + + if(DEFINED CPM_ARGS_URL) + # If a name or version aren't provided, try to infer them from the URL + list(GET CPM_ARGS_URL 0 firstUrl) + cpm_package_name_and_ver_from_url(${firstUrl} nameFromUrl verFromUrl) + # If we fail to obtain name and version from the first URL, we could try other URLs if any. + # However multiple URLs are expected to be quite rare, so for now we won't bother. + + # If the caller provided their own name and version, they trump the inferred ones. + if(NOT DEFINED CPM_ARGS_NAME) + set(CPM_ARGS_NAME ${nameFromUrl}) + endif() + if(NOT DEFINED CPM_ARGS_VERSION) + set(CPM_ARGS_VERSION ${verFromUrl}) + endif() + + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS URL "${CPM_ARGS_URL}") + endif() + + # Check for required arguments + + if(NOT DEFINED CPM_ARGS_NAME) + message( + FATAL_ERROR + "${CPM_INDENT} 'NAME' was not provided and couldn't be automatically inferred for package added with arguments: '${ARGN}'" + ) + endif() + + # Check if package has been added before + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + if(CPM_PACKAGE_ALREADY_ADDED) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for manual overrides + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_${CPM_ARGS_NAME}_SOURCE}" STREQUAL "") + set(PACKAGE_SOURCE ${CPM_${CPM_ARGS_NAME}_SOURCE}) + set(CPM_${CPM_ARGS_NAME}_SOURCE "") + CPMAddPackage( + NAME "${CPM_ARGS_NAME}" + SOURCE_DIR "${PACKAGE_SOURCE}" + EXCLUDE_FROM_ALL "${CPM_ARGS_EXCLUDE_FROM_ALL}" + SYSTEM "${CPM_ARGS_SYSTEM}" + OPTIONS "${CPM_ARGS_OPTIONS}" + SOURCE_SUBDIR "${CPM_ARGS_SOURCE_SUBDIR}" + DOWNLOAD_ONLY "${DOWNLOAD_ONLY}" + FORCE True + ) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + # Check for available declaration + if(NOT CPM_ARGS_FORCE AND NOT "${CPM_DECLARATION_${CPM_ARGS_NAME}}" STREQUAL "") + set(declaration ${CPM_DECLARATION_${CPM_ARGS_NAME}}) + set(CPM_DECLARATION_${CPM_ARGS_NAME} "") + CPMAddPackage(${declaration}) + cpm_export_variables(${CPM_ARGS_NAME}) + # checking again to ensure version and option compatibility + cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") + return() + endif() + + if(NOT CPM_ARGS_FORCE) + if(CPM_USE_LOCAL_PACKAGES OR CPM_LOCAL_PACKAGES_ONLY) + cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) + + if(CPM_PACKAGE_FOUND) + cpm_export_variables(${CPM_ARGS_NAME}) + return() + endif() + + if(CPM_LOCAL_PACKAGES_ONLY) + message( + SEND_ERROR + "${CPM_INDENT} ${CPM_ARGS_NAME} not found via find_package(${CPM_ARGS_NAME} ${CPM_ARGS_VERSION})" + ) + endif() + endif() + endif() + + CPMRegisterPackage("${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}") + + if(DEFINED CPM_ARGS_GIT_TAG) + set(PACKAGE_INFO "${CPM_ARGS_GIT_TAG}") + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + set(PACKAGE_INFO "${CPM_ARGS_SOURCE_DIR}") + else() + set(PACKAGE_INFO "${CPM_ARGS_VERSION}") + endif() + + if(DEFINED FETCHCONTENT_BASE_DIR) + # respect user's FETCHCONTENT_BASE_DIR if set + set(CPM_FETCHCONTENT_BASE_DIR ${FETCHCONTENT_BASE_DIR}) + else() + set(CPM_FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_deps) + endif() + + if(DEFINED CPM_ARGS_DOWNLOAD_COMMAND) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS DOWNLOAD_COMMAND ${CPM_ARGS_DOWNLOAD_COMMAND}) + elseif(DEFINED CPM_ARGS_SOURCE_DIR) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${CPM_ARGS_SOURCE_DIR}) + if(NOT IS_ABSOLUTE ${CPM_ARGS_SOURCE_DIR}) + # Expand `CPM_ARGS_SOURCE_DIR` relative path. This is important because EXISTS doesn't work + # for relative paths. + get_filename_component( + source_directory ${CPM_ARGS_SOURCE_DIR} REALPATH BASE_DIR ${CMAKE_CURRENT_BINARY_DIR} + ) + else() + set(source_directory ${CPM_ARGS_SOURCE_DIR}) + endif() + if(NOT EXISTS ${source_directory}) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild") + endif() + elseif(CPM_SOURCE_CACHE AND NOT CPM_ARGS_NO_CACHE) + string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) + set(origin_parameters ${CPM_ARGS_UNPARSED_ARGUMENTS}) + list(SORT origin_parameters) + if(CPM_USE_NAMED_CACHE_DIRECTORIES) + string(SHA1 origin_hash "${origin_parameters};NEW_CACHE_STRUCTURE_TAG") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}/${CPM_ARGS_NAME}) + else() + string(SHA1 origin_hash "${origin_parameters}") + set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}) + endif() + # Expand `download_directory` relative path. This is important because EXISTS doesn't work for + # relative paths. + get_filename_component(download_directory ${download_directory} ABSOLUTE) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory}) + + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock) + endif() + + if(EXISTS ${download_directory}) + if(CPM_SOURCE_CACHE) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} "${download_directory}" + "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" + ) + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + + if(DEFINED CPM_ARGS_GIT_TAG AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS)) + # warn if cache has been changed since checkout + cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN) + if(NOT ${IS_CLEAN}) + message( + WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty" + ) + endif() + endif() + + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + set(PACKAGE_INFO "${PACKAGE_INFO} at ${download_directory}") + + # As the source dir is already cached/populated, we override the call to FetchContent. + set(CPM_SKIP_FETCH TRUE) + cpm_override_fetchcontent( + "${lower_case_name}" SOURCE_DIR "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + BINARY_DIR "${${CPM_ARGS_NAME}_BINARY_DIR}" + ) + + else() + # Enable shallow clone when GIT_TAG is not a commit hash. Our guess may not be accurate, but + # it should guarantee no commit hash get mis-detected. + if(NOT DEFINED CPM_ARGS_GIT_SHALLOW) + cpm_is_git_tag_commit_hash("${CPM_ARGS_GIT_TAG}" IS_HASH) + if(NOT ${IS_HASH}) + list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW TRUE) + endif() + endif() + + # remove timestamps so CMake will re-download the dependency + file(REMOVE_RECURSE ${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild) + set(PACKAGE_INFO "${PACKAGE_INFO} to ${download_directory}") + endif() + endif() + + cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")") + + if(CPM_PACKAGE_LOCK_ENABLED) + if((CPM_ARGS_VERSION AND NOT CPM_ARGS_SOURCE_DIR) OR CPM_INCLUDE_ALL_IN_PACKAGE_LOCK) + cpm_add_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + elseif(CPM_ARGS_SOURCE_DIR) + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "local directory") + else() + cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") + endif() + endif() + + cpm_message( + STATUS "${CPM_INDENT} Adding package ${CPM_ARGS_NAME}@${CPM_ARGS_VERSION} (${PACKAGE_INFO})" + ) + + if(NOT CPM_SKIP_FETCH) + cpm_declare_fetch( + "${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}" "${PACKAGE_INFO}" "${CPM_ARGS_UNPARSED_ARGUMENTS}" + ) + cpm_fetch_package("${CPM_ARGS_NAME}" populated) + if(CPM_SOURCE_CACHE AND download_directory) + file(LOCK ${download_directory}/../cmake.lock RELEASE) + endif() + if(${populated}) + cpm_add_subdirectory( + "${CPM_ARGS_NAME}" + "${DOWNLOAD_ONLY}" + "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" + "${${CPM_ARGS_NAME}_BINARY_DIR}" + "${CPM_ARGS_EXCLUDE_FROM_ALL}" + "${CPM_ARGS_SYSTEM}" + "${CPM_ARGS_OPTIONS}" + ) + endif() + cpm_get_fetch_properties("${CPM_ARGS_NAME}") + endif() + + set(${CPM_ARGS_NAME}_ADDED YES) + cpm_export_variables("${CPM_ARGS_NAME}") +endfunction() + +# Fetch a previously declared package +macro(CPMGetPackage Name) + if(DEFINED "CPM_DECLARATION_${Name}") + CPMAddPackage(NAME ${Name}) + else() + message(SEND_ERROR "${CPM_INDENT} Cannot retrieve package ${Name}: no declaration available") + endif() +endmacro() + +# export variables available to the caller to the parent scope expects ${CPM_ARGS_NAME} to be set +macro(cpm_export_variables name) + set(${name}_SOURCE_DIR + "${${name}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${name}_BINARY_DIR + "${${name}_BINARY_DIR}" + PARENT_SCOPE + ) + set(${name}_ADDED + "${${name}_ADDED}" + PARENT_SCOPE + ) + set(CPM_LAST_PACKAGE_NAME + "${name}" + PARENT_SCOPE + ) +endmacro() + +# declares a package, so that any call to CPMAddPackage for the package name will use these +# arguments instead. Previous declarations will not be overridden. +macro(CPMDeclarePackage Name) + if(NOT DEFINED "CPM_DECLARATION_${Name}") + set("CPM_DECLARATION_${Name}" "${ARGN}") + endif() +endmacro() + +function(cpm_add_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN false ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} "# ${Name}\nCPMDeclarePackage(${Name}\n${PRETTY_ARGN})\n") + endif() +endfunction() + +function(cpm_add_comment_to_package_lock Name) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + cpm_prettify_package_arguments(PRETTY_ARGN true ${ARGN}) + file(APPEND ${CPM_PACKAGE_LOCK_FILE} + "# ${Name} (unversioned)\n# CPMDeclarePackage(${Name}\n${PRETTY_ARGN}#)\n" + ) + endif() +endfunction() + +# includes the package lock file if it exists and creates a target `cpm-update-package-lock` to +# update it +macro(CPMUsePackageLock file) + if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) + get_filename_component(CPM_ABSOLUTE_PACKAGE_LOCK_PATH ${file} ABSOLUTE) + if(EXISTS ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + include(${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) + endif() + if(NOT TARGET cpm-update-package-lock) + add_custom_target( + cpm-update-package-lock COMMAND ${CMAKE_COMMAND} -E copy ${CPM_PACKAGE_LOCK_FILE} + ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH} + ) + endif() + set(CPM_PACKAGE_LOCK_ENABLED true) + endif() +endmacro() + +# registers a package that has been added to CPM +function(CPMRegisterPackage PACKAGE VERSION) + list(APPEND CPM_PACKAGES ${PACKAGE}) + set(CPM_PACKAGES + ${CPM_PACKAGES} + CACHE INTERNAL "" + ) + set("CPM_PACKAGE_${PACKAGE}_VERSION" + ${VERSION} + CACHE INTERNAL "" + ) +endfunction() + +# retrieve the current version of the package to ${OUTPUT} +function(CPMGetPackageVersion PACKAGE OUTPUT) + set(${OUTPUT} + "${CPM_PACKAGE_${PACKAGE}_VERSION}" + PARENT_SCOPE + ) +endfunction() + +# declares a package in FetchContent_Declare +function(cpm_declare_fetch PACKAGE VERSION INFO) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package not declared (dry run)") + return() + endif() + + FetchContent_Declare(${PACKAGE} ${ARGN}) +endfunction() + +# returns properties for a package previously defined by cpm_declare_fetch +function(cpm_get_fetch_properties PACKAGE) + if(${CPM_DRY_RUN}) + return() + endif() + + set(${PACKAGE}_SOURCE_DIR + "${CPM_PACKAGE_${PACKAGE}_SOURCE_DIR}" + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + "${CPM_PACKAGE_${PACKAGE}_BINARY_DIR}" + PARENT_SCOPE + ) +endfunction() + +function(cpm_store_fetch_properties PACKAGE source_dir binary_dir) + if(${CPM_DRY_RUN}) + return() + endif() + + set(CPM_PACKAGE_${PACKAGE}_SOURCE_DIR + "${source_dir}" + CACHE INTERNAL "" + ) + set(CPM_PACKAGE_${PACKAGE}_BINARY_DIR + "${binary_dir}" + CACHE INTERNAL "" + ) +endfunction() + +# adds a package as a subdirectory if viable, according to provided options +function( + cpm_add_subdirectory + PACKAGE + DOWNLOAD_ONLY + SOURCE_DIR + BINARY_DIR + EXCLUDE + SYSTEM + OPTIONS +) + + if(NOT DOWNLOAD_ONLY AND EXISTS ${SOURCE_DIR}/CMakeLists.txt) + set(addSubdirectoryExtraArgs "") + if(EXCLUDE) + list(APPEND addSubdirectoryExtraArgs EXCLUDE_FROM_ALL) + endif() + if("${SYSTEM}" AND "${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.25") + # https://cmake.org/cmake/help/latest/prop_dir/SYSTEM.html#prop_dir:SYSTEM + list(APPEND addSubdirectoryExtraArgs SYSTEM) + endif() + if(OPTIONS) + foreach(OPTION ${OPTIONS}) + cpm_parse_option("${OPTION}") + set(${OPTION_KEY} "${OPTION_VALUE}") + endforeach() + endif() + set(CPM_OLD_INDENT "${CPM_INDENT}") + set(CPM_INDENT "${CPM_INDENT} ${PACKAGE}:") + add_subdirectory(${SOURCE_DIR} ${BINARY_DIR} ${addSubdirectoryExtraArgs}) + set(CPM_INDENT "${CPM_OLD_INDENT}") + endif() +endfunction() + +# downloads a previously declared package via FetchContent and exports the variables +# `${PACKAGE}_SOURCE_DIR` and `${PACKAGE}_BINARY_DIR` to the parent scope +function(cpm_fetch_package PACKAGE populated) + set(${populated} + FALSE + PARENT_SCOPE + ) + if(${CPM_DRY_RUN}) + cpm_message(STATUS "${CPM_INDENT} Package ${PACKAGE} not fetched (dry run)") + return() + endif() + + FetchContent_GetProperties(${PACKAGE}) + + string(TOLOWER "${PACKAGE}" lower_case_name) + + if(NOT ${lower_case_name}_POPULATED) + FetchContent_Populate(${PACKAGE}) + set(${populated} + TRUE + PARENT_SCOPE + ) + endif() + + cpm_store_fetch_properties( + ${CPM_ARGS_NAME} ${${lower_case_name}_SOURCE_DIR} ${${lower_case_name}_BINARY_DIR} + ) + + set(${PACKAGE}_SOURCE_DIR + ${${lower_case_name}_SOURCE_DIR} + PARENT_SCOPE + ) + set(${PACKAGE}_BINARY_DIR + ${${lower_case_name}_BINARY_DIR} + PARENT_SCOPE + ) +endfunction() + +# splits a package option +function(cpm_parse_option OPTION) + string(REGEX MATCH "^[^ ]+" OPTION_KEY "${OPTION}") + string(LENGTH "${OPTION}" OPTION_LENGTH) + string(LENGTH "${OPTION_KEY}" OPTION_KEY_LENGTH) + if(OPTION_KEY_LENGTH STREQUAL OPTION_LENGTH) + # no value for key provided, assume user wants to set option to "ON" + set(OPTION_VALUE "ON") + else() + math(EXPR OPTION_KEY_LENGTH "${OPTION_KEY_LENGTH}+1") + string(SUBSTRING "${OPTION}" "${OPTION_KEY_LENGTH}" "-1" OPTION_VALUE) + endif() + set(OPTION_KEY + "${OPTION_KEY}" + PARENT_SCOPE + ) + set(OPTION_VALUE + "${OPTION_VALUE}" + PARENT_SCOPE + ) +endfunction() + +# guesses the package version from a git tag +function(cpm_get_version_from_git_tag GIT_TAG RESULT) + string(LENGTH ${GIT_TAG} length) + if(length EQUAL 40) + # GIT_TAG is probably a git hash + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + string(REGEX MATCH "v?([0123456789.]*).*" _ ${GIT_TAG}) + set(${RESULT} + ${CMAKE_MATCH_1} + PARENT_SCOPE + ) + endif() +endfunction() + +# guesses if the git tag is a commit hash or an actual tag or a branch name. +function(cpm_is_git_tag_commit_hash GIT_TAG RESULT) + string(LENGTH "${GIT_TAG}" length) + # full hash has 40 characters, and short hash has at least 7 characters. + if(length LESS 7 OR length GREATER 40) + set(${RESULT} + 0 + PARENT_SCOPE + ) + else() + if(${GIT_TAG} MATCHES "^[a-fA-F0-9]+$") + set(${RESULT} + 1 + PARENT_SCOPE + ) + else() + set(${RESULT} + 0 + PARENT_SCOPE + ) + endif() + endif() +endfunction() + +function(cpm_prettify_package_arguments OUT_VAR IS_IN_COMMENT) + set(oneValueArgs + NAME + FORCE + VERSION + GIT_TAG + DOWNLOAD_ONLY + GITHUB_REPOSITORY + GITLAB_REPOSITORY + BITBUCKET_REPOSITORY + GIT_REPOSITORY + SOURCE_DIR + FIND_PACKAGE_ARGUMENTS + NO_CACHE + SYSTEM + GIT_SHALLOW + EXCLUDE_FROM_ALL + SOURCE_SUBDIR + ) + set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) + cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + foreach(oneArgName ${oneValueArgs}) + if(DEFINED CPM_ARGS_${oneArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + if(${oneArgName} STREQUAL "SOURCE_DIR") + string(REPLACE ${CMAKE_SOURCE_DIR} "\${CMAKE_SOURCE_DIR}" CPM_ARGS_${oneArgName} + ${CPM_ARGS_${oneArgName}} + ) + endif() + string(APPEND PRETTY_OUT_VAR " ${oneArgName} ${CPM_ARGS_${oneArgName}}\n") + endif() + endforeach() + foreach(multiArgName ${multiValueArgs}) + if(DEFINED CPM_ARGS_${multiArgName}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ${multiArgName}\n") + foreach(singleOption ${CPM_ARGS_${multiArgName}}) + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " \"${singleOption}\"\n") + endforeach() + endif() + endforeach() + + if(NOT "${CPM_ARGS_UNPARSED_ARGUMENTS}" STREQUAL "") + if(${IS_IN_COMMENT}) + string(APPEND PRETTY_OUT_VAR "#") + endif() + string(APPEND PRETTY_OUT_VAR " ") + foreach(CPM_ARGS_UNPARSED_ARGUMENT ${CPM_ARGS_UNPARSED_ARGUMENTS}) + string(APPEND PRETTY_OUT_VAR " ${CPM_ARGS_UNPARSED_ARGUMENT}") + endforeach() + string(APPEND PRETTY_OUT_VAR "\n") + endif() + + set(${OUT_VAR} + ${PRETTY_OUT_VAR} + PARENT_SCOPE + ) + +endfunction() diff --git a/deps/occa-transpiler b/deps/occa-transpiler new file mode 160000 index 000000000..9015b78fe --- /dev/null +++ b/deps/occa-transpiler @@ -0,0 +1 @@ +Subproject commit 9015b78fe98e7baa26ea126127bc8fa5fedfc704 diff --git a/include/occa/dtype/builtins.hpp b/include/occa/dtype/builtins.hpp index 3de51e5aa..6288b26dd 100644 --- a/include/occa/dtype/builtins.hpp +++ b/include/occa/dtype/builtins.hpp @@ -22,6 +22,7 @@ namespace occa { extern const dtype_t short_; extern const dtype_t int_; extern const dtype_t long_; + extern const dtype_t ulong_; extern const dtype_t float_; extern const dtype_t double_; diff --git a/src/dtype/builtins.cpp b/src/dtype/builtins.cpp index 48174b4b8..5281f044d 100644 --- a/src/dtype/builtins.cpp +++ b/src/dtype/builtins.cpp @@ -12,6 +12,7 @@ namespace occa { const dtype_t short_("short", sizeof(short), true); const dtype_t int_("int", sizeof(int), true); const dtype_t long_("long", sizeof(long), true); + const dtype_t ulong_("unsigned long", sizeof(unsigned long), true); const dtype_t float_("float", sizeof(float), true); const dtype_t double_("double", sizeof(double), true); @@ -111,7 +112,7 @@ namespace occa { } template <> dtype_t get() { - return long_; + return ulong_; } template <> dtype_t get() { diff --git a/src/dtype/dtype.cpp b/src/dtype/dtype.cpp index b742c4c00..0a061e567 100644 --- a/src/dtype/dtype.cpp +++ b/src/dtype/dtype.cpp @@ -413,6 +413,7 @@ namespace occa { dtypeMap["long"] = &dtype::long_; dtypeMap["float"] = &dtype::float_; dtypeMap["double"] = &dtype::double_; + dtypeMap["unsigned long"] = &dtype::ulong_; // Sized primitives dtypeMap["int8"] = dtype::get().ref; diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index 15bd42437..cf50c290b 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include @@ -13,6 +15,16 @@ #include #include +#ifdef BUILD_WITH_OCCA_TRANSPILER +#include +#include "oklt/pipeline/normalizer_and_transpiler.h" +#include "oklt/util/io_helper.h" +#include "oklt/core/error.h" +#endif + +#include + + namespace occa { namespace bin { std::string envEcho(const std::string &arg) { @@ -123,6 +135,208 @@ namespace occa { return true; } +#ifdef BUILD_WITH_OCCA_TRANSPILER + namespace v3 { + + bool runTranslate(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + static const std::map targetBackends = + { + {"openmp", oklt::TargetBackend::OPENMP}, + {"cuda", oklt::TargetBackend::CUDA}, + {"hip", oklt::TargetBackend::HIP}, + {"dpcpp", oklt::TargetBackend::DPCPP}, + }; + + const bool printLauncher = options["launcher"]; + const std::string filename = arguments[0]; + + if (!io::exists(filename)) { + printError("File [" + filename + "] doesn't exist" ); + ::exit(1); + } + + auto transpiler = targetBackends.find(mode); + if(transpiler == targetBackends.end()) { + printError("Unable to translate for mode [" + originalMode + "]"); + ::exit(1); + } + + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + auto hash = transpiler::getKernelHash(kernelProps); + + std::filesystem::path sourcePath = io::expandFilename(filename); + auto sourceCode = oklt::util::readFileAsStr(sourcePath); + if(!sourceCode) { + printError("Can't open file: " + sourcePath.string()); + ::exit(sourceCode.error()); + } + oklt::UserInput input { + .backend = transpiler->second, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = sourcePath, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = std::move(hash) + }; + auto result = normalizeAndTranspile(std::move(input)); + + if(!result) { + std::stringstream ss; + for(const auto &err: result.error()) { + ss << err.desc << std::endl; + } + printError(ss.str()); + ::exit(1); + } + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; + } + + auto userOutput = result.value(); + bool hasLauncher = transpiler->second == oklt::TargetBackend::CUDA || + transpiler->second == oklt::TargetBackend::HIP || + transpiler->second == oklt::TargetBackend::DPCPP; + if(printLauncher && hasLauncher) { + io::stdout << userOutput.launcher.source; + } else { + io::stdout << userOutput.kernel.source; + } + + return true; + } + } +#endif + + namespace v2 { + bool runTranslate(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + using ParserBuildFunc = std::function(const json ¶ms)>; + static const std::map originalParserBackends = + { + {"", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"serial", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"openmp", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"cuda", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"hip", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"opencl", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"metal", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"dpcpp", [](const json ¶ms) { + return std::make_unique(params); + }} + }; + + const bool printLauncher = options["launcher"]; + const std::string filename = arguments[0]; + + if (!io::exists(filename)) { + printError("File [" + filename + "] doesn't exist" ); + ::exit(1); + } + + auto parserIt = originalParserBackends.find(mode); + if(parserIt == originalParserBackends.end()) { + printError("Unable to translate for mode [" + originalMode + "]"); + ::exit(1); + } + + std::unique_ptr parser = parserIt->second(kernelProps); + parser->parseFile(filename); + + bool success = parser->succeeded(); + if (!success) { + ::exit(1); + } + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; + } + + if (printLauncher && ((mode == "cuda") + || (mode == "hip") + || (mode == "opencl") + || (mode == "dpcpp") + || (mode == "metal"))) { + lang::parser_t *launcherParser = &(((occa::lang::okl::withLauncher*) parser.get())->launcherParser); + io::stdout << launcherParser->toString(); + } else { + io::stdout << parser->toString(); + } + return true; + } + } + + int getTranspilerVersion(const json &options) { + json jsonTranspileVersion = options["transpiler-version"]; + int transpilerVersion = 2; + //INFO: have no idea why json here has array type + if(jsonTranspileVersion.isArray()) { + json elem = jsonTranspileVersion.asArray()[0]; + if(elem.isString()) { + try { + transpilerVersion = std::stoi(elem.string()); + } catch(const std::exception &) + {} + } + } + return transpilerVersion; + } + bool runTranslate(const json &args) { const json &options = args["options"]; const json &arguments = args["arguments"]; @@ -130,85 +344,20 @@ namespace occa { const std::string originalMode = options["mode"]; const std::string mode = lowercase(originalMode); - const bool printLauncher = options["launcher"]; - const std::string filename = arguments[0]; - - if (!io::exists(filename)) { - printError("File [" + filename + "] doesn't exist" ); - ::exit(1); - } - json kernelProps = getOptionProperties(options["kernel-props"]); kernelProps["mode"] = mode; kernelProps["defines"].asObject() += getOptionDefines(options["define"]); kernelProps["okl/include_paths"] = options["include-path"]; - lang::parser_t *parser = NULL; - lang::parser_t *launcherParser = NULL; - if (mode == "" || mode == "serial") { - parser = new lang::okl::serialParser(kernelProps); - } else if (mode == "openmp") { - parser = new lang::okl::openmpParser(kernelProps); - } else if (mode == "cuda") { - parser = new lang::okl::cudaParser(kernelProps); - } else if (mode == "hip") { - parser = new lang::okl::hipParser(kernelProps); - } else if (mode == "opencl") { - parser = new lang::okl::openclParser(kernelProps); - } else if (mode == "metal") { - parser = new lang::okl::metalParser(kernelProps); - } else if (mode == "dpcpp") { - parser = new lang::okl::dpcppParser(kernelProps); +#ifdef BUILD_WITH_OCCA_TRANSPILER + int transpilerVersion = getTranspilerVersion(options); + if(transpilerVersion > 2) { + return v3::runTranslate(options, arguments, kernelProps, originalMode, mode); } - - if (!parser) { - printError("Unable to translate for mode [" + originalMode + "]"); - ::exit(1); - } - - parser->parseFile(filename); - - bool success = parser->succeeded(); - if (!success) { - delete parser; - ::exit(1); - } - - if (options["verbose"]) { - json translationInfo; - // Filename - translationInfo["translate_info/filename"] = io::expandFilename(filename); - // Date information - translationInfo["translate_info/date"] = sys::date(); - translationInfo["translate_info/human_date"] = sys::humanDate(); - // Version information - translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; - translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; - // Kernel properties - translationInfo["kernel_properties"] = kernelProps; - - io::stdout - << "/* Translation Info:\n" - << translationInfo - << "*/\n"; - } - - if (printLauncher && ((mode == "cuda") - || (mode == "hip") - || (mode == "opencl") - || (mode == "dpcpp") - || (mode == "metal"))) { - launcherParser = &(((occa::lang::okl::withLauncher*) parser)->launcherParser); - io::stdout << launcherParser->toString(); - } else { - io::stdout << parser->toString(); - } - - delete parser; - if (!success) { - ::exit(1); - } - return true; + return v2::runTranslate(options, arguments, kernelProps, originalMode, mode); +#else + return v2::runTranslate(options, arguments, kernelProps, originalMode, mode); +#endif } bool runCompile(const json &args) { @@ -229,6 +378,7 @@ namespace occa { kernelProps["verbose"] = kernelProps.get("verbose", true); kernelProps["okl/include_paths"] = options["include-path"]; kernelProps["defines"].asObject() += getOptionDefines(options["define"]); + kernelProps["transpiler-version"] = getTranspilerVersion(options); device device(deviceProps); device.buildKernel(filename, kernelName, kernelProps); @@ -367,6 +517,9 @@ namespace occa { .withArg()) .addOption(cli::option('v', "verbose", "Verbose output")) + .addOption(cli::option('t', "transpiler-version", + "provide transpiler version") + .reusable().withArg()) .addArgument(cli::argument("FILE", "An .okl file") .isRequired() @@ -393,6 +546,9 @@ namespace occa { "Add additional define") .reusable() .withArg()) + .addOption(cli::option('t', "transpiler-version", + "provide transpiler version") + .reusable().withArg()) .addArgument(cli::argument("FILE", "An .okl file") .isRequired() diff --git a/src/occa/internal/core/launchedDevice.cpp b/src/occa/internal/core/launchedDevice.cpp index 821f97626..e2bf8ce6c 100644 --- a/src/occa/internal/core/launchedDevice.cpp +++ b/src/occa/internal/core/launchedDevice.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -8,30 +6,119 @@ #include #include +#ifdef BUILD_WITH_OCCA_TRANSPILER +#include +#include "oklt/pipeline/normalizer_and_transpiler.h" +#include "oklt/util/io_helper.h" +#include "oklt/core/error.h" +#endif + +#include +#include + namespace occa { launchedModeDevice_t::launchedModeDevice_t(const occa::json &properties_) : modeDevice_t(properties_) { needsLauncherKernel = true; } +#ifdef BUILD_WITH_OCCA_TRANSPILER + bool launchedModeDevice_t::transpileFile(const std::string &filename, + const std::string &outputFile, + const std::string &launcherOutputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &launcherMetadata, + lang::sourceMetadata_t &deviceMetadata) + { + static const std::map targetBackends = + { + {"cuda", oklt::TargetBackend::CUDA}, + {"hip", oklt::TargetBackend::HIP}, + {"dpcpp", oklt::TargetBackend::DPCPP}, + }; + + std::string normalized_mode = lowercase(mode); + auto targetIter = targetBackends.find(normalized_mode); + if(targetIter == targetBackends.end()) { + std::string errorDescription = "Can't find target backend: " + mode; + OCCA_FORCE_ERROR(errorDescription + + ", unable to transform OKL kernel [" << filename << "]"); + return false; + } + + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + auto hash = transpiler::getKernelHash(kernelProps); + + std::filesystem::path sourcePath = io::expandFilename(filename); + auto sourceCode = oklt::util::readFileAsStr(sourcePath); + if(!sourceCode) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); + return false; + } + oklt::UserInput input { + .backend = targetIter->second, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = sourcePath, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = std::move(hash), + }; + auto result = normalizeAndTranspile(std::move(input)); + + if(!result) { + if (!kernelProps.get("silent", false)) { + std::stringstream ss; + ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; + ss << "Transpilation errors occured: " << std::endl; + for(const auto &err: result.error()) { + ss << err.desc << std::endl; + } + OCCA_FORCE_ERROR(ss.str()); + } + return false; + } + + auto userOutput = result.value(); + io::stageFiles( + { outputFile, launcherOutputFile }, + true, + [&](const strVector &tempFilenames) -> bool { + + + std::filesystem::path transpiledSource(tempFilenames[0]); + std::filesystem::path launcherSource(tempFilenames[1]); + + auto ret1 = oklt::util::writeFileAsStr(transpiledSource, userOutput.kernel.source); + auto ret2 = oklt::util::writeFileAsStr(launcherSource, userOutput.launcher.source); + return ret1 && ret2; + }); + + transpiler::makeMetadata(launcherMetadata, userOutput.launcher.metadata); + transpiler::makeMetadata(deviceMetadata, userOutput.kernel.metadata); + + return true; + } +#endif + bool launchedModeDevice_t::parseFile(const std::string &filename, const std::string &outputFile, const std::string &launcherOutputFile, const occa::json &kernelProps, lang::sourceMetadata_t &launcherMetadata, lang::sourceMetadata_t &deviceMetadata) { - lang::okl::withLauncher &parser = *(createParser(kernelProps)); - parser.parseFile(filename); + std::unique_ptr parser(createParser(kernelProps)); + parser->parseFile(filename); // Verify if parsing succeeded - if (!parser.succeeded()) { + if (!parser->succeeded()) { if (!kernelProps.get("silent", false)) { OCCA_FORCE_ERROR("Unable to transform OKL kernel [" << filename << "]"); } - delete &parser; return false; } - io::stageFiles( { outputFile, launcherOutputFile }, true, @@ -39,17 +126,16 @@ namespace occa { const std::string &tempOutputFilename = tempFilenames[0]; const std::string &tempLauncherOutputFilename = tempFilenames[1]; - parser.writeToFile(tempOutputFilename); - parser.launcherParser.writeToFile(tempLauncherOutputFilename); + parser->writeToFile(tempOutputFilename); + parser->launcherParser.writeToFile(tempLauncherOutputFilename); return true; } ); - parser.launcherParser.setSourceMetadata(launcherMetadata); - parser.setSourceMetadata(deviceMetadata); + parser->launcherParser.setSourceMetadata(launcherMetadata); + parser->setSourceMetadata(deviceMetadata); - delete &parser; return true; } @@ -138,24 +224,48 @@ namespace occa { lang::sourceMetadata_t launcherMetadata, deviceMetadata; if (usingOkl) { // Cache raw origin - sourceFilename = ( - io::cacheFile(filename, - kc::cachedRawSourceFilename(filename), - kernelHash, - assembleKernelHeader(kernelProps)) - ); + sourceFilename = io::cacheFile(filename, + kc::cachedRawSourceFilename(filename), + kernelHash, + assembleKernelHeader(kernelProps)); const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); const std::string launcherOutputFile = hashDir + kc::launcherSourceFile; - bool valid = parseFile(sourceFilename, - outputFile, - launcherOutputFile, - kernelProps, - launcherMetadata, - deviceMetadata); - if (!valid) { - return NULL; + +#ifdef BUILD_WITH_OCCA_TRANSPILER + int transpilerVersion = kernelProps.get("transpiler-version", 2); + bool isValid = false; + if(transpilerVersion > 2) { + isValid = transpileFile(sourceFilename, + outputFile, + launcherOutputFile, + kernelProps, + launcherMetadata, + deviceMetadata); + } else { + isValid = parseFile(sourceFilename, + outputFile, + launcherOutputFile, + kernelProps, + launcherMetadata, + deviceMetadata); + + } + + if (!isValid) { + return nullptr; + } +#else + if(!parseFile(sourceFilename, + outputFile, + launcherOutputFile, + kernelProps, + launcherMetadata, + deviceMetadata)) + { + return nullptr; } +#endif sourceFilename = outputFile; buildLauncherKernel(kernelHash, diff --git a/src/occa/internal/core/launchedDevice.hpp b/src/occa/internal/core/launchedDevice.hpp index a8e319f10..c2a9275fa 100644 --- a/src/occa/internal/core/launchedDevice.hpp +++ b/src/occa/internal/core/launchedDevice.hpp @@ -69,6 +69,16 @@ namespace occa { const occa::json &kernelProps ) = 0; //================================== + +#ifdef BUILD_WITH_OCCA_TRANSPILER + private: + bool transpileFile(const std::string &filename, + const std::string &outputFile, + const std::string &launcherOutputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &launcherMetadata, + lang::sourceMetadata_t &deviceMetadata); +#endif }; } diff --git a/src/occa/internal/modes/openmp/device.cpp b/src/occa/internal/modes/openmp/device.cpp index eb354a205..59218d32f 100644 --- a/src/occa/internal/modes/openmp/device.cpp +++ b/src/occa/internal/modes/openmp/device.cpp @@ -6,6 +6,13 @@ #include #include +#ifdef BUILD_WITH_OCCA_TRANSPILER +#include +#include "oklt/pipeline/normalizer_and_transpiler.h" +#include "oklt/core/error.h" +#include "oklt/util/io_helper.h" +#endif + namespace occa { namespace openmp { device::device(const occa::json &properties_) : @@ -25,6 +32,61 @@ namespace occa { ); } +#ifdef BUILD_WITH_OCCA_TRANSPILER + bool device::transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata) + { + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + auto hash = transpiler::getKernelHash(kernelProps); + + std::filesystem::path sourcePath = io::expandFilename(filename); + auto sourceCode = oklt::util::readFileAsStr(sourcePath); + if(!sourceCode) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); + return false; + } + oklt::UserInput input { + .backend = oklt::TargetBackend::OPENMP, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = sourcePath, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = std::move(hash), + }; + auto result = normalizeAndTranspile(std::move(input)); + if(!result) { + if (!kernelProps.get("silent", false)) { + std::stringstream ss; + ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; + ss << "Transpilation errors occured: " << std::endl; + for(const auto &err: result.error()) { + ss << err.desc << std::endl; + } + OCCA_FORCE_ERROR(ss.str()); + } + return false; + } + + auto userOutput = result.value(); + + io::stageFile( + outputFile, + true, + [&](const std::string &tempFilename) -> bool { + std::filesystem::path transpiledSource(tempFilename); + auto ret = oklt::util::writeFileAsStr(tempFilename, userOutput.kernel.source); + return ret.has_value(); + }); + transpiler::makeMetadata(metadata, userOutput.kernel.metadata); + return true; + } +#endif + bool device::parseFile(const std::string &filename, const std::string &outputFile, const occa::json &kernelProps, diff --git a/src/occa/internal/modes/openmp/device.hpp b/src/occa/internal/modes/openmp/device.hpp index b821b17ed..babc11081 100644 --- a/src/occa/internal/modes/openmp/device.hpp +++ b/src/occa/internal/modes/openmp/device.hpp @@ -12,6 +12,13 @@ namespace occa { std::string lastCompiler; std::string lastCompilerOpenMPFlag; +#ifdef BUILD_WITH_OCCA_TRANSPILER + bool transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata); +#endif + public: device(const occa::json &properties_); virtual ~device() = default; diff --git a/src/occa/internal/modes/serial/device.cpp b/src/occa/internal/modes/serial/device.cpp index 97376e0ec..e89e4a8b7 100644 --- a/src/occa/internal/modes/serial/device.cpp +++ b/src/occa/internal/modes/serial/device.cpp @@ -11,6 +11,13 @@ #include #include +#ifdef BUILD_WITH_OCCA_TRANSPILER +#include +#include "oklt/pipeline/normalizer_and_transpiler.h" +#include "oklt/core/error.h" +#include "oklt/util/io_helper.h" +#endif + namespace occa { namespace serial { device::device(const occa::json &properties_) : @@ -70,6 +77,61 @@ namespace occa { return (srEndTag->time - srStartTag->time); } + +#ifdef BUILD_WITH_OCCA_TRANSPILER + bool device::transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata) + { + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + + std::filesystem::path sourcePath = io::expandFilename(filename); + auto sourceCode = oklt::util::readFileAsStr(sourcePath); + if(!sourceCode) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); + return false; + } + oklt::UserInput input { + .backend = oklt::TargetBackend::SERIAL, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = sourcePath, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = "", + }; + auto result = normalizeAndTranspile(std::move(input)); + if(!result) { + if (!kernelProps.get("silent", false)) { + std::stringstream ss; + ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; + ss << "Transpilation errors occured: " << std::endl; + for(const auto &err: result.error()) { + ss << err.desc << std::endl; + } + OCCA_FORCE_ERROR(ss.str()); + } + return false; + } + + auto userOutput = result.value(); + + io::stageFile( + outputFile, + true, + [&](const std::string &tempFilename) -> bool { + std::filesystem::path transpiledSourcePath(tempFilename); + auto ret = oklt::util::writeFileAsStr(transpiledSourcePath, userOutput.kernel.source); + return ret.has_value(); + }); + transpiler::makeMetadata(metadata, userOutput.kernel.metadata); + return true; + } +#endif + //================================== //---[ Kernel ]--------------------- @@ -286,13 +348,36 @@ namespace occa { if (compilingOkl) { const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); - bool valid = parseFile(sourceFilename, + +#ifdef BUILD_WITH_OCCA_TRANSPILER + int transpilerVersion = kernelProps.get("transpiler-version", 2); + + bool valid = false; + if(transpilerVersion > 2) { + valid = transpileFile(sourceFilename, + outputFile, + kernelProps, + metadata); + } else { + valid = parseFile(sourceFilename, outputFile, kernelProps, metadata); + + } + if (!valid) { - return NULL; + return nullptr; } +#else + if(!parseFile(sourceFilename, + outputFile, + kernelProps, + metadata)) + { + return nullptr; + } +#endif sourceFilename = outputFile; writeKernelBuildFile(hashDir + kc::buildFile, diff --git a/src/occa/internal/modes/serial/device.hpp b/src/occa/internal/modes/serial/device.hpp index 5796c1a49..65f678e08 100644 --- a/src/occa/internal/modes/serial/device.hpp +++ b/src/occa/internal/modes/serial/device.hpp @@ -75,6 +75,14 @@ namespace occa { //================================ void* unwrap() override; + +#ifdef BUILD_WITH_OCCA_TRANSPILER + private: + bool transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata); +#endif }; } } diff --git a/src/occa/internal/utils/transpiler_utils.cpp b/src/occa/internal/utils/transpiler_utils.cpp new file mode 100644 index 000000000..79a3f850e --- /dev/null +++ b/src/occa/internal/utils/transpiler_utils.cpp @@ -0,0 +1,73 @@ +#include +#include + +namespace occa { +namespace transpiler { + +std::string getKernelHash(const json &kernelProp) { + auto hashStr = kernelProp.get("hash"); + if(hashStr.empty()) { + throw std::runtime_error("kernel proerties does not contain hash entry"); + } + return hashStr; +} + +std::vector buildDefines(const json &kernelProp) { + const json &defines = kernelProp["defines"]; + if (!defines.isObject()) { + {}; + } + + std::vector definesStrings; + const jsonObject &defineMap = defines.object(); + jsonObject::const_iterator it = defineMap.cbegin(); + while (it != defineMap.end()) { + const std::string &define = it->first; + const json value = it->second; + + std::string defineString = define + "=" + value.toString(); + definesStrings.push_back(std::move(defineString)); + ++it; + } + return definesStrings; +} + +std::vector buildIncludes(const json &kernelProp) { + std::vector includes; + json oklIncludePaths = kernelProp.get("okl/include_paths", json{}); + if (oklIncludePaths.isArray()) { + jsonArray pathArray = oklIncludePaths.array(); + const int pathCount = (int) pathArray.size(); + for (int i = 0; i < pathCount; ++i) { + json path = pathArray[i]; + if (path.isString()) { + includes.push_back(std::filesystem::path(path.string())); + } + } + } + auto &envIncludes = env::OCCA_INCLUDE_PATH; + for(const auto &includePath: envIncludes) { + includes.push_back(includePath); + } + return includes; +} + +void makeMetadata(lang::sourceMetadata_t &sourceMetadata, + const std::string &jsonStr) +{ + + lang::kernelMetadataMap &metadataMap = sourceMetadata.kernelsMetadata; + auto json = json::parse(jsonStr); + auto metadataObj = json.get("metadata"); + if(metadataObj.isArray()) { + jsonArray metaArr = metadataObj.asArray().array(); + for(const auto &elem: metaArr) { + auto name = elem.get("name"); + auto kernelObj = lang::kernelMetadata_t::fromJson(elem); + metadataMap.insert(std::make_pair(name, kernelObj)); + } + } +} + +} +} diff --git a/src/occa/internal/utils/transpiler_utils.h b/src/occa/internal/utils/transpiler_utils.h new file mode 100644 index 000000000..144cdbfdc --- /dev/null +++ b/src/occa/internal/utils/transpiler_utils.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +namespace occa { +namespace transpiler { + +std::string getKernelHash(const json &kernelProp); +std::vector buildDefines(const json &kernelProp); +std::vector buildIncludes(const json &kernelProp); +void makeMetadata(lang::sourceMetadata_t &sourceMetadata, + const std::string &jsonStr); + +} +} diff --git a/tests/src/internal/bin/occa.cpp b/tests/src/internal/bin/occa.cpp index 5a6e587d3..d47f15de7 100644 --- a/tests/src/internal/bin/occa.cpp +++ b/tests/src/internal/bin/occa.cpp @@ -173,7 +173,7 @@ int main(const int argc, const char **argv) { //---[ Compile ]------------------ const std::string compileOptions = ( - "--define --device-props --help --include-path --kernel-props -D -I -d -h -k" + "--define --device-props --help --include-path --kernel-props --transpiler-version -D -I -d -h -k -t" ); ASSERT_AUTOCOMPLETE_EQ( @@ -203,7 +203,7 @@ int main(const int argc, const char **argv) { //---[ Translate ]------------------ const std::string translateOptions = ( - "--define --help --include-path --kernel-props --launcher --mode --verbose -D -I -h -k -l -m -v" + "--define --help --include-path --kernel-props --launcher --mode --transpiler-version --verbose -D -I -h -k -l -m -t -v" ); ASSERT_AUTOCOMPLETE_EQ( From 0bf0c9d28b592cf77a1e0d64182e3715a12ee646 Mon Sep 17 00:00:00 2001 From: Viktor Yastrebov Date: Thu, 9 May 2024 16:03:08 +0300 Subject: [PATCH 02/25] added missing GitSubmodules.cmake --- cmake/GitSubmodules.cmake | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 cmake/GitSubmodules.cmake diff --git a/cmake/GitSubmodules.cmake b/cmake/GitSubmodules.cmake new file mode 100644 index 000000000..8944e1fb8 --- /dev/null +++ b/cmake/GitSubmodules.cmake @@ -0,0 +1,26 @@ +#################################################################################################### +# This function recursivelly update git submodules. +# Params: project base directory +# Example: +# init_submodules(${PROJECT_SOURCE_DIR}) +#################################################################################################### + +function(init_submodules + project_dir) + + find_package(Git QUIET) + if(GIT_FOUND AND EXISTS "${project_dir}/.git") + # Update submodules as needed + message(STATUS "Submodule update") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive + WORKING_DIRECTORY ${project_dir} + RESULT_VARIABLE GIT_SUBMOD_RESULT) + if(NOT GIT_SUBMOD_RESULT EQUAL "0") + message(FATAL_ERROR "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + endif() + else() + message(STATUS "Submodule update has not been run") + endif() + +endfunction() + From 5fc800ee18486008f3d1ef06ba1344d660d5bdda Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Thu, 9 May 2024 20:50:44 +0300 Subject: [PATCH 03/25] fixes for code review & OpenMP/Serial bug fix of non-polymorphic call was used --- CMakeLists.txt | 8 +- README.md | 2 +- src/occa/internal/bin/occa.cpp | 229 +-------------- src/occa/internal/core/launchedDevice.cpp | 12 +- src/occa/internal/core/launchedDevice.hpp | 2 +- src/occa/internal/modes/openmp/device.cpp | 10 +- src/occa/internal/modes/openmp/device.hpp | 4 +- src/occa/internal/modes/serial/device.cpp | 12 +- src/occa/internal/modes/serial/device.hpp | 10 +- src/occa/internal/utils/transpiler_utils.cpp | 279 ++++++++++++++++++- src/occa/internal/utils/transpiler_utils.h | 9 + 11 files changed, 308 insertions(+), 269 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41f678f77..8d4af5535 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ option(OCCA_ENABLE_DPCPP "Build with SYCL/DPCPP if available" ON) option(OCCA_ENABLE_TESTS "Build tests" OFF) option(OCCA_ENABLE_EXAMPLES "Build simple examples" OFF) option(OCCA_ENABLE_FORTRAN "Enable Fortran interface" OFF) -option(CLANG_BASED_TRANSPILER "Build with occa-transpiler dependecy" OFF) +option(OCCA_TRANSPILER "Build with occa-transpiler dependecy" OFF) if(OCCA_ENABLE_FORTRAN) enable_language(Fortran) @@ -69,7 +69,7 @@ else() endif() # INFO: order is important, deps should not apply compiler flags -if (CLANG_BASED_TRANSPILER) +if (OCCA_TRANSPILER) include(cmake/CPM_0.38.6.cmake) include(cmake/GitSubmodules.cmake) @@ -128,12 +128,12 @@ target_include_directories(libocca PRIVATE $) target_compile_definitions(libocca PRIVATE -DUSE_CMAKE) -if (CLANG_BASED_TRANSPILER) +if (OCCA_TRANSPILER) target_link_libraries(libocca PRIVATE occa-transpiler ) - target_compile_definitions(libocca PRIVATE -DBUILD_WITH_OCCA_TRANSPILER) + target_compile_definitions(libocca PRIVATE -DBUILD_WITH_CLANG_BASED_TRANSPILER) endif() #======================================= diff --git a/README.md b/README.md index 5f21f5e5d..d259b8446 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ For building with C++ support/occa-transpiler: ```shell $ mkdir build $ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release -DCLANG_BASED_TRANSPILER=ON .. +$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_TRANSPILER=ON .. $ cmake --build . --parallel $ cmake --install . --prefix install ``` diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index cf50c290b..46457b26d 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -3,24 +3,13 @@ #include #include - #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include -#ifdef BUILD_WITH_OCCA_TRANSPILER #include -#include "oklt/pipeline/normalizer_and_transpiler.h" -#include "oklt/util/io_helper.h" -#include "oklt/core/error.h" -#endif #include @@ -135,208 +124,6 @@ namespace occa { return true; } -#ifdef BUILD_WITH_OCCA_TRANSPILER - namespace v3 { - - bool runTranslate(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode) - { - static const std::map targetBackends = - { - {"openmp", oklt::TargetBackend::OPENMP}, - {"cuda", oklt::TargetBackend::CUDA}, - {"hip", oklt::TargetBackend::HIP}, - {"dpcpp", oklt::TargetBackend::DPCPP}, - }; - - const bool printLauncher = options["launcher"]; - const std::string filename = arguments[0]; - - if (!io::exists(filename)) { - printError("File [" + filename + "] doesn't exist" ); - ::exit(1); - } - - auto transpiler = targetBackends.find(mode); - if(transpiler == targetBackends.end()) { - printError("Unable to translate for mode [" + originalMode + "]"); - ::exit(1); - } - - auto defines = transpiler::buildDefines(kernelProps); - auto includes = transpiler::buildIncludes(kernelProps); - auto hash = transpiler::getKernelHash(kernelProps); - - std::filesystem::path sourcePath = io::expandFilename(filename); - auto sourceCode = oklt::util::readFileAsStr(sourcePath); - if(!sourceCode) { - printError("Can't open file: " + sourcePath.string()); - ::exit(sourceCode.error()); - } - oklt::UserInput input { - .backend = transpiler->second, - .source = std::move(sourceCode.value()), - .headers = {}, - .sourcePath = sourcePath, - .includeDirectories = std::move(includes), - .defines = std::move(defines), - .hash = std::move(hash) - }; - auto result = normalizeAndTranspile(std::move(input)); - - if(!result) { - std::stringstream ss; - for(const auto &err: result.error()) { - ss << err.desc << std::endl; - } - printError(ss.str()); - ::exit(1); - } - - if (options["verbose"]) { - json translationInfo; - // Filename - translationInfo["translate_info/filename"] = io::expandFilename(filename); - // Date information - translationInfo["translate_info/date"] = sys::date(); - translationInfo["translate_info/human_date"] = sys::humanDate(); - // Version information - translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; - translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; - // Kernel properties - translationInfo["kernel_properties"] = kernelProps; - - io::stdout - << "/* Translation Info:\n" - << translationInfo - << "*/\n"; - } - - auto userOutput = result.value(); - bool hasLauncher = transpiler->second == oklt::TargetBackend::CUDA || - transpiler->second == oklt::TargetBackend::HIP || - transpiler->second == oklt::TargetBackend::DPCPP; - if(printLauncher && hasLauncher) { - io::stdout << userOutput.launcher.source; - } else { - io::stdout << userOutput.kernel.source; - } - - return true; - } - } -#endif - - namespace v2 { - bool runTranslate(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode) - { - using ParserBuildFunc = std::function(const json ¶ms)>; - static const std::map originalParserBackends = - { - {"", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"serial", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"openmp", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"cuda", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"hip", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"opencl", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"metal", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"dpcpp", [](const json ¶ms) { - return std::make_unique(params); - }} - }; - - const bool printLauncher = options["launcher"]; - const std::string filename = arguments[0]; - - if (!io::exists(filename)) { - printError("File [" + filename + "] doesn't exist" ); - ::exit(1); - } - - auto parserIt = originalParserBackends.find(mode); - if(parserIt == originalParserBackends.end()) { - printError("Unable to translate for mode [" + originalMode + "]"); - ::exit(1); - } - - std::unique_ptr parser = parserIt->second(kernelProps); - parser->parseFile(filename); - - bool success = parser->succeeded(); - if (!success) { - ::exit(1); - } - - if (options["verbose"]) { - json translationInfo; - // Filename - translationInfo["translate_info/filename"] = io::expandFilename(filename); - // Date information - translationInfo["translate_info/date"] = sys::date(); - translationInfo["translate_info/human_date"] = sys::humanDate(); - // Version information - translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; - translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; - // Kernel properties - translationInfo["kernel_properties"] = kernelProps; - - io::stdout - << "/* Translation Info:\n" - << translationInfo - << "*/\n"; - } - - if (printLauncher && ((mode == "cuda") - || (mode == "hip") - || (mode == "opencl") - || (mode == "dpcpp") - || (mode == "metal"))) { - lang::parser_t *launcherParser = &(((occa::lang::okl::withLauncher*) parser.get())->launcherParser); - io::stdout << launcherParser->toString(); - } else { - io::stdout << parser->toString(); - } - return true; - } - } - - int getTranspilerVersion(const json &options) { - json jsonTranspileVersion = options["transpiler-version"]; - int transpilerVersion = 2; - //INFO: have no idea why json here has array type - if(jsonTranspileVersion.isArray()) { - json elem = jsonTranspileVersion.asArray()[0]; - if(elem.isString()) { - try { - transpilerVersion = std::stoi(elem.string()); - } catch(const std::exception &) - {} - } - } - return transpilerVersion; - } - bool runTranslate(const json &args) { const json &options = args["options"]; const json &arguments = args["arguments"]; @@ -349,15 +136,7 @@ namespace occa { kernelProps["defines"].asObject() += getOptionDefines(options["define"]); kernelProps["okl/include_paths"] = options["include-path"]; -#ifdef BUILD_WITH_OCCA_TRANSPILER - int transpilerVersion = getTranspilerVersion(options); - if(transpilerVersion > 2) { - return v3::runTranslate(options, arguments, kernelProps, originalMode, mode); - } - return v2::runTranslate(options, arguments, kernelProps, originalMode, mode); -#else - return v2::runTranslate(options, arguments, kernelProps, originalMode, mode); -#endif + return transpiler::runTranspiler(options, arguments, kernelProps, originalMode, mode); } bool runCompile(const json &args) { @@ -378,7 +157,7 @@ namespace occa { kernelProps["verbose"] = kernelProps.get("verbose", true); kernelProps["okl/include_paths"] = options["include-path"]; kernelProps["defines"].asObject() += getOptionDefines(options["define"]); - kernelProps["transpiler-version"] = getTranspilerVersion(options); + kernelProps["transpiler-version"] = transpiler::getTranspilerVersion(options); device device(deviceProps); device.buildKernel(filename, kernelName, kernelProps); diff --git a/src/occa/internal/core/launchedDevice.cpp b/src/occa/internal/core/launchedDevice.cpp index e2bf8ce6c..8dd4c1c1c 100644 --- a/src/occa/internal/core/launchedDevice.cpp +++ b/src/occa/internal/core/launchedDevice.cpp @@ -6,11 +6,11 @@ #include #include -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER #include -#include "oklt/pipeline/normalizer_and_transpiler.h" -#include "oklt/util/io_helper.h" -#include "oklt/core/error.h" +#include +#include +#include #endif #include @@ -22,7 +22,7 @@ namespace occa { needsLauncherKernel = true; } -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool launchedModeDevice_t::transpileFile(const std::string &filename, const std::string &outputFile, const std::string &launcherOutputFile, @@ -232,7 +232,7 @@ namespace occa { const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); const std::string launcherOutputFile = hashDir + kc::launcherSourceFile; -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER int transpilerVersion = kernelProps.get("transpiler-version", 2); bool isValid = false; if(transpilerVersion > 2) { diff --git a/src/occa/internal/core/launchedDevice.hpp b/src/occa/internal/core/launchedDevice.hpp index c2a9275fa..cf88850f9 100644 --- a/src/occa/internal/core/launchedDevice.hpp +++ b/src/occa/internal/core/launchedDevice.hpp @@ -70,7 +70,7 @@ namespace occa { ) = 0; //================================== -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER private: bool transpileFile(const std::string &filename, const std::string &outputFile, diff --git a/src/occa/internal/modes/openmp/device.cpp b/src/occa/internal/modes/openmp/device.cpp index 59218d32f..c23a09572 100644 --- a/src/occa/internal/modes/openmp/device.cpp +++ b/src/occa/internal/modes/openmp/device.cpp @@ -6,11 +6,11 @@ #include #include -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER #include -#include "oklt/pipeline/normalizer_and_transpiler.h" -#include "oklt/core/error.h" -#include "oklt/util/io_helper.h" +#include +#include +#include #endif namespace occa { @@ -32,7 +32,7 @@ namespace occa { ); } -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool device::transpileFile(const std::string &filename, const std::string &outputFile, const occa::json &kernelProps, diff --git a/src/occa/internal/modes/openmp/device.hpp b/src/occa/internal/modes/openmp/device.hpp index babc11081..7449a83cb 100644 --- a/src/occa/internal/modes/openmp/device.hpp +++ b/src/occa/internal/modes/openmp/device.hpp @@ -12,11 +12,11 @@ namespace occa { std::string lastCompiler; std::string lastCompilerOpenMPFlag; -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool transpileFile(const std::string &filename, const std::string &outputFile, const occa::json &kernelProps, - lang::sourceMetadata_t &metadata); + lang::sourceMetadata_t &metadata) override; #endif public: diff --git a/src/occa/internal/modes/serial/device.cpp b/src/occa/internal/modes/serial/device.cpp index e89e4a8b7..c5a189528 100644 --- a/src/occa/internal/modes/serial/device.cpp +++ b/src/occa/internal/modes/serial/device.cpp @@ -11,11 +11,11 @@ #include #include -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER #include -#include "oklt/pipeline/normalizer_and_transpiler.h" -#include "oklt/core/error.h" -#include "oklt/util/io_helper.h" +#include +#include +#include #endif namespace occa { @@ -78,7 +78,7 @@ namespace occa { return (srEndTag->time - srStartTag->time); } -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool device::transpileFile(const std::string &filename, const std::string &outputFile, const occa::json &kernelProps, @@ -349,7 +349,7 @@ namespace occa { if (compilingOkl) { const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER int transpilerVersion = kernelProps.get("transpiler-version", 2); bool valid = false; diff --git a/src/occa/internal/modes/serial/device.hpp b/src/occa/internal/modes/serial/device.hpp index 65f678e08..f65515bf1 100644 --- a/src/occa/internal/modes/serial/device.hpp +++ b/src/occa/internal/modes/serial/device.hpp @@ -76,12 +76,12 @@ namespace occa { void* unwrap() override; -#ifdef BUILD_WITH_OCCA_TRANSPILER +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER private: - bool transpileFile(const std::string &filename, - const std::string &outputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &metadata); + virtual bool transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata); #endif }; } diff --git a/src/occa/internal/utils/transpiler_utils.cpp b/src/occa/internal/utils/transpiler_utils.cpp index 79a3f850e..ebece02f1 100644 --- a/src/occa/internal/utils/transpiler_utils.cpp +++ b/src/occa/internal/utils/transpiler_utils.cpp @@ -1,9 +1,49 @@ #include #include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER +#include +#include +#include +#endif namespace occa { namespace transpiler { +int getTranspilerVersion(const json &options) { + json jsonTranspileVersion = options["transpiler-version"]; + int transpilerVersion = 2; + //INFO: have no idea why json here has array type + if(!jsonTranspileVersion.isArray()) { + return transpilerVersion; + } + json elem = jsonTranspileVersion.asArray()[0]; + if(!elem.isString()) { + return transpilerVersion; + } + + try { + transpilerVersion = std::stoi(elem.string()); + } catch(const std::exception &) + { + return transpilerVersion; + } + return transpilerVersion; +} + std::string getKernelHash(const json &kernelProp) { auto hashStr = kernelProp.get("hash"); if(hashStr.empty()) { @@ -35,16 +75,19 @@ std::vector buildDefines(const json &kernelProp) { std::vector buildIncludes(const json &kernelProp) { std::vector includes; json oklIncludePaths = kernelProp.get("okl/include_paths", json{}); - if (oklIncludePaths.isArray()) { - jsonArray pathArray = oklIncludePaths.array(); - const int pathCount = (int) pathArray.size(); - for (int i = 0; i < pathCount; ++i) { - json path = pathArray[i]; - if (path.isString()) { - includes.push_back(std::filesystem::path(path.string())); - } + if (!oklIncludePaths.isArray()) { + return {}; + } + + jsonArray pathArray = oklIncludePaths.array(); + const int pathCount = (int) pathArray.size(); + for (int i = 0; i < pathCount; ++i) { + json path = pathArray[i]; + if (path.isString()) { + includes.push_back(std::filesystem::path(path.string())); } } + auto &envIncludes = env::OCCA_INCLUDE_PATH; for(const auto &includePath: envIncludes) { includes.push_back(includePath); @@ -59,14 +102,222 @@ void makeMetadata(lang::sourceMetadata_t &sourceMetadata, lang::kernelMetadataMap &metadataMap = sourceMetadata.kernelsMetadata; auto json = json::parse(jsonStr); auto metadataObj = json.get("metadata"); - if(metadataObj.isArray()) { - jsonArray metaArr = metadataObj.asArray().array(); - for(const auto &elem: metaArr) { - auto name = elem.get("name"); - auto kernelObj = lang::kernelMetadata_t::fromJson(elem); - metadataMap.insert(std::make_pair(name, kernelObj)); + if(!metadataObj.isArray()) { + printError("Can't get the metadata"); + return; + } + jsonArray metaArr = metadataObj.asArray().array(); + for(const auto &elem: metaArr) { + auto name = elem.get("name"); + auto kernelObj = lang::kernelMetadata_t::fromJson(elem); + metadataMap.insert(std::make_pair(name, kernelObj)); + } +} + + +namespace v2 { + bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + using ParserBuildFunc = std::function(const json ¶ms)>; + static const std::map originalParserBackends = + { + {"", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"serial", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"openmp", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"cuda", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"hip", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"opencl", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"metal", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"dpcpp", [](const json ¶ms) { + return std::make_unique(params); + }} + }; + + const bool printLauncher = options["launcher"]; + const std::string filename = arguments[0]; + + if (!io::exists(filename)) { + printError("File [" + filename + "] doesn't exist" ); + ::exit(1); + } + + auto parserIt = originalParserBackends.find(mode); + if(parserIt == originalParserBackends.end()) { + printError("Unable to translate for mode [" + originalMode + "]"); + ::exit(1); + } + + std::unique_ptr parser = parserIt->second(kernelProps); + parser->parseFile(filename); + + bool success = parser->succeeded(); + if (!success) { + ::exit(1); + } + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; } + + if (printLauncher && ((mode == "cuda") + || (mode == "hip") + || (mode == "opencl") + || (mode == "dpcpp") + || (mode == "metal"))) { + lang::parser_t *launcherParser = &(((occa::lang::okl::withLauncher*) parser.get())->launcherParser); + io::stdout << launcherParser->toString(); + } else { + io::stdout << parser->toString(); + } + return true; + } +} + +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER +namespace v3 { + bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + static const std::map targetBackends = + { + {"openmp", oklt::TargetBackend::OPENMP}, + {"cuda", oklt::TargetBackend::CUDA}, + {"hip", oklt::TargetBackend::HIP}, + {"dpcpp", oklt::TargetBackend::DPCPP}, + {"serial", oklt::TargetBackend::SERIAL}, + }; + + const bool printLauncher = options["launcher"]; + const std::string filename = arguments[0]; + + if (!io::exists(filename)) { + printError("File [" + filename + "] doesn't exist" ); + ::exit(1); + } + + auto transpiler = targetBackends.find(mode); + if(transpiler == targetBackends.end()) { + printError("Unable to translate for mode [" + originalMode + "]"); + ::exit(1); + } + + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + auto hash = transpiler::getKernelHash(kernelProps); + + std::filesystem::path sourcePath = io::expandFilename(filename); + auto sourceCode = oklt::util::readFileAsStr(sourcePath); + if(!sourceCode) { + printError("Can't open file: " + sourcePath.string()); + ::exit(sourceCode.error()); + } + oklt::UserInput input { + .backend = transpiler->second, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = sourcePath, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = std::move(hash) + }; + auto result = normalizeAndTranspile(std::move(input)); + + if(!result) { + std::stringstream ss; + for(const auto &err: result.error()) { + ss << err.desc << std::endl; + } + printError(ss.str()); + ::exit(1); + } + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; + } + + auto userOutput = result.value(); + bool hasLauncher = transpiler->second == oklt::TargetBackend::CUDA || + transpiler->second == oklt::TargetBackend::HIP || + transpiler->second == oklt::TargetBackend::DPCPP; + if(printLauncher && hasLauncher) { + io::stdout << userOutput.launcher.source; + } else { + io::stdout << userOutput.kernel.source; + } + + return true; + } +} +#endif + +bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) +{ + int transpilerVersion = getTranspilerVersion(options); +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER + if(transpilerVersion > 2) { + return v3::runTranspiler(options, arguments, kernelProps, originalMode, mode); + } +#endif + if (transpilerVersion > 2) { + printError("OCCA compiler is built without BUILD_WITH_CLANG_BASED_TRANSPILER support"); + return false; } + return v2::runTranspiler(options, arguments, kernelProps, originalMode, mode); } } diff --git a/src/occa/internal/utils/transpiler_utils.h b/src/occa/internal/utils/transpiler_utils.h index 144cdbfdc..6dfbd6880 100644 --- a/src/occa/internal/utils/transpiler_utils.h +++ b/src/occa/internal/utils/transpiler_utils.h @@ -7,11 +7,20 @@ namespace occa { namespace transpiler { +int getTranspilerVersion(const json &options); + std::string getKernelHash(const json &kernelProp); std::vector buildDefines(const json &kernelProp); std::vector buildIncludes(const json &kernelProp); void makeMetadata(lang::sourceMetadata_t &sourceMetadata, const std::string &jsonStr); +bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode); + + } } From 240fed30c968dddc897dafc9b4aa06b55b324a73 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Fri, 10 May 2024 19:02:16 +0300 Subject: [PATCH 04/25] refactoring of integration, use function composition & callbacks strategy --- CMakeLists.txt | 6 +- README.md | 2 +- src/occa/internal/bin/occa.cpp | 189 ++++++++++++- src/occa/internal/core/launchedDevice.cpp | 126 ++++----- src/occa/internal/core/launchedDevice.hpp | 9 - src/occa/internal/modes/openmp/device.cpp | 114 ++++---- src/occa/internal/modes/openmp/device.hpp | 7 - src/occa/internal/modes/serial/device.cpp | 122 +++++---- src/occa/internal/modes/serial/device.hpp | 7 - src/occa/internal/utils/transpiler_utils.cpp | 271 +++++-------------- src/occa/internal/utils/transpiler_utils.h | 45 ++- 11 files changed, 465 insertions(+), 433 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d4af5535..4100541a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ option(OCCA_ENABLE_DPCPP "Build with SYCL/DPCPP if available" ON) option(OCCA_ENABLE_TESTS "Build tests" OFF) option(OCCA_ENABLE_EXAMPLES "Build simple examples" OFF) option(OCCA_ENABLE_FORTRAN "Enable Fortran interface" OFF) -option(OCCA_TRANSPILER "Build with occa-transpiler dependecy" OFF) +option(OCCA_CLANG_BASED_TRANSPILER "Build with occa-transpiler dependecy" OFF) if(OCCA_ENABLE_FORTRAN) enable_language(Fortran) @@ -69,7 +69,7 @@ else() endif() # INFO: order is important, deps should not apply compiler flags -if (OCCA_TRANSPILER) +if (OCCA_CLANG_BASED_TRANSPILER) include(cmake/CPM_0.38.6.cmake) include(cmake/GitSubmodules.cmake) @@ -128,7 +128,7 @@ target_include_directories(libocca PRIVATE $) target_compile_definitions(libocca PRIVATE -DUSE_CMAKE) -if (OCCA_TRANSPILER) +if (OCCA_CLANG_BASED_TRANSPILER) target_link_libraries(libocca PRIVATE occa-transpiler diff --git a/README.md b/README.md index d259b8446..c88548e8a 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ For building with C++ support/occa-transpiler: ```shell $ mkdir build $ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_TRANSPILER=ON .. +$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON .. $ cmake --build . --parallel $ cmake --install . --prefix install ``` diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index 46457b26d..d713e4e96 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -7,9 +7,20 @@ #include #include #include + +#include +#include +#include +#include +#include +#include +#include +#include #include +#if BUILD_WITH_CLANG_BASED_TRANSPILER #include +#endif #include @@ -124,6 +135,181 @@ namespace occa { return true; } + namespace v2 { + bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + using ParserBuildFunc = std::function(const json ¶ms)>; + static const std::map originalParserBackends = + { + {"", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"serial", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"openmp", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"cuda", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"hip", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"opencl", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"metal", [](const json ¶ms) { + return std::make_unique(params); + }}, + {"dpcpp", [](const json ¶ms) { + return std::make_unique(params); + }} + }; + + const bool printLauncher = options["launcher"]; + const std::string filename = arguments[0]; + + if (!io::exists(filename)) { + printError("File [" + filename + "] doesn't exist" ); + ::exit(1); + } + + auto parserIt = originalParserBackends.find(mode); + if(parserIt == originalParserBackends.end()) { + printError("Unable to translate for mode [" + originalMode + "]"); + ::exit(1); + } + + std::unique_ptr parser = parserIt->second(kernelProps); + parser->parseFile(filename); + + bool success = parser->succeeded(); + if (!success) { + ::exit(1); + } + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; + } + + if (printLauncher && ((mode == "cuda") + || (mode == "hip") + || (mode == "opencl") + || (mode == "dpcpp") + || (mode == "metal"))) { + lang::parser_t *launcherParser = &(((occa::lang::okl::withLauncher*) parser.get())->launcherParser); + io::stdout << launcherParser->toString(); + } else { + io::stdout << parser->toString(); + } + return true; + } + } + + namespace v3 { + bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + + auto onFileNotExists = [](const std::string &file) { + printError("File [" + file + "] doesn't exist" ); + ::exit(1); + }; + + auto onWrongBackend = [](const std::string &m) { + printError("Unsupported target backend: [" + m + "]"); + ::exit(1); + }; + + auto onFail = [](const std::vector &errors) { + std::stringstream ss; + for(const auto &err: errors) { + ss << err.desc << std::endl; + } + printError(ss.str()); + ::exit(1); + }; + + std::string filename = arguments[0]; + auto onSuccess = [&](const oklt::UserOutput &output, bool hasLauncher) -> bool { + const bool printLauncher = options["launcher"]; + + if (options["verbose"]) { + json translationInfo; + // Filename + translationInfo["translate_info/filename"] = io::expandFilename(filename); + // Date information + translationInfo["translate_info/date"] = sys::date(); + translationInfo["translate_info/human_date"] = sys::humanDate(); + // Version information + translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; + translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; + // Kernel properties + translationInfo["kernel_properties"] = kernelProps; + + io::stdout + << "/* Translation Info:\n" + << translationInfo + << "*/\n"; + } + + if(printLauncher && hasLauncher) { + io::stdout << output.launcher.source; + } else { + io::stdout << output.kernel.source; + } + + return true; + }; + + transpiler::Transpiler transpiler(onSuccess, onFail, onFileNotExists, onWrongBackend); + return transpiler.run(filename, mode, kernelProps); + } + } + + bool runTranspiler(const json &options, + const json &arguments, + const json &kernelProps, + const std::string &originalMode, + const std::string &mode) + { + int transpilerVersion = transpiler::getTranspilerVersion(options); + #ifdef BUILD_WITH_CLANG_BASED_TRANSPILER + if(transpilerVersion > 2) { + return v3::runTranspiler(options, arguments, kernelProps, originalMode, mode); + } + #endif + if (transpilerVersion > 2) { + printError("OCCA compiler is built without BUILD_WITH_CLANG_BASED_TRANSPILER support"); + return false; + } + return v2::runTranspiler(options, arguments, kernelProps, originalMode, mode); + } + + bool runTranslate(const json &args) { const json &options = args["options"]; const json &arguments = args["arguments"]; @@ -135,8 +321,7 @@ namespace occa { kernelProps["mode"] = mode; kernelProps["defines"].asObject() += getOptionDefines(options["define"]); kernelProps["okl/include_paths"] = options["include-path"]; - - return transpiler::runTranspiler(options, arguments, kernelProps, originalMode, mode); + return runTranspiler(options, arguments, kernelProps, originalMode, mode); } bool runCompile(const json &args) { diff --git a/src/occa/internal/core/launchedDevice.cpp b/src/occa/internal/core/launchedDevice.cpp index 8dd4c1c1c..f0c94de34 100644 --- a/src/occa/internal/core/launchedDevice.cpp +++ b/src/occa/internal/core/launchedDevice.cpp @@ -17,91 +17,70 @@ #include namespace occa { - launchedModeDevice_t::launchedModeDevice_t(const occa::json &properties_) : - modeDevice_t(properties_) { - needsLauncherKernel = true; - } #ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - bool launchedModeDevice_t::transpileFile(const std::string &filename, - const std::string &outputFile, - const std::string &launcherOutputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &launcherMetadata, - lang::sourceMetadata_t &deviceMetadata) - { - static const std::map targetBackends = - { - {"cuda", oklt::TargetBackend::CUDA}, - {"hip", oklt::TargetBackend::HIP}, - {"dpcpp", oklt::TargetBackend::DPCPP}, +namespace v3 { +bool transpileFile(const std::string &filename, + const std::string &outputFile, + const std::string &launcherOutputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &launcherMetadata, + lang::sourceMetadata_t &deviceMetadata, + const std::string &mode) +{ + auto onFileNotExists = [](const std::string &file) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << file); }; - std::string normalized_mode = lowercase(mode); - auto targetIter = targetBackends.find(normalized_mode); - if(targetIter == targetBackends.end()) { - std::string errorDescription = "Can't find target backend: " + mode; + auto onWrongBackend = [&](const std::string &m) { + std::string errorDescription = "Unsupported target backend: " + m; OCCA_FORCE_ERROR(errorDescription + - ", unable to transform OKL kernel [" << filename << "]"); - return false; - } - - auto defines = transpiler::buildDefines(kernelProps); - auto includes = transpiler::buildIncludes(kernelProps); - auto hash = transpiler::getKernelHash(kernelProps); - - std::filesystem::path sourcePath = io::expandFilename(filename); - auto sourceCode = oklt::util::readFileAsStr(sourcePath); - if(!sourceCode) { - std::string errorDescription = "Can't read file: "; - OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); - return false; - } - oklt::UserInput input { - .backend = targetIter->second, - .source = std::move(sourceCode.value()), - .headers = {}, - .sourcePath = sourcePath, - .includeDirectories = std::move(includes), - .defines = std::move(defines), - .hash = std::move(hash), + ", for OKL kernel [" << filename << "]"); }; - auto result = normalizeAndTranspile(std::move(input)); - if(!result) { - if (!kernelProps.get("silent", false)) { + bool isSilent = kernelProps.get("silent", false); + auto onFail = [&](const std::vector &errors) { + if (!isSilent) { std::stringstream ss; ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; ss << "Transpilation errors occured: " << std::endl; - for(const auto &err: result.error()) { + for(const auto &err: errors) { ss << err.desc << std::endl; } OCCA_FORCE_ERROR(ss.str()); } - return false; - } + }; - auto userOutput = result.value(); - io::stageFiles( - { outputFile, launcherOutputFile }, - true, - [&](const strVector &tempFilenames) -> bool { + auto onSuccess = [&](const oklt::UserOutput &output, bool hasLauncher) -> bool { + io::stageFiles( + { outputFile, launcherOutputFile }, + true, + [&](const strVector &tempFilenames) -> bool { - std::filesystem::path transpiledSource(tempFilenames[0]); - std::filesystem::path launcherSource(tempFilenames[1]); + std::filesystem::path transpiledSource(tempFilenames[0]); + std::filesystem::path launcherSource(tempFilenames[1]); - auto ret1 = oklt::util::writeFileAsStr(transpiledSource, userOutput.kernel.source); - auto ret2 = oklt::util::writeFileAsStr(launcherSource, userOutput.launcher.source); - return ret1 && ret2; - }); + auto ret1 = oklt::util::writeFileAsStr(transpiledSource, output.kernel.source); + auto ret2 = oklt::util::writeFileAsStr(launcherSource, output.launcher.source); + return ret1 && ret2; + }); - transpiler::makeMetadata(launcherMetadata, userOutput.launcher.metadata); - transpiler::makeMetadata(deviceMetadata, userOutput.kernel.metadata); + transpiler::makeMetadata(launcherMetadata, output.launcher.metadata); + transpiler::makeMetadata(deviceMetadata, output.kernel.metadata); + return true; + }; + transpiler::Transpiler transpiler(onSuccess, onFail, onFileNotExists, onWrongBackend); + return transpiler.run(filename, mode, kernelProps); +} +} +#endif - return true; + launchedModeDevice_t::launchedModeDevice_t(const occa::json &properties_) : + modeDevice_t(properties_) { + needsLauncherKernel = true; } -#endif bool launchedModeDevice_t::parseFile(const std::string &filename, const std::string &outputFile, @@ -232,16 +211,17 @@ namespace occa { const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); const std::string launcherOutputFile = hashDir + kc::launcherSourceFile; -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER int transpilerVersion = kernelProps.get("transpiler-version", 2); +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool isValid = false; if(transpilerVersion > 2) { - isValid = transpileFile(sourceFilename, - outputFile, - launcherOutputFile, - kernelProps, - launcherMetadata, - deviceMetadata); + isValid = v3::transpileFile(sourceFilename, + outputFile, + launcherOutputFile, + kernelProps, + launcherMetadata, + deviceMetadata, + mode); } else { isValid = parseFile(sourceFilename, outputFile, @@ -256,6 +236,10 @@ namespace occa { return nullptr; } #else + if(transpilerVersion > 2) { + OCCA_FORCE_ERROR("OCCA compiler is built without BUILD_WITH_CLANG_BASED_TRANSPILER support"); + return nullptr; + } if(!parseFile(sourceFilename, outputFile, launcherOutputFile, diff --git a/src/occa/internal/core/launchedDevice.hpp b/src/occa/internal/core/launchedDevice.hpp index cf88850f9..bc834be44 100644 --- a/src/occa/internal/core/launchedDevice.hpp +++ b/src/occa/internal/core/launchedDevice.hpp @@ -70,15 +70,6 @@ namespace occa { ) = 0; //================================== -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - private: - bool transpileFile(const std::string &filename, - const std::string &outputFile, - const std::string &launcherOutputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &launcherMetadata, - lang::sourceMetadata_t &deviceMetadata); -#endif }; } diff --git a/src/occa/internal/modes/openmp/device.cpp b/src/occa/internal/modes/openmp/device.cpp index c23a09572..d027b139b 100644 --- a/src/occa/internal/modes/openmp/device.cpp +++ b/src/occa/internal/modes/openmp/device.cpp @@ -15,6 +15,57 @@ namespace occa { namespace openmp { + +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER + namespace v3 { + bool transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata, + const std::string &mode) + { + auto onFileNotExists = [](const std::string &file) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << file); + }; + + auto onWrongBackend = [&](const std::string &) { + std::string errorDescription = "Should translate to OpenMP backend"; + OCCA_FORCE_ERROR(errorDescription + + ", unable to transform OKL kernel [" << filename << "]"); + }; + + bool isSilent = kernelProps.get("silent", false); + auto onFail = [&](const std::vector &errors) { + if (!isSilent) { + std::stringstream ss; + ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; + ss << "Transpilation errors occured: " << std::endl; + for(const auto &err: errors) { + ss << err.desc << std::endl; + } + OCCA_FORCE_ERROR(ss.str()); + } + }; + + auto onSuccess = [&](const oklt::UserOutput &output, bool hasLauncher) -> bool { + io::stageFile( + outputFile, + true, + [&](const std::string &tempFilename) -> bool { + std::filesystem::path transpiledSource(tempFilename); + auto ret = oklt::util::writeFileAsStr(tempFilename, output.kernel.source); + return ret.has_value(); + }); + transpiler::makeMetadata(metadata, output.kernel.metadata); + return true; + }; + transpiler::Transpiler transpiler(onSuccess, onFail, onFileNotExists, onWrongBackend); + return transpiler.run(filename, mode, kernelProps); + } + } +#endif + device::device(const occa::json &properties_) : serial::device(properties_) {} @@ -32,65 +83,18 @@ namespace occa { ); } -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - bool device::transpileFile(const std::string &filename, - const std::string &outputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &metadata) - { - auto defines = transpiler::buildDefines(kernelProps); - auto includes = transpiler::buildIncludes(kernelProps); - auto hash = transpiler::getKernelHash(kernelProps); - - std::filesystem::path sourcePath = io::expandFilename(filename); - auto sourceCode = oklt::util::readFileAsStr(sourcePath); - if(!sourceCode) { - std::string errorDescription = "Can't read file: "; - OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); - return false; - } - oklt::UserInput input { - .backend = oklt::TargetBackend::OPENMP, - .source = std::move(sourceCode.value()), - .headers = {}, - .sourcePath = sourcePath, - .includeDirectories = std::move(includes), - .defines = std::move(defines), - .hash = std::move(hash), - }; - auto result = normalizeAndTranspile(std::move(input)); - if(!result) { - if (!kernelProps.get("silent", false)) { - std::stringstream ss; - ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; - ss << "Transpilation errors occured: " << std::endl; - for(const auto &err: result.error()) { - ss << err.desc << std::endl; - } - OCCA_FORCE_ERROR(ss.str()); - } - return false; - } - - auto userOutput = result.value(); - - io::stageFile( - outputFile, - true, - [&](const std::string &tempFilename) -> bool { - std::filesystem::path transpiledSource(tempFilename); - auto ret = oklt::util::writeFileAsStr(tempFilename, userOutput.kernel.source); - return ret.has_value(); - }); - transpiler::makeMetadata(metadata, userOutput.kernel.metadata); - return true; - } -#endif - bool device::parseFile(const std::string &filename, const std::string &outputFile, const occa::json &kernelProps, lang::sourceMetadata_t &metadata) { + +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER + int transpilerVersion = kernelProps.get("transpiler-version", 2); + if(transpilerVersion > 2) { + return v3::transpileFile(filename, outputFile, kernelProps, metadata, mode); + } +#endif + lang::okl::openmpParser parser(kernelProps); parser.parseFile(filename); diff --git a/src/occa/internal/modes/openmp/device.hpp b/src/occa/internal/modes/openmp/device.hpp index 7449a83cb..b821b17ed 100644 --- a/src/occa/internal/modes/openmp/device.hpp +++ b/src/occa/internal/modes/openmp/device.hpp @@ -12,13 +12,6 @@ namespace occa { std::string lastCompiler; std::string lastCompilerOpenMPFlag; -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - bool transpileFile(const std::string &filename, - const std::string &outputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &metadata) override; -#endif - public: device(const occa::json &properties_); virtual ~device() = default; diff --git a/src/occa/internal/modes/serial/device.cpp b/src/occa/internal/modes/serial/device.cpp index c5a189528..8709ca8df 100644 --- a/src/occa/internal/modes/serial/device.cpp +++ b/src/occa/internal/modes/serial/device.cpp @@ -20,6 +20,57 @@ namespace occa { namespace serial { + +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER + namespace v3 { + bool transpileFile(const std::string &filename, + const std::string &outputFile, + const occa::json &kernelProps, + lang::sourceMetadata_t &metadata, + const std::string &mode) + { + auto onFileNotExists = [](const std::string &file) { + std::string errorDescription = "Can't read file: "; + OCCA_FORCE_ERROR(errorDescription << file); + }; + + auto onWrongBackend = [=](const std::string &) { + std::string errorDescription = "Should translate to Serial backend"; + OCCA_FORCE_ERROR(errorDescription + + ", for OKL kernel [" << filename << "]"); + }; + + bool isSilent = kernelProps.get("silent", false); + auto onFail = [=](const std::vector &errors) { + if (!isSilent) { + std::stringstream ss; + ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; + ss << "Transpilation errors occured: " << std::endl; + for(const auto &err: errors) { + ss << err.desc << std::endl; + } + OCCA_FORCE_ERROR(ss.str()); + } + }; + + auto onSuccess = [&](const oklt::UserOutput &output, bool hasLauncher) -> bool { + io::stageFile( + outputFile, + true, + [&](const std::string &tempFilename) -> bool { + std::filesystem::path transpiledSource(tempFilename); + auto ret = oklt::util::writeFileAsStr(tempFilename, output.kernel.source); + return ret.has_value(); + }); + transpiler::makeMetadata(metadata, output.kernel.metadata); + return true; + }; + transpiler::Transpiler transpiler(onSuccess, onFail, onFileNotExists, onWrongBackend); + return transpiler.run(filename, mode, kernelProps); + } + } +#endif + device::device(const occa::json &properties_) : occa::modeDevice_t(properties_) { // TODO: Maybe theres something more descriptive we can populate here @@ -78,60 +129,6 @@ namespace occa { return (srEndTag->time - srStartTag->time); } -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - bool device::transpileFile(const std::string &filename, - const std::string &outputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &metadata) - { - auto defines = transpiler::buildDefines(kernelProps); - auto includes = transpiler::buildIncludes(kernelProps); - - std::filesystem::path sourcePath = io::expandFilename(filename); - auto sourceCode = oklt::util::readFileAsStr(sourcePath); - if(!sourceCode) { - std::string errorDescription = "Can't read file: "; - OCCA_FORCE_ERROR(errorDescription << sourcePath.string()); - return false; - } - oklt::UserInput input { - .backend = oklt::TargetBackend::SERIAL, - .source = std::move(sourceCode.value()), - .headers = {}, - .sourcePath = sourcePath, - .includeDirectories = std::move(includes), - .defines = std::move(defines), - .hash = "", - }; - auto result = normalizeAndTranspile(std::move(input)); - if(!result) { - if (!kernelProps.get("silent", false)) { - std::stringstream ss; - ss << "Unable to transform OKL kernel [" << filename << "]" << std::endl; - ss << "Transpilation errors occured: " << std::endl; - for(const auto &err: result.error()) { - ss << err.desc << std::endl; - } - OCCA_FORCE_ERROR(ss.str()); - } - return false; - } - - auto userOutput = result.value(); - - io::stageFile( - outputFile, - true, - [&](const std::string &tempFilename) -> bool { - std::filesystem::path transpiledSourcePath(tempFilename); - auto ret = oklt::util::writeFileAsStr(transpiledSourcePath, userOutput.kernel.source); - return ret.has_value(); - }); - transpiler::makeMetadata(metadata, userOutput.kernel.metadata); - return true; - } -#endif - //================================== //---[ Kernel ]--------------------- @@ -349,15 +346,15 @@ namespace occa { if (compilingOkl) { const std::string outputFile = hashDir + kc::cachedSourceFilename(filename); -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER int transpilerVersion = kernelProps.get("transpiler-version", 2); - +#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER bool valid = false; if(transpilerVersion > 2) { - valid = transpileFile(sourceFilename, - outputFile, - kernelProps, - metadata); + valid = v3::transpileFile(sourceFilename, + outputFile, + kernelProps, + metadata, + mode); } else { valid = parseFile(sourceFilename, outputFile, @@ -370,6 +367,11 @@ namespace occa { return nullptr; } #else + if(transpilerVersion > 2) { + OCCA_FORCE_ERROR("OCCA compiler is built without BUILD_WITH_CLANG_BASED_TRANSPILER support"); + return nullptr; + } + if(!parseFile(sourceFilename, outputFile, kernelProps, diff --git a/src/occa/internal/modes/serial/device.hpp b/src/occa/internal/modes/serial/device.hpp index f65515bf1..b3a7a5c03 100644 --- a/src/occa/internal/modes/serial/device.hpp +++ b/src/occa/internal/modes/serial/device.hpp @@ -76,13 +76,6 @@ namespace occa { void* unwrap() override; -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - private: - virtual bool transpileFile(const std::string &filename, - const std::string &outputFile, - const occa::json &kernelProps, - lang::sourceMetadata_t &metadata); -#endif }; } } diff --git a/src/occa/internal/utils/transpiler_utils.cpp b/src/occa/internal/utils/transpiler_utils.cpp index ebece02f1..56beade48 100644 --- a/src/occa/internal/utils/transpiler_utils.cpp +++ b/src/occa/internal/utils/transpiler_utils.cpp @@ -2,18 +2,9 @@ #include #include #include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include - - #ifdef BUILD_WITH_CLANG_BASED_TRANSPILER #include #include @@ -114,211 +105,71 @@ void makeMetadata(lang::sourceMetadata_t &sourceMetadata, } } - -namespace v2 { - bool runTranspiler(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode) - { - using ParserBuildFunc = std::function(const json ¶ms)>; - static const std::map originalParserBackends = - { - {"", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"serial", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"openmp", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"cuda", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"hip", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"opencl", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"metal", [](const json ¶ms) { - return std::make_unique(params); - }}, - {"dpcpp", [](const json ¶ms) { - return std::make_unique(params); - }} - }; - - const bool printLauncher = options["launcher"]; - const std::string filename = arguments[0]; - - if (!io::exists(filename)) { - printError("File [" + filename + "] doesn't exist" ); - ::exit(1); - } - - auto parserIt = originalParserBackends.find(mode); - if(parserIt == originalParserBackends.end()) { - printError("Unable to translate for mode [" + originalMode + "]"); - ::exit(1); - } - - std::unique_ptr parser = parserIt->second(kernelProps); - parser->parseFile(filename); - - bool success = parser->succeeded(); - if (!success) { - ::exit(1); - } - - if (options["verbose"]) { - json translationInfo; - // Filename - translationInfo["translate_info/filename"] = io::expandFilename(filename); - // Date information - translationInfo["translate_info/date"] = sys::date(); - translationInfo["translate_info/human_date"] = sys::humanDate(); - // Version information - translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; - translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; - // Kernel properties - translationInfo["kernel_properties"] = kernelProps; - - io::stdout - << "/* Translation Info:\n" - << translationInfo - << "*/\n"; - } - - if (printLauncher && ((mode == "cuda") - || (mode == "hip") - || (mode == "opencl") - || (mode == "dpcpp") - || (mode == "metal"))) { - lang::parser_t *launcherParser = &(((occa::lang::okl::withLauncher*) parser.get())->launcherParser); - io::stdout << launcherParser->toString(); - } else { - io::stdout << parser->toString(); - } - return true; +#if BUILD_WITH_CLANG_BASED_TRANSPILER +Transpiler::Transpiler(SuccessFunc success, + FailFunc fail, + WrongInputFile wrongInputFile, + WrongBackend wrongBackend) + :_success(std::move(success)) + , _fail(std::move(fail)) + , _wrongInput(std::move(wrongInputFile)) + , _wrongBackend(std::move(wrongBackend)) +{} + +bool Transpiler::run(const std::string &filename, + const std::string &mode, + const occa::json &kernelProps) +{ + static const std::map targetBackends = + { + {"openmp", oklt::TargetBackend::OPENMP}, + {"cuda", oklt::TargetBackend::CUDA}, + {"hip", oklt::TargetBackend::HIP}, + {"dpcpp", oklt::TargetBackend::DPCPP}, + {"serial", oklt::TargetBackend::SERIAL}, + }; + + std::string normalizedMode = lowercase(mode); + auto backend = targetBackends.find(normalizedMode); + if(backend == targetBackends.end()) { + _wrongBackend(mode); + return false; } -} - -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER -namespace v3 { - bool runTranspiler(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode) - { - static const std::map targetBackends = - { - {"openmp", oklt::TargetBackend::OPENMP}, - {"cuda", oklt::TargetBackend::CUDA}, - {"hip", oklt::TargetBackend::HIP}, - {"dpcpp", oklt::TargetBackend::DPCPP}, - {"serial", oklt::TargetBackend::SERIAL}, - }; - - const bool printLauncher = options["launcher"]; - const std::string filename = arguments[0]; - - if (!io::exists(filename)) { - printError("File [" + filename + "] doesn't exist" ); - ::exit(1); - } - - auto transpiler = targetBackends.find(mode); - if(transpiler == targetBackends.end()) { - printError("Unable to translate for mode [" + originalMode + "]"); - ::exit(1); - } - - auto defines = transpiler::buildDefines(kernelProps); - auto includes = transpiler::buildIncludes(kernelProps); - auto hash = transpiler::getKernelHash(kernelProps); - - std::filesystem::path sourcePath = io::expandFilename(filename); - auto sourceCode = oklt::util::readFileAsStr(sourcePath); - if(!sourceCode) { - printError("Can't open file: " + sourcePath.string()); - ::exit(sourceCode.error()); - } - oklt::UserInput input { - .backend = transpiler->second, - .source = std::move(sourceCode.value()), - .headers = {}, - .sourcePath = sourcePath, - .includeDirectories = std::move(includes), - .defines = std::move(defines), - .hash = std::move(hash) - }; - auto result = normalizeAndTranspile(std::move(input)); - - if(!result) { - std::stringstream ss; - for(const auto &err: result.error()) { - ss << err.desc << std::endl; - } - printError(ss.str()); - ::exit(1); - } - - if (options["verbose"]) { - json translationInfo; - // Filename - translationInfo["translate_info/filename"] = io::expandFilename(filename); - // Date information - translationInfo["translate_info/date"] = sys::date(); - translationInfo["translate_info/human_date"] = sys::humanDate(); - // Version information - translationInfo["translate_info/occa_version"] = OCCA_VERSION_STR; - translationInfo["translate_info/okl_version"] = OKL_VERSION_STR; - // Kernel properties - translationInfo["kernel_properties"] = kernelProps; - - io::stdout - << "/* Translation Info:\n" - << translationInfo - << "*/\n"; - } - - auto userOutput = result.value(); - bool hasLauncher = transpiler->second == oklt::TargetBackend::CUDA || - transpiler->second == oklt::TargetBackend::HIP || - transpiler->second == oklt::TargetBackend::DPCPP; - if(printLauncher && hasLauncher) { - io::stdout << userOutput.launcher.source; - } else { - io::stdout << userOutput.kernel.source; - } - - return true; + auto expandedFile = io::expandFilename(filename); + if (!io::exists(filename)) { + _wrongInput(filename); + return false; } -} -#endif - -bool runTranspiler(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode) -{ - int transpilerVersion = getTranspilerVersion(options); -#ifdef BUILD_WITH_CLANG_BASED_TRANSPILER - if(transpilerVersion > 2) { - return v3::runTranspiler(options, arguments, kernelProps, originalMode, mode); + auto sourceCode = oklt::util::readFileAsStr(expandedFile); + if(!sourceCode) { + _wrongInput(filename); + return false; } -#endif - if (transpilerVersion > 2) { - printError("OCCA compiler is built without BUILD_WITH_CLANG_BASED_TRANSPILER support"); + + auto defines = transpiler::buildDefines(kernelProps); + auto includes = transpiler::buildIncludes(kernelProps); + auto hash = transpiler::getKernelHash(kernelProps); + + oklt::UserInput input { + .backend = backend->second, + .source = std::move(sourceCode.value()), + .headers = {}, + .sourcePath = expandedFile, + .includeDirectories = std::move(includes), + .defines = std::move(defines), + .hash = std::move(hash) + }; + auto result = normalizeAndTranspile(std::move(input)); + if(!result) { + _fail(result.error()); return false; } - return v2::runTranspiler(options, arguments, kernelProps, originalMode, mode); + bool hasLauncher = backend->second == oklt::TargetBackend::CUDA || + backend->second == oklt::TargetBackend::HIP || + backend->second == oklt::TargetBackend::DPCPP; + auto userOutput = result.value(); + return _success(userOutput, hasLauncher); } - +#endif } } diff --git a/src/occa/internal/utils/transpiler_utils.h b/src/occa/internal/utils/transpiler_utils.h index 6dfbd6880..5ccf8d12d 100644 --- a/src/occa/internal/utils/transpiler_utils.h +++ b/src/occa/internal/utils/transpiler_utils.h @@ -3,23 +3,52 @@ #include #include #include +#include + +#if BUILD_WITH_CLANG_BASED_TRANSPILER +#include +#include +#endif namespace occa { namespace transpiler { int getTranspilerVersion(const json &options); -std::string getKernelHash(const json &kernelProp); -std::vector buildDefines(const json &kernelProp); -std::vector buildIncludes(const json &kernelProp); +//std::string getKernelHash(const json &kernelProp); +//std::vector buildDefines(const json &kernelProp); +//std::vector buildIncludes(const json &kernelProp); void makeMetadata(lang::sourceMetadata_t &sourceMetadata, const std::string &jsonStr); -bool runTranspiler(const json &options, - const json &arguments, - const json &kernelProps, - const std::string &originalMode, - const std::string &mode); + +#if BUILD_WITH_CLANG_BASED_TRANSPILER +struct Transpiler { + using SuccessFunc = std::function; + using FailFunc = std::function &errors)>; + using WrongInputFile = std::function; + using WrongBackend = std::function; + + Transpiler(SuccessFunc success, + FailFunc fail, + WrongInputFile wrongInputFile, + WrongBackend wrongBackend); + ~Transpiler() = default; + Transpiler(const Transpiler&) = delete; + Transpiler & operator = (const Transpiler &) = delete; + Transpiler(Transpiler &&) = delete; + Transpiler & operator = (Transpiler &&) = delete; + + bool run(const std::string &filename, + const std::string &mode, + const occa::json &kernelProps); +private: + SuccessFunc _success; + FailFunc _fail; + WrongInputFile _wrongInput; + WrongBackend _wrongBackend; +}; +#endif } From 0892ad7e6657819ad2af138cbdc1b8cc222f0806 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Fri, 10 May 2024 19:06:32 +0300 Subject: [PATCH 05/25] make unchanged files unchanged --- src/occa/internal/core/launchedDevice.hpp | 1 - src/occa/internal/modes/serial/device.hpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/occa/internal/core/launchedDevice.hpp b/src/occa/internal/core/launchedDevice.hpp index bc834be44..a8e319f10 100644 --- a/src/occa/internal/core/launchedDevice.hpp +++ b/src/occa/internal/core/launchedDevice.hpp @@ -69,7 +69,6 @@ namespace occa { const occa::json &kernelProps ) = 0; //================================== - }; } diff --git a/src/occa/internal/modes/serial/device.hpp b/src/occa/internal/modes/serial/device.hpp index b3a7a5c03..5796c1a49 100644 --- a/src/occa/internal/modes/serial/device.hpp +++ b/src/occa/internal/modes/serial/device.hpp @@ -75,7 +75,6 @@ namespace occa { //================================ void* unwrap() override; - }; } } From 8e2f4a0c903cd965aadd2cd3ffb450486cd92ab5 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Tue, 14 May 2024 13:17:39 +0300 Subject: [PATCH 06/25] fix hipDeviceProp_t type to be the same as original HIP & revert back buildIncludes implementation --- src/occa/internal/modes/hip/polyfill.hpp | 5 +-- src/occa/internal/modes/hip/registration.cpp | 2 +- src/occa/internal/utils/transpiler_utils.cpp | 36 ++++++++------------ 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/occa/internal/modes/hip/polyfill.hpp b/src/occa/internal/modes/hip/polyfill.hpp index e6d933fc4..d8483950e 100644 --- a/src/occa/internal/modes/hip/polyfill.hpp +++ b/src/occa/internal/modes/hip/polyfill.hpp @@ -34,7 +34,8 @@ namespace occa { class hipDeviceProp_t { public: - char *name; + //INFO: original HIP has exact this definition of name field + char name[256]; size_t totalGlobalMem; int maxThreadsPerBlock; char gcnArchName[256]; @@ -42,7 +43,7 @@ namespace occa { int minor; inline hipDeviceProp_t() : - name(NULL), + name{0}, totalGlobalMem(0), maxThreadsPerBlock(-1), major(-1), diff --git a/src/occa/internal/modes/hip/registration.cpp b/src/occa/internal/modes/hip/registration.cpp index 9e4e84623..7322eff9b 100644 --- a/src/occa/internal/modes/hip/registration.cpp +++ b/src/occa/internal/modes/hip/registration.cpp @@ -25,7 +25,7 @@ namespace occa { hipDeviceProp_t props; OCCA_HIP_ERROR("Getting device properties", hipGetDeviceProperties(&props, deviceId)); - if (props.name != NULL) { + if (std::strlen(props.name) != 0) { strcpy(deviceName, props.name); } diff --git a/src/occa/internal/utils/transpiler_utils.cpp b/src/occa/internal/utils/transpiler_utils.cpp index 56beade48..27cec83c4 100644 --- a/src/occa/internal/utils/transpiler_utils.cpp +++ b/src/occa/internal/utils/transpiler_utils.cpp @@ -66,19 +66,16 @@ std::vector buildDefines(const json &kernelProp) { std::vector buildIncludes(const json &kernelProp) { std::vector includes; json oklIncludePaths = kernelProp.get("okl/include_paths", json{}); - if (!oklIncludePaths.isArray()) { - return {}; - } - - jsonArray pathArray = oklIncludePaths.array(); - const int pathCount = (int) pathArray.size(); - for (int i = 0; i < pathCount; ++i) { - json path = pathArray[i]; - if (path.isString()) { - includes.push_back(std::filesystem::path(path.string())); + if (oklIncludePaths.isArray()) { + jsonArray pathArray = oklIncludePaths.array(); + const int pathCount = (int) pathArray.size(); + for (int i = 0; i < pathCount; ++i) { + json path = pathArray[i]; + if (path.isString()) { + includes.push_back(std::filesystem::path(path.string())); + } } } - auto &envIncludes = env::OCCA_INCLUDE_PATH; for(const auto &includePath: envIncludes) { includes.push_back(includePath); @@ -89,19 +86,16 @@ std::vector buildIncludes(const json &kernelProp) { void makeMetadata(lang::sourceMetadata_t &sourceMetadata, const std::string &jsonStr) { - lang::kernelMetadataMap &metadataMap = sourceMetadata.kernelsMetadata; auto json = json::parse(jsonStr); auto metadataObj = json.get("metadata"); - if(!metadataObj.isArray()) { - printError("Can't get the metadata"); - return; - } - jsonArray metaArr = metadataObj.asArray().array(); - for(const auto &elem: metaArr) { - auto name = elem.get("name"); - auto kernelObj = lang::kernelMetadata_t::fromJson(elem); - metadataMap.insert(std::make_pair(name, kernelObj)); + if(metadataObj.isArray()) { + jsonArray metaArr = metadataObj.asArray().array(); + for(const auto &elem: metaArr) { + auto name = elem.get("name"); + auto kernelObj = lang::kernelMetadata_t::fromJson(elem); + metadataMap.insert(std::make_pair(name, kernelObj)); + } } } From 3e0f167f099ebaca34693d435f38aef1e234d580 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Tue, 14 May 2024 16:31:37 +0300 Subject: [PATCH 07/25] fix package build without occa-transpiler --- src/occa/internal/bin/occa.cpp | 6 ++---- src/occa/internal/utils/transpiler_utils.h | 4 ---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index d713e4e96..4e1d41c59 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -18,10 +18,7 @@ #include #include -#if BUILD_WITH_CLANG_BASED_TRANSPILER #include -#endif - #include @@ -226,6 +223,7 @@ namespace occa { } } +#if BUILD_WITH_CLANG_BASED_TRANSPILER namespace v3 { bool runTranspiler(const json &options, const json &arguments, @@ -289,7 +287,7 @@ namespace occa { return transpiler.run(filename, mode, kernelProps); } } - +#endif bool runTranspiler(const json &options, const json &arguments, const json &kernelProps, diff --git a/src/occa/internal/utils/transpiler_utils.h b/src/occa/internal/utils/transpiler_utils.h index 5ccf8d12d..d4593d092 100644 --- a/src/occa/internal/utils/transpiler_utils.h +++ b/src/occa/internal/utils/transpiler_utils.h @@ -14,10 +14,6 @@ namespace occa { namespace transpiler { int getTranspilerVersion(const json &options); - -//std::string getKernelHash(const json &kernelProp); -//std::vector buildDefines(const json &kernelProp); -//std::vector buildIncludes(const json &kernelProp); void makeMetadata(lang::sourceMetadata_t &sourceMetadata, const std::string &jsonStr); From 948425516d4a1f345c5618e747b2422ae3503c3c Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Tue, 14 May 2024 20:19:02 +0300 Subject: [PATCH 08/25] update occa-transpiler version to v1.1 --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index 9015b78fe..c1b993a62 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit 9015b78fe98e7baa26ea126127bc8fa5fedfc704 +Subproject commit c1b993a62af9242ab1a37320619dba54ef03ac62 From 1728fe7e0c237d570eddb1d202c3d2bd519c9833 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Wed, 15 May 2024 14:40:32 +0300 Subject: [PATCH 09/25] update occa-transpiler to latest devel(fix cuda/hip intrinsics) --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index c1b993a62..f5495d403 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit c1b993a62af9242ab1a37320619dba54ef03ac62 +Subproject commit f5495d403413083b680f894c572f42f770d35d72 From 7a98cc1c29d4fee769f497690f8f1eb5b555f4fe Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Wed, 15 May 2024 18:20:15 +0300 Subject: [PATCH 10/25] update occa-transpiler taggeed version --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index f5495d403..d3d3bffad 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit f5495d403413083b680f894c572f42f770d35d72 +Subproject commit d3d3bffadb673c7f5ef65e55af5591d2baf53c54 From 6186901ba8d9c962d2becd5a48eb558de7d35259 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Thu, 16 May 2024 19:43:03 +0300 Subject: [PATCH 11/25] move to tag v1.1 occa-transpiler --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index d3d3bffad..f063ed87c 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit d3d3bffadb673c7f5ef65e55af5591d2baf53c54 +Subproject commit f063ed87c5a881eed89b0f8b551e7c9627b22d0b From 85b3c3580fbb3da52429c247ce2761d4e398962b Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Mon, 20 May 2024 10:42:57 +0300 Subject: [PATCH 12/25] added example with occa-transpiler and C++ featured okl kernel --- .../cpp/31_oklt_v3_moving_avg/CMakeLists.txt | 10 ++ .../cpp/31_oklt_v3_moving_avg/constants.h | 5 + examples/cpp/31_oklt_v3_moving_avg/main.cpp | 93 +++++++++++++++++++ .../31_oklt_v3_moving_avg/movingAverage.okl | 79 ++++++++++++++++ examples/cpp/CMakeLists.txt | 4 + 5 files changed, 191 insertions(+) create mode 100644 examples/cpp/31_oklt_v3_moving_avg/CMakeLists.txt create mode 100644 examples/cpp/31_oklt_v3_moving_avg/constants.h create mode 100644 examples/cpp/31_oklt_v3_moving_avg/main.cpp create mode 100644 examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl diff --git a/examples/cpp/31_oklt_v3_moving_avg/CMakeLists.txt b/examples/cpp/31_oklt_v3_moving_avg/CMakeLists.txt new file mode 100644 index 000000000..6c319e147 --- /dev/null +++ b/examples/cpp/31_oklt_v3_moving_avg/CMakeLists.txt @@ -0,0 +1,10 @@ +compile_cpp_example_with_modes(oklt_v3_moving_avg main.cpp) + +add_custom_target(cpp_example_oklt_v3_moving_avg_cpy ALL + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/constants.h constants.h + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/movingAverage.okl movingAverage.okl) +add_dependencies(examples_cpp_oklt_v3_moving_avg cpp_example_oklt_v3_moving_avg_cpy) +target_sources(examples_cpp_oklt_v3_moving_avg + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/movingAverage.okl + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/constants.h + ) diff --git a/examples/cpp/31_oklt_v3_moving_avg/constants.h b/examples/cpp/31_oklt_v3_moving_avg/constants.h new file mode 100644 index 000000000..0406ad392 --- /dev/null +++ b/examples/cpp/31_oklt_v3_moving_avg/constants.h @@ -0,0 +1,5 @@ +#pragma once + +constexpr const int THREADS_PER_BLOCK = 1024; +//INFO: it's not possible to setup dynamicaly extern @shared array for CUDA +constexpr const int WINDOW_SIZE = 16; diff --git a/examples/cpp/31_oklt_v3_moving_avg/main.cpp b/examples/cpp/31_oklt_v3_moving_avg/main.cpp new file mode 100644 index 000000000..08c736534 --- /dev/null +++ b/examples/cpp/31_oklt_v3_moving_avg/main.cpp @@ -0,0 +1,93 @@ +#include +#include +#include +#include "constants.h" + +std::vector buildData(std::size_t size, + float initialValue, + float fluctuation) +{ + std::vector buffer(size); + float currentValue = initialValue; + float longIncrement = 1.0f; + float fluctuationIncrement = fluctuation; + for(std::size_t i = 0; i < buffer.size(); ++i) { + buffer[i] = currentValue; + fluctuationIncrement = -fluctuationIncrement; + if(i % WINDOW_SIZE == 0) { + longIncrement = -longIncrement; + } + currentValue += longIncrement + fluctuationIncrement; + } + return buffer; +} + +std::vector goldMovingAverage(const std::vector &hostVector) { + std::vector result(hostVector.size() - WINDOW_SIZE); + for(std::size_t i = 0; i < result.size(); ++i) { + float value = 0.0f; + for(std::size_t j = 0; j < WINDOW_SIZE; ++j) { + value += hostVector[i + j]; + } + result[i] = value / WINDOW_SIZE; + } + return result; +} + +bool starts_with(const std::string &str, const std::string &substring) { + return str.rfind(substring, 0) == 0; +} + +occa::json getDeviceOptions(int argc, const char **argv) { + for(int i = 0; i < argc; ++i) { + std::string argument(argv[i]); + if((starts_with(argument,"-d") || starts_with(argument, "--device")) && i + 1 < argc) + { + std::string value(argv[i + 1]); + return occa::json::parse(value); + } + } + return occa::json::parse("{mode: 'Serial'}"); +} + +int main(int argc, const char **argv) { + + occa::json deviceOpts = getDeviceOptions(argc, argv); + auto inputHostBuffer = buildData(THREADS_PER_BLOCK * WINDOW_SIZE + WINDOW_SIZE, 10.0f, 4.0f); + std::vector outputHostBuffer(inputHostBuffer.size() - WINDOW_SIZE); + + occa::device device(deviceOpts); + occa::memory deviceInput = device.malloc(inputHostBuffer.size()); + occa::memory deviceOutput = device.malloc(outputHostBuffer.size()); + + occa::json buildProps({ + {"transpiler-version", 3} + }); + + occa::kernel movingAverageKernel = device.buildKernel("movingAverage.okl", "movingAverage32f", buildProps); + + deviceInput.copyFrom(inputHostBuffer.data(), inputHostBuffer.size()); + + movingAverageKernel(deviceInput, + static_cast(inputHostBuffer.size()), + deviceOutput, + static_cast(deviceOutput.size())); + + // Copy result to the host + deviceOutput.copyTo(&outputHostBuffer[0], outputHostBuffer.size()); + + auto goldValue = goldMovingAverage(inputHostBuffer); + + constexpr const float EPSILON = 0.001f; + for(std::size_t i = 0; i < outputHostBuffer.size(); ++i) { + bool isValid = std::abs(std::abs(goldValue[i]) - std::abs(outputHostBuffer[i])) < EPSILON; + if(!isValid) { + std::cout << "Comparison with gold values has failed" << std::endl; + return 1; + } + } + std::cout << "Comparison with gold has passed" << std::endl; + std::cout << "Moving average finished" << std::endl; + + return 0; +} diff --git a/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl b/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl new file mode 100644 index 000000000..a71ae6d4b --- /dev/null +++ b/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl @@ -0,0 +1,79 @@ +#include "constants.h" + +template +struct MovingAverage { + MovingAverage(int inputSize, + int outputSize, + T *shared_input, + T *shared_output) + :_inputSize(inputSize) + ,_outputSize(outputSize) + ,_shared_data(shared_input) + ,_result_data(shared_output) + {} + + void syncCopyFrom(const T *input, int block_idx, int thread_idx) { + int linearIdx = block_idx * THREADS + thread_idx; + //INFO: copy base chunk + if(linearIdx < _inputSize) { + _shared_data[thread_idx] = input[linearIdx]; + } + //INFO: copy WINDOW chunk + int tailIdx = (block_idx + 1) * THREADS + thread_idx; + if(tailIdx < _inputSize && thread_idx < WINDOW) { + _shared_data[THREADS + thread_idx] = input[tailIdx]; + } + @barrier; + } + + void process(int thread_idx) { + T sum = T(); + for(int i = 0; i < WINDOW; ++i) { + sum += _shared_data[thread_idx + i]; + } + _result_data[thread_idx] = sum / WINDOW; + @barrier; + } + + void syncCopyTo(T *output, int block_idx, int thread_idx) { + int linearIdx = block_idx * THREADS + thread_idx; + if(linearIdx < _outputSize) { + output[linearIdx] = _result_data[thread_idx]; + } + @barrier; + } +private: + int _inputSize; + int _outputSize; + + //INFO: not supported + // @shared T _data[THREADS_PER_BLOCK + WINDOW_SIZE]; + // @shared T _result[THREADS_PER_BLOCK]; + + T *_shared_data; + T *_result_data; +}; + +@kernel void movingAverage32f(@restrict const float *inputData, + int inputSize, + @restrict float *outputData, + int outputSize) +{ + @outer(0) for (int block_idx = 0; block_idx < outputSize / THREADS_PER_BLOCK + 1; ++block_idx) { + @shared float blockInput[THREADS_PER_BLOCK + WINDOW_SIZE]; + @shared float blockResult[THREADS_PER_BLOCK]; + MovingAverage ma{ + inputSize, + outputSize, + blockInput, + blockResult + }; + @inner(0) for(int thread_idx = 0; thread_idx < THREADS_PER_BLOCK; ++thread_idx) { + ma.syncCopyFrom(inputData, block_idx, thread_idx); + ma.process(thread_idx); + ma.syncCopyTo(outputData, block_idx, thread_idx); + } + } +} diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 6c51ab1e9..466e33f99 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -18,6 +18,10 @@ add_subdirectory(19_stream_tags) add_subdirectory(20_native_dpcpp_kernel) add_subdirectory(30_device_function) + +if (OCCA_CLANG_BASED_TRANSPILER) + add_subdirectory(31_oklt_v3_moving_avg) +endif() # Don't force-compile OpenGL examples # add_subdirectory(16_finite_difference) # add_subdirectory(17_mandelbulb) From 576738968b2e75dee0b6cd1130fccdd710668e3a Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Fri, 7 Jun 2024 17:04:41 +0300 Subject: [PATCH 13/25] fixes for code review, move getTranspilerVersion from options to bin/occa.cpp as local function --- cmake/GitSubmodules.cmake | 2 +- examples/cpp/31_oklt_v3_moving_avg/main.cpp | 2 +- src/occa/internal/bin/occa.cpp | 27 ++++++++++++++++++-- src/occa/internal/utils/transpiler_utils.cpp | 21 --------------- src/occa/internal/utils/transpiler_utils.h | 1 - 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/cmake/GitSubmodules.cmake b/cmake/GitSubmodules.cmake index 8944e1fb8..c20392211 100644 --- a/cmake/GitSubmodules.cmake +++ b/cmake/GitSubmodules.cmake @@ -1,5 +1,5 @@ #################################################################################################### -# This function recursivelly update git submodules. +# This function recursively update git submodules. # Params: project base directory # Example: # init_submodules(${PROJECT_SOURCE_DIR}) diff --git a/examples/cpp/31_oklt_v3_moving_avg/main.cpp b/examples/cpp/31_oklt_v3_moving_avg/main.cpp index 08c736534..462024f63 100644 --- a/examples/cpp/31_oklt_v3_moving_avg/main.cpp +++ b/examples/cpp/31_oklt_v3_moving_avg/main.cpp @@ -80,7 +80,7 @@ int main(int argc, const char **argv) { constexpr const float EPSILON = 0.001f; for(std::size_t i = 0; i < outputHostBuffer.size(); ++i) { - bool isValid = std::abs(std::abs(goldValue[i]) - std::abs(outputHostBuffer[i])) < EPSILON; + bool isValid = std::abs(goldValue[i] - outputHostBuffer[i]) < EPSILON; if(!isValid) { std::cout << "Comparison with gold values has failed" << std::endl; return 1; diff --git a/src/occa/internal/bin/occa.cpp b/src/occa/internal/bin/occa.cpp index 4e1d41c59..315daec2c 100644 --- a/src/occa/internal/bin/occa.cpp +++ b/src/occa/internal/bin/occa.cpp @@ -18,7 +18,9 @@ #include #include +#if BUILD_WITH_CLANG_BASED_TRANSPILER #include +#endif #include @@ -132,6 +134,27 @@ namespace occa { return true; } + int getTranspilerVersion(const json &options) { + json jsonTranspileVersion = options["transpiler-version"]; + int transpilerVersion = 2; + //INFO: have no idea why json here has array type + if(!jsonTranspileVersion.isArray()) { + return transpilerVersion; + } + json elem = jsonTranspileVersion.asArray()[0]; + if(!elem.isString()) { + return transpilerVersion; + } + + try { + transpilerVersion = std::stoi(elem.string()); + } catch(const std::exception &) + { + return transpilerVersion; + } + return transpilerVersion; + } + namespace v2 { bool runTranspiler(const json &options, const json &arguments, @@ -294,7 +317,7 @@ namespace occa { const std::string &originalMode, const std::string &mode) { - int transpilerVersion = transpiler::getTranspilerVersion(options); + int transpilerVersion = getTranspilerVersion(options); #ifdef BUILD_WITH_CLANG_BASED_TRANSPILER if(transpilerVersion > 2) { return v3::runTranspiler(options, arguments, kernelProps, originalMode, mode); @@ -340,7 +363,7 @@ namespace occa { kernelProps["verbose"] = kernelProps.get("verbose", true); kernelProps["okl/include_paths"] = options["include-path"]; kernelProps["defines"].asObject() += getOptionDefines(options["define"]); - kernelProps["transpiler-version"] = transpiler::getTranspilerVersion(options); + kernelProps["transpiler-version"] = getTranspilerVersion(options); device device(deviceProps); device.buildKernel(filename, kernelName, kernelProps); diff --git a/src/occa/internal/utils/transpiler_utils.cpp b/src/occa/internal/utils/transpiler_utils.cpp index 27cec83c4..ba345ac24 100644 --- a/src/occa/internal/utils/transpiler_utils.cpp +++ b/src/occa/internal/utils/transpiler_utils.cpp @@ -14,27 +14,6 @@ namespace occa { namespace transpiler { -int getTranspilerVersion(const json &options) { - json jsonTranspileVersion = options["transpiler-version"]; - int transpilerVersion = 2; - //INFO: have no idea why json here has array type - if(!jsonTranspileVersion.isArray()) { - return transpilerVersion; - } - json elem = jsonTranspileVersion.asArray()[0]; - if(!elem.isString()) { - return transpilerVersion; - } - - try { - transpilerVersion = std::stoi(elem.string()); - } catch(const std::exception &) - { - return transpilerVersion; - } - return transpilerVersion; -} - std::string getKernelHash(const json &kernelProp) { auto hashStr = kernelProp.get("hash"); if(hashStr.empty()) { diff --git a/src/occa/internal/utils/transpiler_utils.h b/src/occa/internal/utils/transpiler_utils.h index d4593d092..b1ca25b41 100644 --- a/src/occa/internal/utils/transpiler_utils.h +++ b/src/occa/internal/utils/transpiler_utils.h @@ -13,7 +13,6 @@ namespace occa { namespace transpiler { -int getTranspilerVersion(const json &options); void makeMetadata(lang::sourceMetadata_t &sourceMetadata, const std::string &jsonStr); From 415ef2595783c88b537f17316eacbffb7a5df025 Mon Sep 17 00:00:00 2001 From: vyast-softserveinc Date: Wed, 26 Jun 2024 14:55:07 +0300 Subject: [PATCH 14/25] update INSTALL.md & README.md documentation files --- INSTALL.md | 23 ++++++++++++++++++++++- README.md | 9 --------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 13de75448..0c6198fa3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -16,6 +16,7 @@ - SYCL 2020 or later - OpenCL 2.0 or later - OpenMP 4.0 or later + - Support Clang based transpiler ## Linux @@ -51,6 +52,7 @@ $ CC=clang CXX=clang++ OCCA_ENABLE_OPENMP="OFF" ./configure-cmake.sh #### Dependency Paths + The following environment variables can be used to specify the path to third-party dependencies needed by different OCCA backends. The value assigned should be an absolute path to the parent directory, which typically contains subdirectories `bin`, `include`, and `lib`. | Backend | Environment Variable | Description | @@ -67,7 +69,26 @@ After CMake configuration is complete, OCCA can be built with the command $ cmake --build build --parallel ``` -When cross compiling for a different platform, the targeted hardware doesn't need to be available; however all dependencies—e.g., headers, libraries—must be present. Commonly this is the case for large HPC systems, where code is compiled on login nodes and run on compute nodes. +When cross compiling for a different platform, the targeted hardware doesn't need to be available; however all dependencies—e.g., headers, libraries—must be present. Commonly this is the case for large HPC systems, where code is compiled on login nodes and run on compute nodes. + + +#### Building with Clang transplier option + + +Hard dependecy is clang-17. How to install it please refer to the original +[clang occa-transpiler](https://github.com/libocca/occa-transpiler/blob/main/README.md) + +The rest dependecies are represented as git submodules and are fetched automatically by cmake script. +Building the project with clang based transpiler now is supported only by *CMake* build system. +All options must be provided directly. Here is the following example: + +```shell +$ mkdir build +$ cd build +$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON .. +$ cmake --build . --parallel +$ cmake --install . --prefix install +``` ### Testing diff --git a/README.md b/README.md index c88548e8a..a1d4826aa 100644 --- a/README.md +++ b/README.md @@ -68,15 +68,6 @@ $ cmake --install build --prefix install If dependencies are installed in a non-standard location, set the corresponding [environment variable](INSTALL.md#dependency-paths) to this path. -For building with C++ support/occa-transpiler: -```shell -$ mkdir build -$ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON .. -$ cmake --build . --parallel -$ cmake --install . --prefix install -``` - ## Use ### Environment From 73f9381c2821a3a75a5f6d16fb6c8602d194a6df Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Thu, 1 Aug 2024 13:05:07 +0100 Subject: [PATCH 15/25] update occa-transpiler repo --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index f063ed87c..9450215d6 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit f063ed87c5a881eed89b0f8b551e7c9627b22d0b +Subproject commit 9450215d62ce318c99495bb86f219ad95e8d8c6d From 02d32bfb8f75b60b3b197aa7233bdd99b3aba6f7 Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Fri, 2 Aug 2024 10:50:59 +0100 Subject: [PATCH 16/25] add option to build new transpiler with local installed clang --- CMakeLists.txt | 20 ++++++++++++++------ INSTALL.md | 16 +++++++++++++--- deps/occa-transpiler | 2 +- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4100541a8..09bdc6726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,11 +75,19 @@ if (OCCA_CLANG_BASED_TRANSPILER) init_submodules(${PROJECT_SOURCE_DIR}) - CPMAddPackage( - NAME occa-transpiler - SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler - OPTIONS "TRANSPILER_TESTS OFF" - ) + if (NOT OCCA_LOCAL_CLANG_PATH) + CPMAddPackage( + NAME occa-transpiler + SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler + OPTIONS "TRANSPILER_TESTS OFF" + ) + else() + CPMAddPackage( + NAME occa-transpiler + SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler + OPTIONS "TRANSPILER_TESTS OFF" "OCCA_LOCAL_CLANG_PATH ${OCCA_LOCAL_CLANG_PATH}" + ) + endif() endif() include(SetCompilerFlags) @@ -254,7 +262,7 @@ if(OCCA_ENABLE_METAL AND APPLE) endif() endif() #======================================= - + if(NOT OCCA_IS_TOP_LEVEL) # OCCA is being built as a subdirectory in another project set(OCCA_OPENMP_ENABLED ${OCCA_OPENMP_ENABLED} PARENT_SCOPE) diff --git a/INSTALL.md b/INSTALL.md index 0c6198fa3..02b9c7012 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -47,6 +47,8 @@ $ CC=clang CXX=clang++ OCCA_ENABLE_OPENMP="OFF" ./configure-cmake.sh | OCCA_ENABLE_TESTS | Build OCCA's test harness | `ON` | | OCCA_ENABLE_EXAMPLES | Build OCCA examples | `ON` | | OCCA_ENABLE_FORTRAN | Build the Fortran language bindings | `OFF`| +| OCCA_CLANG_BASED_TRANSPILER | Build clang based transpiler that support C++ in OKL | `OFF`| +| OCCA_LOCAL_CLANG_PATH | Set path to local clang dir for clang based transpiler | `STRING`| | FC | Fortran 90 compiler | `gfortran` | | FFLAGS | Fortran compiler flags | *empty* | @@ -75,12 +77,12 @@ When cross compiling for a different platform, the targeted hardware doesn't nee #### Building with Clang transplier option -Hard dependecy is clang-17. How to install it please refer to the original +Hard dependency is clang-17 exactly. So far clang based transpiler does not have compatibility layer to support differences of C++ API in clang tooling in newer versions. How to install it please refer to the original [clang occa-transpiler](https://github.com/libocca/occa-transpiler/blob/main/README.md) -The rest dependecies are represented as git submodules and are fetched automatically by cmake script. +The rest dependencies are represented as git submodules and are fetched automatically by cmake script. Building the project with clang based transpiler now is supported only by *CMake* build system. -All options must be provided directly. Here is the following example: +All options must be provided directly. The following example shows how to build the new transpiler with system Clang: ```shell $ mkdir build @@ -90,6 +92,14 @@ $ cmake --build . --parallel $ cmake --install . --prefix install ``` +For Clang that is built locally the install prefix should be specified: +```shell +$ mkdir build +$ cd build +$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON -DOCCA_LOCAL_CLANG_PATH=/home/ikobein/rnd/projects/shell/okl-transpiler/clang-17-rel-instal.. +$ cmake --build . --parallel +$ cmake --install . --prefix install +``` ### Testing CTest is used for the OCCA test harness and can be run using the command diff --git a/deps/occa-transpiler b/deps/occa-transpiler index 9450215d6..0204918b1 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit 9450215d62ce318c99495bb86f219ad95e8d8c6d +Subproject commit 0204918b1eb4a0cc65ec6dddf5e6587244d9d13d From 7f82ac668fa6bdb03711eae2df6a29f7d36fa434 Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Mon, 23 Sep 2024 22:39:01 +0100 Subject: [PATCH 17/25] fix example of new oklt to support serial, openmp modes; remove debug print --- deps/occa-transpiler | 2 +- examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index 0204918b1..d79d6392a 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit 0204918b1eb4a0cc65ec6dddf5e6587244d9d13d +Subproject commit d79d6392af9d440296f8be6b09e61d5c800bd1ac diff --git a/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl b/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl index a71ae6d4b..05666a6dc 100644 --- a/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl +++ b/examples/cpp/31_oklt_v3_moving_avg/movingAverage.okl @@ -72,7 +72,13 @@ private: }; @inner(0) for(int thread_idx = 0; thread_idx < THREADS_PER_BLOCK; ++thread_idx) { ma.syncCopyFrom(inputData, block_idx, thread_idx); + } + + @inner(0) for(int thread_idx = 0; thread_idx < THREADS_PER_BLOCK; ++thread_idx) { ma.process(thread_idx); + } + + @inner(0) for(int thread_idx = 0; thread_idx < THREADS_PER_BLOCK; ++thread_idx) { ma.syncCopyTo(outputData, block_idx, thread_idx); } } From 388bb478198af9a7c13fbffc01ab148d70e29f4c Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Wed, 25 Sep 2024 21:19:05 +0100 Subject: [PATCH 18/25] add unsigned int to OCCA builtin types --- include/occa/dtype/builtins.hpp | 1 + src/dtype/builtins.cpp | 1 + src/dtype/dtype.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/include/occa/dtype/builtins.hpp b/include/occa/dtype/builtins.hpp index 6288b26dd..4c58c44c8 100644 --- a/include/occa/dtype/builtins.hpp +++ b/include/occa/dtype/builtins.hpp @@ -21,6 +21,7 @@ namespace occa { extern const dtype_t char_; extern const dtype_t short_; extern const dtype_t int_; + extern const dtype_t uint_; extern const dtype_t long_; extern const dtype_t ulong_; extern const dtype_t float_; diff --git a/src/dtype/builtins.cpp b/src/dtype/builtins.cpp index 5281f044d..8a2c874a9 100644 --- a/src/dtype/builtins.cpp +++ b/src/dtype/builtins.cpp @@ -11,6 +11,7 @@ namespace occa { const dtype_t char_("char", sizeof(char), true); const dtype_t short_("short", sizeof(short), true); const dtype_t int_("int", sizeof(int), true); + const dtype_t uint_("unsigned int", sizeof(unsigned int), true); const dtype_t long_("long", sizeof(long), true); const dtype_t ulong_("unsigned long", sizeof(unsigned long), true); const dtype_t float_("float", sizeof(float), true); diff --git a/src/dtype/dtype.cpp b/src/dtype/dtype.cpp index 0a061e567..d30803002 100644 --- a/src/dtype/dtype.cpp +++ b/src/dtype/dtype.cpp @@ -414,6 +414,7 @@ namespace occa { dtypeMap["float"] = &dtype::float_; dtypeMap["double"] = &dtype::double_; dtypeMap["unsigned long"] = &dtype::ulong_; + dtypeMap["unsigned int"] = &dtype::uint_; // Sized primitives dtypeMap["int8"] = dtype::get().ref; From 97736d2555b9f6cc1c350a1c5ef76cd9ca3cfa8c Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Fri, 4 Oct 2024 17:48:39 +0100 Subject: [PATCH 19/25] update README and deps --- README.md | 2 +- deps/occa-transpiler | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a1d4826aa..1e4a613d2 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Mission critical computational science and engineering applications from the pub - SYCL 2020 or later - OpenCL 2.0 or later - OpenMP 4.0 or later - - C++ support with clang based occa-transpiler + - C++ support for OKL with clang based transpiler [new-okl-transpiler](https://github.com/libocca/occa-transpiler) ## Build, Test, Install diff --git a/deps/occa-transpiler b/deps/occa-transpiler index d79d6392a..bf70899c8 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit d79d6392af9d440296f8be6b09e61d5c800bd1ac +Subproject commit bf70899c86d52990cf0e1115dfc3cbb8477f9e4e From 9718257609df3613a4e8add43cad1240504b7463 Mon Sep 17 00:00:00 2001 From: Iurii Kobein Date: Wed, 16 Oct 2024 23:07:33 +0100 Subject: [PATCH 20/25] update occa-transpiler to v1.1 --- deps/occa-transpiler | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/occa-transpiler b/deps/occa-transpiler index bf70899c8..9a17e2e93 160000 --- a/deps/occa-transpiler +++ b/deps/occa-transpiler @@ -1 +1 @@ -Subproject commit bf70899c86d52990cf0e1115dfc3cbb8477f9e4e +Subproject commit 9a17e2e93a1c5dd6c353df902d61612daa9fb0c2 From 264c37e300f71ddab7438cade7559977d084827b Mon Sep 17 00:00:00 2001 From: Thilina Ratnayaka Date: Thu, 24 Oct 2024 14:06:08 -0500 Subject: [PATCH 21/25] Remove occa-tranpiler as a submodule --- .gitmodules | 3 --- deps/occa-transpiler | 1 - 2 files changed, 4 deletions(-) delete mode 160000 deps/occa-transpiler diff --git a/.gitmodules b/.gitmodules index c68eb4cc2..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "deps/occa-transpiler"] - path = deps/occa-transpiler - url = https://github.com/libocca/occa-transpiler.git diff --git a/deps/occa-transpiler b/deps/occa-transpiler deleted file mode 160000 index 9a17e2e93..000000000 --- a/deps/occa-transpiler +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9a17e2e93a1c5dd6c353df902d61612daa9fb0c2 From 125e578244735ec0a9de92de273fc810dc9adba1 Mon Sep 17 00:00:00 2001 From: Thilina Ratnayaka Date: Thu, 24 Oct 2024 14:14:35 -0500 Subject: [PATCH 22/25] Make changes to link occa-transpiler as a library --- CMakeLists.txt | 26 +- INSTALL.md | 18 +- cmake/CPM_0.38.6.cmake | 1155 ------------------------------------- cmake/GitSubmodules.cmake | 26 - 4 files changed, 4 insertions(+), 1221 deletions(-) delete mode 100644 cmake/CPM_0.38.6.cmake delete mode 100644 cmake/GitSubmodules.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 09bdc6726..7fd6f9ac7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,24 +70,7 @@ endif() # INFO: order is important, deps should not apply compiler flags if (OCCA_CLANG_BASED_TRANSPILER) - include(cmake/CPM_0.38.6.cmake) - include(cmake/GitSubmodules.cmake) - - init_submodules(${PROJECT_SOURCE_DIR}) - - if (NOT OCCA_LOCAL_CLANG_PATH) - CPMAddPackage( - NAME occa-transpiler - SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler - OPTIONS "TRANSPILER_TESTS OFF" - ) - else() - CPMAddPackage( - NAME occa-transpiler - SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps/occa-transpiler - OPTIONS "TRANSPILER_TESTS OFF" "OCCA_LOCAL_CLANG_PATH ${OCCA_LOCAL_CLANG_PATH}" - ) - endif() + find_package(oklt REQUIRED) endif() include(SetCompilerFlags) @@ -137,11 +120,8 @@ target_include_directories(libocca PRIVATE target_compile_definitions(libocca PRIVATE -DUSE_CMAKE) if (OCCA_CLANG_BASED_TRANSPILER) - target_link_libraries(libocca - PRIVATE - occa-transpiler - ) - target_compile_definitions(libocca PRIVATE -DBUILD_WITH_CLANG_BASED_TRANSPILER) + target_link_libraries(libocca PRIVATE occa::occa-transpiler) + target_compile_definitions(libocca PRIVATE -DBUILD_WITH_CLANG_BASED_TRANSPILER) endif() #======================================= diff --git a/INSTALL.md b/INSTALL.md index 02b9c7012..2c63e2b2e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -48,13 +48,11 @@ $ CC=clang CXX=clang++ OCCA_ENABLE_OPENMP="OFF" ./configure-cmake.sh | OCCA_ENABLE_EXAMPLES | Build OCCA examples | `ON` | | OCCA_ENABLE_FORTRAN | Build the Fortran language bindings | `OFF`| | OCCA_CLANG_BASED_TRANSPILER | Build clang based transpiler that support C++ in OKL | `OFF`| -| OCCA_LOCAL_CLANG_PATH | Set path to local clang dir for clang based transpiler | `STRING`| | FC | Fortran 90 compiler | `gfortran` | | FFLAGS | Fortran compiler flags | *empty* | #### Dependency Paths - The following environment variables can be used to specify the path to third-party dependencies needed by different OCCA backends. The value assigned should be an absolute path to the parent directory, which typically contains subdirectories `bin`, `include`, and `lib`. | Backend | Environment Variable | Description | @@ -76,13 +74,7 @@ When cross compiling for a different platform, the targeted hardware doesn't nee #### Building with Clang transplier option - -Hard dependency is clang-17 exactly. So far clang based transpiler does not have compatibility layer to support differences of C++ API in clang tooling in newer versions. How to install it please refer to the original -[clang occa-transpiler](https://github.com/libocca/occa-transpiler/blob/main/README.md) - -The rest dependencies are represented as git submodules and are fetched automatically by cmake script. -Building the project with clang based transpiler now is supported only by *CMake* build system. -All options must be provided directly. The following example shows how to build the new transpiler with system Clang: +The following example shows how to build OCCA with the new transpiler: ```shell $ mkdir build @@ -92,14 +84,6 @@ $ cmake --build . --parallel $ cmake --install . --prefix install ``` -For Clang that is built locally the install prefix should be specified: -```shell -$ mkdir build -$ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON -DOCCA_LOCAL_CLANG_PATH=/home/ikobein/rnd/projects/shell/okl-transpiler/clang-17-rel-instal.. -$ cmake --build . --parallel -$ cmake --install . --prefix install -``` ### Testing CTest is used for the OCCA test harness and can be run using the command diff --git a/cmake/CPM_0.38.6.cmake b/cmake/CPM_0.38.6.cmake deleted file mode 100644 index 7ffb16b60..000000000 --- a/cmake/CPM_0.38.6.cmake +++ /dev/null @@ -1,1155 +0,0 @@ -# CPM.cmake - CMake's missing package manager -# =========================================== -# See https://github.com/cpm-cmake/CPM.cmake for usage and update instructions. -# -# MIT License -# ----------- -#[[ - Copyright (c) 2019-2023 Lars Melchior and contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -]] - -cmake_minimum_required(VERSION 3.14 FATAL_ERROR) - -# Initialize logging prefix -if(NOT CPM_INDENT) - set(CPM_INDENT - "CPM:" - CACHE INTERNAL "" - ) -endif() - -if(NOT COMMAND cpm_message) - function(cpm_message) - message(${ARGV}) - endfunction() -endif() - -set(CURRENT_CPM_VERSION 0.38.6) - -get_filename_component(CPM_CURRENT_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" REALPATH) -if(CPM_DIRECTORY) - if(NOT CPM_DIRECTORY STREQUAL CPM_CURRENT_DIRECTORY) - if(CPM_VERSION VERSION_LESS CURRENT_CPM_VERSION) - message( - AUTHOR_WARNING - "${CPM_INDENT} \ -A dependency is using a more recent CPM version (${CURRENT_CPM_VERSION}) than the current project (${CPM_VERSION}). \ -It is recommended to upgrade CPM to the most recent version. \ -See https://github.com/cpm-cmake/CPM.cmake for more information." - ) - endif() - if(${CMAKE_VERSION} VERSION_LESS "3.17.0") - include(FetchContent) - endif() - return() - endif() - - get_property( - CPM_INITIALIZED GLOBAL "" - PROPERTY CPM_INITIALIZED - SET - ) - if(CPM_INITIALIZED) - return() - endif() -endif() - -if(CURRENT_CPM_VERSION MATCHES "development-version") - message( - WARNING "${CPM_INDENT} Your project is using an unstable development version of CPM.cmake. \ -Please update to a recent release if possible. \ -See https://github.com/cpm-cmake/CPM.cmake for details." - ) -endif() - -set_property(GLOBAL PROPERTY CPM_INITIALIZED true) - -macro(cpm_set_policies) - # the policy allows us to change options without caching - cmake_policy(SET CMP0077 NEW) - set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - - # the policy allows us to change set(CACHE) without caching - if(POLICY CMP0126) - cmake_policy(SET CMP0126 NEW) - set(CMAKE_POLICY_DEFAULT_CMP0126 NEW) - endif() - - # The policy uses the download time for timestamp, instead of the timestamp in the archive. This - # allows for proper rebuilds when a projects url changes - if(POLICY CMP0135) - cmake_policy(SET CMP0135 NEW) - set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) - endif() -endmacro() -cpm_set_policies() - -option(CPM_USE_LOCAL_PACKAGES "Always try to use `find_package` to get dependencies" - $ENV{CPM_USE_LOCAL_PACKAGES} -) -option(CPM_LOCAL_PACKAGES_ONLY "Only use `find_package` to get dependencies" - $ENV{CPM_LOCAL_PACKAGES_ONLY} -) -option(CPM_DOWNLOAD_ALL "Always download dependencies from source" $ENV{CPM_DOWNLOAD_ALL}) -option(CPM_DONT_UPDATE_MODULE_PATH "Don't update the module path to allow using find_package" - $ENV{CPM_DONT_UPDATE_MODULE_PATH} -) -option(CPM_DONT_CREATE_PACKAGE_LOCK "Don't create a package lock file in the binary path" - $ENV{CPM_DONT_CREATE_PACKAGE_LOCK} -) -option(CPM_INCLUDE_ALL_IN_PACKAGE_LOCK - "Add all packages added through CPM.cmake to the package lock" - $ENV{CPM_INCLUDE_ALL_IN_PACKAGE_LOCK} -) -option(CPM_USE_NAMED_CACHE_DIRECTORIES - "Use additional directory of package name in cache on the most nested level." - $ENV{CPM_USE_NAMED_CACHE_DIRECTORIES} -) - -set(CPM_VERSION - ${CURRENT_CPM_VERSION} - CACHE INTERNAL "" -) -set(CPM_DIRECTORY - ${CPM_CURRENT_DIRECTORY} - CACHE INTERNAL "" -) -set(CPM_FILE - ${CMAKE_CURRENT_LIST_FILE} - CACHE INTERNAL "" -) -set(CPM_PACKAGES - "" - CACHE INTERNAL "" -) -set(CPM_DRY_RUN - OFF - CACHE INTERNAL "Don't download or configure dependencies (for testing)" -) - -if(DEFINED ENV{CPM_SOURCE_CACHE}) - set(CPM_SOURCE_CACHE_DEFAULT $ENV{CPM_SOURCE_CACHE}) -else() - set(CPM_SOURCE_CACHE_DEFAULT OFF) -endif() - -set(CPM_SOURCE_CACHE - ${CPM_SOURCE_CACHE_DEFAULT} - CACHE PATH "Directory to download CPM dependencies" -) - -if(NOT CPM_DONT_UPDATE_MODULE_PATH) - set(CPM_MODULE_PATH - "${CMAKE_BINARY_DIR}/CPM_modules" - CACHE INTERNAL "" - ) - # remove old modules - file(REMOVE_RECURSE ${CPM_MODULE_PATH}) - file(MAKE_DIRECTORY ${CPM_MODULE_PATH}) - # locally added CPM modules should override global packages - set(CMAKE_MODULE_PATH "${CPM_MODULE_PATH};${CMAKE_MODULE_PATH}") -endif() - -if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) - set(CPM_PACKAGE_LOCK_FILE - "${CMAKE_BINARY_DIR}/cpm-package-lock.cmake" - CACHE INTERNAL "" - ) - file(WRITE ${CPM_PACKAGE_LOCK_FILE} - "# CPM Package Lock\n# This file should be committed to version control\n\n" - ) -endif() - -include(FetchContent) - -# Try to infer package name from git repository uri (path or url) -function(cpm_package_name_from_git_uri URI RESULT) - if("${URI}" MATCHES "([^/:]+)/?.git/?$") - set(${RESULT} - ${CMAKE_MATCH_1} - PARENT_SCOPE - ) - else() - unset(${RESULT} PARENT_SCOPE) - endif() -endfunction() - -# Try to infer package name and version from a url -function(cpm_package_name_and_ver_from_url url outName outVer) - if(url MATCHES "[/\\?]([a-zA-Z0-9_\\.-]+)\\.(tar|tar\\.gz|tar\\.bz2|zip|ZIP)(\\?|/|$)") - # We matched an archive - set(filename "${CMAKE_MATCH_1}") - - if(filename MATCHES "([a-zA-Z0-9_\\.-]+)[_-]v?(([0-9]+\\.)*[0-9]+[a-zA-Z0-9]*)") - # We matched - (ie foo-1.2.3) - set(${outName} - "${CMAKE_MATCH_1}" - PARENT_SCOPE - ) - set(${outVer} - "${CMAKE_MATCH_2}" - PARENT_SCOPE - ) - elseif(filename MATCHES "(([0-9]+\\.)+[0-9]+[a-zA-Z0-9]*)") - # We couldn't find a name, but we found a version - # - # In many cases (which we don't handle here) the url would look something like - # `irrelevant/ACTUAL_PACKAGE_NAME/irrelevant/1.2.3.zip`. In such a case we can't possibly - # distinguish the package name from the irrelevant bits. Moreover if we try to match the - # package name from the filename, we'd get bogus at best. - unset(${outName} PARENT_SCOPE) - set(${outVer} - "${CMAKE_MATCH_1}" - PARENT_SCOPE - ) - else() - # Boldly assume that the file name is the package name. - # - # Yes, something like `irrelevant/ACTUAL_NAME/irrelevant/download.zip` will ruin our day, but - # such cases should be quite rare. No popular service does this... we think. - set(${outName} - "${filename}" - PARENT_SCOPE - ) - unset(${outVer} PARENT_SCOPE) - endif() - else() - # No ideas yet what to do with non-archives - unset(${outName} PARENT_SCOPE) - unset(${outVer} PARENT_SCOPE) - endif() -endfunction() - -function(cpm_find_package NAME VERSION) - string(REPLACE " " ";" EXTRA_ARGS "${ARGN}") - find_package(${NAME} ${VERSION} ${EXTRA_ARGS} QUIET) - if(${CPM_ARGS_NAME}_FOUND) - if(DEFINED ${CPM_ARGS_NAME}_VERSION) - set(VERSION ${${CPM_ARGS_NAME}_VERSION}) - endif() - cpm_message(STATUS "${CPM_INDENT} Using local package ${CPM_ARGS_NAME}@${VERSION}") - CPMRegisterPackage(${CPM_ARGS_NAME} "${VERSION}") - set(CPM_PACKAGE_FOUND - YES - PARENT_SCOPE - ) - else() - set(CPM_PACKAGE_FOUND - NO - PARENT_SCOPE - ) - endif() -endfunction() - -# Create a custom FindXXX.cmake module for a CPM package This prevents `find_package(NAME)` from -# finding the system library -function(cpm_create_module_file Name) - if(NOT CPM_DONT_UPDATE_MODULE_PATH) - # erase any previous modules - file(WRITE ${CPM_MODULE_PATH}/Find${Name}.cmake - "include(\"${CPM_FILE}\")\n${ARGN}\nset(${Name}_FOUND TRUE)" - ) - endif() -endfunction() - -# Find a package locally or fallback to CPMAddPackage -function(CPMFindPackage) - set(oneValueArgs NAME VERSION GIT_TAG FIND_PACKAGE_ARGUMENTS) - - cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "" ${ARGN}) - - if(NOT DEFINED CPM_ARGS_VERSION) - if(DEFINED CPM_ARGS_GIT_TAG) - cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) - endif() - endif() - - set(downloadPackage ${CPM_DOWNLOAD_ALL}) - if(DEFINED CPM_DOWNLOAD_${CPM_ARGS_NAME}) - set(downloadPackage ${CPM_DOWNLOAD_${CPM_ARGS_NAME}}) - elseif(DEFINED ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) - set(downloadPackage $ENV{CPM_DOWNLOAD_${CPM_ARGS_NAME}}) - endif() - if(downloadPackage) - CPMAddPackage(${ARGN}) - cpm_export_variables(${CPM_ARGS_NAME}) - return() - endif() - - cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") - if(CPM_PACKAGE_ALREADY_ADDED) - cpm_export_variables(${CPM_ARGS_NAME}) - return() - endif() - - cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) - - if(NOT CPM_PACKAGE_FOUND) - CPMAddPackage(${ARGN}) - cpm_export_variables(${CPM_ARGS_NAME}) - endif() - -endfunction() - -# checks if a package has been added before -function(cpm_check_if_package_already_added CPM_ARGS_NAME CPM_ARGS_VERSION) - if("${CPM_ARGS_NAME}" IN_LIST CPM_PACKAGES) - CPMGetPackageVersion(${CPM_ARGS_NAME} CPM_PACKAGE_VERSION) - if("${CPM_PACKAGE_VERSION}" VERSION_LESS "${CPM_ARGS_VERSION}") - message( - WARNING - "${CPM_INDENT} Requires a newer version of ${CPM_ARGS_NAME} (${CPM_ARGS_VERSION}) than currently included (${CPM_PACKAGE_VERSION})." - ) - endif() - cpm_get_fetch_properties(${CPM_ARGS_NAME}) - set(${CPM_ARGS_NAME}_ADDED NO) - set(CPM_PACKAGE_ALREADY_ADDED - YES - PARENT_SCOPE - ) - cpm_export_variables(${CPM_ARGS_NAME}) - else() - set(CPM_PACKAGE_ALREADY_ADDED - NO - PARENT_SCOPE - ) - endif() -endfunction() - -# Parse the argument of CPMAddPackage in case a single one was provided and convert it to a list of -# arguments which can then be parsed idiomatically. For example gh:foo/bar@1.2.3 will be converted -# to: GITHUB_REPOSITORY;foo/bar;VERSION;1.2.3 -function(cpm_parse_add_package_single_arg arg outArgs) - # Look for a scheme - if("${arg}" MATCHES "^([a-zA-Z]+):(.+)$") - string(TOLOWER "${CMAKE_MATCH_1}" scheme) - set(uri "${CMAKE_MATCH_2}") - - # Check for CPM-specific schemes - if(scheme STREQUAL "gh") - set(out "GITHUB_REPOSITORY;${uri}") - set(packageType "git") - elseif(scheme STREQUAL "gl") - set(out "GITLAB_REPOSITORY;${uri}") - set(packageType "git") - elseif(scheme STREQUAL "bb") - set(out "BITBUCKET_REPOSITORY;${uri}") - set(packageType "git") - # A CPM-specific scheme was not found. Looks like this is a generic URL so try to determine - # type - elseif(arg MATCHES ".git/?(@|#|$)") - set(out "GIT_REPOSITORY;${arg}") - set(packageType "git") - else() - # Fall back to a URL - set(out "URL;${arg}") - set(packageType "archive") - - # We could also check for SVN since FetchContent supports it, but SVN is so rare these days. - # We just won't bother with the additional complexity it will induce in this function. SVN is - # done by multi-arg - endif() - else() - if(arg MATCHES ".git/?(@|#|$)") - set(out "GIT_REPOSITORY;${arg}") - set(packageType "git") - else() - # Give up - message(FATAL_ERROR "${CPM_INDENT} Can't determine package type of '${arg}'") - endif() - endif() - - # For all packages we interpret @... as version. Only replace the last occurrence. Thus URIs - # containing '@' can be used - string(REGEX REPLACE "@([^@]+)$" ";VERSION;\\1" out "${out}") - - # Parse the rest according to package type - if(packageType STREQUAL "git") - # For git repos we interpret #... as a tag or branch or commit hash - string(REGEX REPLACE "#([^#]+)$" ";GIT_TAG;\\1" out "${out}") - elseif(packageType STREQUAL "archive") - # For archives we interpret #... as a URL hash. - string(REGEX REPLACE "#([^#]+)$" ";URL_HASH;\\1" out "${out}") - # We don't try to parse the version if it's not provided explicitly. cpm_get_version_from_url - # should do this at a later point - else() - # We should never get here. This is an assertion and hitting it means there's a bug in the code - # above. A packageType was set, but not handled by this if-else. - message(FATAL_ERROR "${CPM_INDENT} Unsupported package type '${packageType}' of '${arg}'") - endif() - - set(${outArgs} - ${out} - PARENT_SCOPE - ) -endfunction() - -# Check that the working directory for a git repo is clean -function(cpm_check_git_working_dir_is_clean repoPath gitTag isClean) - - find_package(Git REQUIRED) - - if(NOT GIT_EXECUTABLE) - # No git executable, assume directory is clean - set(${isClean} - TRUE - PARENT_SCOPE - ) - return() - endif() - - # check for uncommitted changes - execute_process( - COMMAND ${GIT_EXECUTABLE} status --porcelain - RESULT_VARIABLE resultGitStatus - OUTPUT_VARIABLE repoStatus - OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET - WORKING_DIRECTORY ${repoPath} - ) - if(resultGitStatus) - # not supposed to happen, assume clean anyway - message(WARNING "${CPM_INDENT} Calling git status on folder ${repoPath} failed") - set(${isClean} - TRUE - PARENT_SCOPE - ) - return() - endif() - - if(NOT "${repoStatus}" STREQUAL "") - set(${isClean} - FALSE - PARENT_SCOPE - ) - return() - endif() - - # check for committed changes - execute_process( - COMMAND ${GIT_EXECUTABLE} diff -s --exit-code ${gitTag} - RESULT_VARIABLE resultGitDiff - OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_QUIET - WORKING_DIRECTORY ${repoPath} - ) - - if(${resultGitDiff} EQUAL 0) - set(${isClean} - TRUE - PARENT_SCOPE - ) - else() - set(${isClean} - FALSE - PARENT_SCOPE - ) - endif() - -endfunction() - -# method to overwrite internal FetchContent properties, to allow using CPM.cmake to overload -# FetchContent calls. As these are internal cmake properties, this method should be used carefully -# and may need modification in future CMake versions. Source: -# https://github.com/Kitware/CMake/blob/dc3d0b5a0a7d26d43d6cfeb511e224533b5d188f/Modules/FetchContent.cmake#L1152 -function(cpm_override_fetchcontent contentName) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "SOURCE_DIR;BINARY_DIR" "") - if(NOT "${arg_UNPARSED_ARGUMENTS}" STREQUAL "") - message(FATAL_ERROR "${CPM_INDENT} Unsupported arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - - string(TOLOWER ${contentName} contentNameLower) - set(prefix "_FetchContent_${contentNameLower}") - - set(propertyName "${prefix}_sourceDir") - define_property( - GLOBAL - PROPERTY ${propertyName} - BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" - FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" - ) - set_property(GLOBAL PROPERTY ${propertyName} "${arg_SOURCE_DIR}") - - set(propertyName "${prefix}_binaryDir") - define_property( - GLOBAL - PROPERTY ${propertyName} - BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" - FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" - ) - set_property(GLOBAL PROPERTY ${propertyName} "${arg_BINARY_DIR}") - - set(propertyName "${prefix}_populated") - define_property( - GLOBAL - PROPERTY ${propertyName} - BRIEF_DOCS "Internal implementation detail of FetchContent_Populate()" - FULL_DOCS "Details used by FetchContent_Populate() for ${contentName}" - ) - set_property(GLOBAL PROPERTY ${propertyName} TRUE) -endfunction() - -# Download and add a package from source -function(CPMAddPackage) - cpm_set_policies() - - list(LENGTH ARGN argnLength) - if(argnLength EQUAL 1) - cpm_parse_add_package_single_arg("${ARGN}" ARGN) - - # The shorthand syntax implies EXCLUDE_FROM_ALL and SYSTEM - set(ARGN "${ARGN};EXCLUDE_FROM_ALL;YES;SYSTEM;YES;") - endif() - - set(oneValueArgs - NAME - FORCE - VERSION - GIT_TAG - DOWNLOAD_ONLY - GITHUB_REPOSITORY - GITLAB_REPOSITORY - BITBUCKET_REPOSITORY - GIT_REPOSITORY - SOURCE_DIR - FIND_PACKAGE_ARGUMENTS - NO_CACHE - SYSTEM - GIT_SHALLOW - EXCLUDE_FROM_ALL - SOURCE_SUBDIR - ) - - set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) - - cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") - - # Set default values for arguments - - if(NOT DEFINED CPM_ARGS_VERSION) - if(DEFINED CPM_ARGS_GIT_TAG) - cpm_get_version_from_git_tag("${CPM_ARGS_GIT_TAG}" CPM_ARGS_VERSION) - endif() - endif() - - if(CPM_ARGS_DOWNLOAD_ONLY) - set(DOWNLOAD_ONLY ${CPM_ARGS_DOWNLOAD_ONLY}) - else() - set(DOWNLOAD_ONLY NO) - endif() - - if(DEFINED CPM_ARGS_GITHUB_REPOSITORY) - set(CPM_ARGS_GIT_REPOSITORY "https://github.com/${CPM_ARGS_GITHUB_REPOSITORY}.git") - elseif(DEFINED CPM_ARGS_GITLAB_REPOSITORY) - set(CPM_ARGS_GIT_REPOSITORY "https://gitlab.com/${CPM_ARGS_GITLAB_REPOSITORY}.git") - elseif(DEFINED CPM_ARGS_BITBUCKET_REPOSITORY) - set(CPM_ARGS_GIT_REPOSITORY "https://bitbucket.org/${CPM_ARGS_BITBUCKET_REPOSITORY}.git") - endif() - - if(DEFINED CPM_ARGS_GIT_REPOSITORY) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_REPOSITORY ${CPM_ARGS_GIT_REPOSITORY}) - if(NOT DEFINED CPM_ARGS_GIT_TAG) - set(CPM_ARGS_GIT_TAG v${CPM_ARGS_VERSION}) - endif() - - # If a name wasn't provided, try to infer it from the git repo - if(NOT DEFINED CPM_ARGS_NAME) - cpm_package_name_from_git_uri(${CPM_ARGS_GIT_REPOSITORY} CPM_ARGS_NAME) - endif() - endif() - - set(CPM_SKIP_FETCH FALSE) - - if(DEFINED CPM_ARGS_GIT_TAG) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_TAG ${CPM_ARGS_GIT_TAG}) - # If GIT_SHALLOW is explicitly specified, honor the value. - if(DEFINED CPM_ARGS_GIT_SHALLOW) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW ${CPM_ARGS_GIT_SHALLOW}) - endif() - endif() - - if(DEFINED CPM_ARGS_URL) - # If a name or version aren't provided, try to infer them from the URL - list(GET CPM_ARGS_URL 0 firstUrl) - cpm_package_name_and_ver_from_url(${firstUrl} nameFromUrl verFromUrl) - # If we fail to obtain name and version from the first URL, we could try other URLs if any. - # However multiple URLs are expected to be quite rare, so for now we won't bother. - - # If the caller provided their own name and version, they trump the inferred ones. - if(NOT DEFINED CPM_ARGS_NAME) - set(CPM_ARGS_NAME ${nameFromUrl}) - endif() - if(NOT DEFINED CPM_ARGS_VERSION) - set(CPM_ARGS_VERSION ${verFromUrl}) - endif() - - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS URL "${CPM_ARGS_URL}") - endif() - - # Check for required arguments - - if(NOT DEFINED CPM_ARGS_NAME) - message( - FATAL_ERROR - "${CPM_INDENT} 'NAME' was not provided and couldn't be automatically inferred for package added with arguments: '${ARGN}'" - ) - endif() - - # Check if package has been added before - cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") - if(CPM_PACKAGE_ALREADY_ADDED) - cpm_export_variables(${CPM_ARGS_NAME}) - return() - endif() - - # Check for manual overrides - if(NOT CPM_ARGS_FORCE AND NOT "${CPM_${CPM_ARGS_NAME}_SOURCE}" STREQUAL "") - set(PACKAGE_SOURCE ${CPM_${CPM_ARGS_NAME}_SOURCE}) - set(CPM_${CPM_ARGS_NAME}_SOURCE "") - CPMAddPackage( - NAME "${CPM_ARGS_NAME}" - SOURCE_DIR "${PACKAGE_SOURCE}" - EXCLUDE_FROM_ALL "${CPM_ARGS_EXCLUDE_FROM_ALL}" - SYSTEM "${CPM_ARGS_SYSTEM}" - OPTIONS "${CPM_ARGS_OPTIONS}" - SOURCE_SUBDIR "${CPM_ARGS_SOURCE_SUBDIR}" - DOWNLOAD_ONLY "${DOWNLOAD_ONLY}" - FORCE True - ) - cpm_export_variables(${CPM_ARGS_NAME}) - return() - endif() - - # Check for available declaration - if(NOT CPM_ARGS_FORCE AND NOT "${CPM_DECLARATION_${CPM_ARGS_NAME}}" STREQUAL "") - set(declaration ${CPM_DECLARATION_${CPM_ARGS_NAME}}) - set(CPM_DECLARATION_${CPM_ARGS_NAME} "") - CPMAddPackage(${declaration}) - cpm_export_variables(${CPM_ARGS_NAME}) - # checking again to ensure version and option compatibility - cpm_check_if_package_already_added(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}") - return() - endif() - - if(NOT CPM_ARGS_FORCE) - if(CPM_USE_LOCAL_PACKAGES OR CPM_LOCAL_PACKAGES_ONLY) - cpm_find_package(${CPM_ARGS_NAME} "${CPM_ARGS_VERSION}" ${CPM_ARGS_FIND_PACKAGE_ARGUMENTS}) - - if(CPM_PACKAGE_FOUND) - cpm_export_variables(${CPM_ARGS_NAME}) - return() - endif() - - if(CPM_LOCAL_PACKAGES_ONLY) - message( - SEND_ERROR - "${CPM_INDENT} ${CPM_ARGS_NAME} not found via find_package(${CPM_ARGS_NAME} ${CPM_ARGS_VERSION})" - ) - endif() - endif() - endif() - - CPMRegisterPackage("${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}") - - if(DEFINED CPM_ARGS_GIT_TAG) - set(PACKAGE_INFO "${CPM_ARGS_GIT_TAG}") - elseif(DEFINED CPM_ARGS_SOURCE_DIR) - set(PACKAGE_INFO "${CPM_ARGS_SOURCE_DIR}") - else() - set(PACKAGE_INFO "${CPM_ARGS_VERSION}") - endif() - - if(DEFINED FETCHCONTENT_BASE_DIR) - # respect user's FETCHCONTENT_BASE_DIR if set - set(CPM_FETCHCONTENT_BASE_DIR ${FETCHCONTENT_BASE_DIR}) - else() - set(CPM_FETCHCONTENT_BASE_DIR ${CMAKE_BINARY_DIR}/_deps) - endif() - - if(DEFINED CPM_ARGS_DOWNLOAD_COMMAND) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS DOWNLOAD_COMMAND ${CPM_ARGS_DOWNLOAD_COMMAND}) - elseif(DEFINED CPM_ARGS_SOURCE_DIR) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${CPM_ARGS_SOURCE_DIR}) - if(NOT IS_ABSOLUTE ${CPM_ARGS_SOURCE_DIR}) - # Expand `CPM_ARGS_SOURCE_DIR` relative path. This is important because EXISTS doesn't work - # for relative paths. - get_filename_component( - source_directory ${CPM_ARGS_SOURCE_DIR} REALPATH BASE_DIR ${CMAKE_CURRENT_BINARY_DIR} - ) - else() - set(source_directory ${CPM_ARGS_SOURCE_DIR}) - endif() - if(NOT EXISTS ${source_directory}) - string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) - # remove timestamps so CMake will re-download the dependency - file(REMOVE_RECURSE "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild") - endif() - elseif(CPM_SOURCE_CACHE AND NOT CPM_ARGS_NO_CACHE) - string(TOLOWER ${CPM_ARGS_NAME} lower_case_name) - set(origin_parameters ${CPM_ARGS_UNPARSED_ARGUMENTS}) - list(SORT origin_parameters) - if(CPM_USE_NAMED_CACHE_DIRECTORIES) - string(SHA1 origin_hash "${origin_parameters};NEW_CACHE_STRUCTURE_TAG") - set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}/${CPM_ARGS_NAME}) - else() - string(SHA1 origin_hash "${origin_parameters}") - set(download_directory ${CPM_SOURCE_CACHE}/${lower_case_name}/${origin_hash}) - endif() - # Expand `download_directory` relative path. This is important because EXISTS doesn't work for - # relative paths. - get_filename_component(download_directory ${download_directory} ABSOLUTE) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory}) - - if(CPM_SOURCE_CACHE) - file(LOCK ${download_directory}/../cmake.lock) - endif() - - if(EXISTS ${download_directory}) - if(CPM_SOURCE_CACHE) - file(LOCK ${download_directory}/../cmake.lock RELEASE) - endif() - - cpm_store_fetch_properties( - ${CPM_ARGS_NAME} "${download_directory}" - "${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build" - ) - cpm_get_fetch_properties("${CPM_ARGS_NAME}") - - if(DEFINED CPM_ARGS_GIT_TAG AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS)) - # warn if cache has been changed since checkout - cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN) - if(NOT ${IS_CLEAN}) - message( - WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty" - ) - endif() - endif() - - cpm_add_subdirectory( - "${CPM_ARGS_NAME}" - "${DOWNLOAD_ONLY}" - "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" - "${${CPM_ARGS_NAME}_BINARY_DIR}" - "${CPM_ARGS_EXCLUDE_FROM_ALL}" - "${CPM_ARGS_SYSTEM}" - "${CPM_ARGS_OPTIONS}" - ) - set(PACKAGE_INFO "${PACKAGE_INFO} at ${download_directory}") - - # As the source dir is already cached/populated, we override the call to FetchContent. - set(CPM_SKIP_FETCH TRUE) - cpm_override_fetchcontent( - "${lower_case_name}" SOURCE_DIR "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" - BINARY_DIR "${${CPM_ARGS_NAME}_BINARY_DIR}" - ) - - else() - # Enable shallow clone when GIT_TAG is not a commit hash. Our guess may not be accurate, but - # it should guarantee no commit hash get mis-detected. - if(NOT DEFINED CPM_ARGS_GIT_SHALLOW) - cpm_is_git_tag_commit_hash("${CPM_ARGS_GIT_TAG}" IS_HASH) - if(NOT ${IS_HASH}) - list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS GIT_SHALLOW TRUE) - endif() - endif() - - # remove timestamps so CMake will re-download the dependency - file(REMOVE_RECURSE ${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-subbuild) - set(PACKAGE_INFO "${PACKAGE_INFO} to ${download_directory}") - endif() - endif() - - cpm_create_module_file(${CPM_ARGS_NAME} "CPMAddPackage(\"${ARGN}\")") - - if(CPM_PACKAGE_LOCK_ENABLED) - if((CPM_ARGS_VERSION AND NOT CPM_ARGS_SOURCE_DIR) OR CPM_INCLUDE_ALL_IN_PACKAGE_LOCK) - cpm_add_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") - elseif(CPM_ARGS_SOURCE_DIR) - cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "local directory") - else() - cpm_add_comment_to_package_lock(${CPM_ARGS_NAME} "${ARGN}") - endif() - endif() - - cpm_message( - STATUS "${CPM_INDENT} Adding package ${CPM_ARGS_NAME}@${CPM_ARGS_VERSION} (${PACKAGE_INFO})" - ) - - if(NOT CPM_SKIP_FETCH) - cpm_declare_fetch( - "${CPM_ARGS_NAME}" "${CPM_ARGS_VERSION}" "${PACKAGE_INFO}" "${CPM_ARGS_UNPARSED_ARGUMENTS}" - ) - cpm_fetch_package("${CPM_ARGS_NAME}" populated) - if(CPM_SOURCE_CACHE AND download_directory) - file(LOCK ${download_directory}/../cmake.lock RELEASE) - endif() - if(${populated}) - cpm_add_subdirectory( - "${CPM_ARGS_NAME}" - "${DOWNLOAD_ONLY}" - "${${CPM_ARGS_NAME}_SOURCE_DIR}/${CPM_ARGS_SOURCE_SUBDIR}" - "${${CPM_ARGS_NAME}_BINARY_DIR}" - "${CPM_ARGS_EXCLUDE_FROM_ALL}" - "${CPM_ARGS_SYSTEM}" - "${CPM_ARGS_OPTIONS}" - ) - endif() - cpm_get_fetch_properties("${CPM_ARGS_NAME}") - endif() - - set(${CPM_ARGS_NAME}_ADDED YES) - cpm_export_variables("${CPM_ARGS_NAME}") -endfunction() - -# Fetch a previously declared package -macro(CPMGetPackage Name) - if(DEFINED "CPM_DECLARATION_${Name}") - CPMAddPackage(NAME ${Name}) - else() - message(SEND_ERROR "${CPM_INDENT} Cannot retrieve package ${Name}: no declaration available") - endif() -endmacro() - -# export variables available to the caller to the parent scope expects ${CPM_ARGS_NAME} to be set -macro(cpm_export_variables name) - set(${name}_SOURCE_DIR - "${${name}_SOURCE_DIR}" - PARENT_SCOPE - ) - set(${name}_BINARY_DIR - "${${name}_BINARY_DIR}" - PARENT_SCOPE - ) - set(${name}_ADDED - "${${name}_ADDED}" - PARENT_SCOPE - ) - set(CPM_LAST_PACKAGE_NAME - "${name}" - PARENT_SCOPE - ) -endmacro() - -# declares a package, so that any call to CPMAddPackage for the package name will use these -# arguments instead. Previous declarations will not be overridden. -macro(CPMDeclarePackage Name) - if(NOT DEFINED "CPM_DECLARATION_${Name}") - set("CPM_DECLARATION_${Name}" "${ARGN}") - endif() -endmacro() - -function(cpm_add_to_package_lock Name) - if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) - cpm_prettify_package_arguments(PRETTY_ARGN false ${ARGN}) - file(APPEND ${CPM_PACKAGE_LOCK_FILE} "# ${Name}\nCPMDeclarePackage(${Name}\n${PRETTY_ARGN})\n") - endif() -endfunction() - -function(cpm_add_comment_to_package_lock Name) - if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) - cpm_prettify_package_arguments(PRETTY_ARGN true ${ARGN}) - file(APPEND ${CPM_PACKAGE_LOCK_FILE} - "# ${Name} (unversioned)\n# CPMDeclarePackage(${Name}\n${PRETTY_ARGN}#)\n" - ) - endif() -endfunction() - -# includes the package lock file if it exists and creates a target `cpm-update-package-lock` to -# update it -macro(CPMUsePackageLock file) - if(NOT CPM_DONT_CREATE_PACKAGE_LOCK) - get_filename_component(CPM_ABSOLUTE_PACKAGE_LOCK_PATH ${file} ABSOLUTE) - if(EXISTS ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) - include(${CPM_ABSOLUTE_PACKAGE_LOCK_PATH}) - endif() - if(NOT TARGET cpm-update-package-lock) - add_custom_target( - cpm-update-package-lock COMMAND ${CMAKE_COMMAND} -E copy ${CPM_PACKAGE_LOCK_FILE} - ${CPM_ABSOLUTE_PACKAGE_LOCK_PATH} - ) - endif() - set(CPM_PACKAGE_LOCK_ENABLED true) - endif() -endmacro() - -# registers a package that has been added to CPM -function(CPMRegisterPackage PACKAGE VERSION) - list(APPEND CPM_PACKAGES ${PACKAGE}) - set(CPM_PACKAGES - ${CPM_PACKAGES} - CACHE INTERNAL "" - ) - set("CPM_PACKAGE_${PACKAGE}_VERSION" - ${VERSION} - CACHE INTERNAL "" - ) -endfunction() - -# retrieve the current version of the package to ${OUTPUT} -function(CPMGetPackageVersion PACKAGE OUTPUT) - set(${OUTPUT} - "${CPM_PACKAGE_${PACKAGE}_VERSION}" - PARENT_SCOPE - ) -endfunction() - -# declares a package in FetchContent_Declare -function(cpm_declare_fetch PACKAGE VERSION INFO) - if(${CPM_DRY_RUN}) - cpm_message(STATUS "${CPM_INDENT} Package not declared (dry run)") - return() - endif() - - FetchContent_Declare(${PACKAGE} ${ARGN}) -endfunction() - -# returns properties for a package previously defined by cpm_declare_fetch -function(cpm_get_fetch_properties PACKAGE) - if(${CPM_DRY_RUN}) - return() - endif() - - set(${PACKAGE}_SOURCE_DIR - "${CPM_PACKAGE_${PACKAGE}_SOURCE_DIR}" - PARENT_SCOPE - ) - set(${PACKAGE}_BINARY_DIR - "${CPM_PACKAGE_${PACKAGE}_BINARY_DIR}" - PARENT_SCOPE - ) -endfunction() - -function(cpm_store_fetch_properties PACKAGE source_dir binary_dir) - if(${CPM_DRY_RUN}) - return() - endif() - - set(CPM_PACKAGE_${PACKAGE}_SOURCE_DIR - "${source_dir}" - CACHE INTERNAL "" - ) - set(CPM_PACKAGE_${PACKAGE}_BINARY_DIR - "${binary_dir}" - CACHE INTERNAL "" - ) -endfunction() - -# adds a package as a subdirectory if viable, according to provided options -function( - cpm_add_subdirectory - PACKAGE - DOWNLOAD_ONLY - SOURCE_DIR - BINARY_DIR - EXCLUDE - SYSTEM - OPTIONS -) - - if(NOT DOWNLOAD_ONLY AND EXISTS ${SOURCE_DIR}/CMakeLists.txt) - set(addSubdirectoryExtraArgs "") - if(EXCLUDE) - list(APPEND addSubdirectoryExtraArgs EXCLUDE_FROM_ALL) - endif() - if("${SYSTEM}" AND "${CMAKE_VERSION}" VERSION_GREATER_EQUAL "3.25") - # https://cmake.org/cmake/help/latest/prop_dir/SYSTEM.html#prop_dir:SYSTEM - list(APPEND addSubdirectoryExtraArgs SYSTEM) - endif() - if(OPTIONS) - foreach(OPTION ${OPTIONS}) - cpm_parse_option("${OPTION}") - set(${OPTION_KEY} "${OPTION_VALUE}") - endforeach() - endif() - set(CPM_OLD_INDENT "${CPM_INDENT}") - set(CPM_INDENT "${CPM_INDENT} ${PACKAGE}:") - add_subdirectory(${SOURCE_DIR} ${BINARY_DIR} ${addSubdirectoryExtraArgs}) - set(CPM_INDENT "${CPM_OLD_INDENT}") - endif() -endfunction() - -# downloads a previously declared package via FetchContent and exports the variables -# `${PACKAGE}_SOURCE_DIR` and `${PACKAGE}_BINARY_DIR` to the parent scope -function(cpm_fetch_package PACKAGE populated) - set(${populated} - FALSE - PARENT_SCOPE - ) - if(${CPM_DRY_RUN}) - cpm_message(STATUS "${CPM_INDENT} Package ${PACKAGE} not fetched (dry run)") - return() - endif() - - FetchContent_GetProperties(${PACKAGE}) - - string(TOLOWER "${PACKAGE}" lower_case_name) - - if(NOT ${lower_case_name}_POPULATED) - FetchContent_Populate(${PACKAGE}) - set(${populated} - TRUE - PARENT_SCOPE - ) - endif() - - cpm_store_fetch_properties( - ${CPM_ARGS_NAME} ${${lower_case_name}_SOURCE_DIR} ${${lower_case_name}_BINARY_DIR} - ) - - set(${PACKAGE}_SOURCE_DIR - ${${lower_case_name}_SOURCE_DIR} - PARENT_SCOPE - ) - set(${PACKAGE}_BINARY_DIR - ${${lower_case_name}_BINARY_DIR} - PARENT_SCOPE - ) -endfunction() - -# splits a package option -function(cpm_parse_option OPTION) - string(REGEX MATCH "^[^ ]+" OPTION_KEY "${OPTION}") - string(LENGTH "${OPTION}" OPTION_LENGTH) - string(LENGTH "${OPTION_KEY}" OPTION_KEY_LENGTH) - if(OPTION_KEY_LENGTH STREQUAL OPTION_LENGTH) - # no value for key provided, assume user wants to set option to "ON" - set(OPTION_VALUE "ON") - else() - math(EXPR OPTION_KEY_LENGTH "${OPTION_KEY_LENGTH}+1") - string(SUBSTRING "${OPTION}" "${OPTION_KEY_LENGTH}" "-1" OPTION_VALUE) - endif() - set(OPTION_KEY - "${OPTION_KEY}" - PARENT_SCOPE - ) - set(OPTION_VALUE - "${OPTION_VALUE}" - PARENT_SCOPE - ) -endfunction() - -# guesses the package version from a git tag -function(cpm_get_version_from_git_tag GIT_TAG RESULT) - string(LENGTH ${GIT_TAG} length) - if(length EQUAL 40) - # GIT_TAG is probably a git hash - set(${RESULT} - 0 - PARENT_SCOPE - ) - else() - string(REGEX MATCH "v?([0123456789.]*).*" _ ${GIT_TAG}) - set(${RESULT} - ${CMAKE_MATCH_1} - PARENT_SCOPE - ) - endif() -endfunction() - -# guesses if the git tag is a commit hash or an actual tag or a branch name. -function(cpm_is_git_tag_commit_hash GIT_TAG RESULT) - string(LENGTH "${GIT_TAG}" length) - # full hash has 40 characters, and short hash has at least 7 characters. - if(length LESS 7 OR length GREATER 40) - set(${RESULT} - 0 - PARENT_SCOPE - ) - else() - if(${GIT_TAG} MATCHES "^[a-fA-F0-9]+$") - set(${RESULT} - 1 - PARENT_SCOPE - ) - else() - set(${RESULT} - 0 - PARENT_SCOPE - ) - endif() - endif() -endfunction() - -function(cpm_prettify_package_arguments OUT_VAR IS_IN_COMMENT) - set(oneValueArgs - NAME - FORCE - VERSION - GIT_TAG - DOWNLOAD_ONLY - GITHUB_REPOSITORY - GITLAB_REPOSITORY - BITBUCKET_REPOSITORY - GIT_REPOSITORY - SOURCE_DIR - FIND_PACKAGE_ARGUMENTS - NO_CACHE - SYSTEM - GIT_SHALLOW - EXCLUDE_FROM_ALL - SOURCE_SUBDIR - ) - set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND) - cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - foreach(oneArgName ${oneValueArgs}) - if(DEFINED CPM_ARGS_${oneArgName}) - if(${IS_IN_COMMENT}) - string(APPEND PRETTY_OUT_VAR "#") - endif() - if(${oneArgName} STREQUAL "SOURCE_DIR") - string(REPLACE ${CMAKE_SOURCE_DIR} "\${CMAKE_SOURCE_DIR}" CPM_ARGS_${oneArgName} - ${CPM_ARGS_${oneArgName}} - ) - endif() - string(APPEND PRETTY_OUT_VAR " ${oneArgName} ${CPM_ARGS_${oneArgName}}\n") - endif() - endforeach() - foreach(multiArgName ${multiValueArgs}) - if(DEFINED CPM_ARGS_${multiArgName}) - if(${IS_IN_COMMENT}) - string(APPEND PRETTY_OUT_VAR "#") - endif() - string(APPEND PRETTY_OUT_VAR " ${multiArgName}\n") - foreach(singleOption ${CPM_ARGS_${multiArgName}}) - if(${IS_IN_COMMENT}) - string(APPEND PRETTY_OUT_VAR "#") - endif() - string(APPEND PRETTY_OUT_VAR " \"${singleOption}\"\n") - endforeach() - endif() - endforeach() - - if(NOT "${CPM_ARGS_UNPARSED_ARGUMENTS}" STREQUAL "") - if(${IS_IN_COMMENT}) - string(APPEND PRETTY_OUT_VAR "#") - endif() - string(APPEND PRETTY_OUT_VAR " ") - foreach(CPM_ARGS_UNPARSED_ARGUMENT ${CPM_ARGS_UNPARSED_ARGUMENTS}) - string(APPEND PRETTY_OUT_VAR " ${CPM_ARGS_UNPARSED_ARGUMENT}") - endforeach() - string(APPEND PRETTY_OUT_VAR "\n") - endif() - - set(${OUT_VAR} - ${PRETTY_OUT_VAR} - PARENT_SCOPE - ) - -endfunction() diff --git a/cmake/GitSubmodules.cmake b/cmake/GitSubmodules.cmake deleted file mode 100644 index c20392211..000000000 --- a/cmake/GitSubmodules.cmake +++ /dev/null @@ -1,26 +0,0 @@ -#################################################################################################### -# This function recursively update git submodules. -# Params: project base directory -# Example: -# init_submodules(${PROJECT_SOURCE_DIR}) -#################################################################################################### - -function(init_submodules - project_dir) - - find_package(Git QUIET) - if(GIT_FOUND AND EXISTS "${project_dir}/.git") - # Update submodules as needed - message(STATUS "Submodule update") - execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${project_dir} - RESULT_VARIABLE GIT_SUBMOD_RESULT) - if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(FATAL_ERROR "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") - endif() - else() - message(STATUS "Submodule update has not been run") - endif() - -endfunction() - From 5c96a24746fa14e95757e85c470af7a1b39960af Mon Sep 17 00:00:00 2001 From: Thilina Ratnayaka Date: Thu, 24 Oct 2024 16:10:28 -0500 Subject: [PATCH 23/25] Add a link to occa-transpiler README in INSTALL.md --- INSTALL.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 2c63e2b2e..92e9570c3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -72,14 +72,18 @@ $ cmake --build build --parallel When cross compiling for a different platform, the targeted hardware doesn't need to be available; however all dependencies—e.g., headers, libraries—must be present. Commonly this is the case for large HPC systems, where code is compiled on login nodes and run on compute nodes. -#### Building with Clang transplier option +#### Building with Clang transpiler -The following example shows how to build OCCA with the new transpiler: +Please refer [occa-transpiler README](https://github.com/libocca/occa-transpiler/blob/main/README.md) for instructions on how to +build and install the occa-transpiler. +Then you can use the following commands to install OCCA with occa-transpiler enabled. +Please replace `` by the root directory of your +occa-transpiler installation. ```shell $ mkdir build $ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON .. +$ cmake -DCMAKE_BUILD_TYPE=Release -DOCCA_CLANG_BASED_TRANSPILER=ON -DCMAKE_PREFIX_PATH=/lib/cmake .. $ cmake --build . --parallel $ cmake --install . --prefix install ``` From 494fde271cfd175c8afc9335fc71cad4c37b67b3 Mon Sep 17 00:00:00 2001 From: Thilina Ratnayaka Date: Thu, 24 Oct 2024 16:11:11 -0500 Subject: [PATCH 24/25] Fix a few typos --- INSTALL.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 92e9570c3..c360605c4 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -10,7 +10,7 @@ ### Optional - - Fortan 90 compiler + - Fortran 90 compiler - CUDA 9 or later - HIP 3.5 or later - SYCL 2020 or later @@ -22,7 +22,7 @@ ### **Configure** -OCCA uses the [CMake] build system. For convenience, the shell script `configure-cmake.sh` has been provided to drive the Cmake build. The following table gives a list of build parameters which are set in the file. To override the default value, it is only necessary to assign the variable an alternate value at the top of the script or at the commandline. +OCCA uses the [CMake] build system. For convenience, the shell script `configure-cmake.sh` has been provided to drive the CMake build. The following table gives a list of build parameters which are set in the file. To override the default value, it is only necessary to assign the variable an alternate value at the top of the script or at the commandline. Example ```shell diff --git a/README.md b/README.md index 1e4a613d2..979232f77 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Mission critical computational science and engineering applications from the pub ### Optional - - Fortan 90 compiler + - Fortran 90 compiler - CUDA 9 or later - HIP 4.2 or later - SYCL 2020 or later From f7fcb5f241c1f15be8cf34cc5e459aba65e979a3 Mon Sep 17 00:00:00 2001 From: Thilina Ratnayaka Date: Fri, 25 Oct 2024 09:16:07 -0500 Subject: [PATCH 25/25] Add a link to occa-transpiler repo --- INSTALL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INSTALL.md b/INSTALL.md index c360605c4..13c52b029 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -74,6 +74,7 @@ When cross compiling for a different platform, the targeted hardware doesn't nee #### Building with Clang transpiler +occa-transpiler repository can be found in [libocca/occa-transpiler](https://github.com/libocca/occa-transpiler/). Please refer [occa-transpiler README](https://github.com/libocca/occa-transpiler/blob/main/README.md) for instructions on how to build and install the occa-transpiler. Then you can use the following commands to install OCCA with occa-transpiler enabled.