Compilation on demand

.github/workflows/CI.yml

@@ -5,27 +5,36 @@ on:
       - main
     tags: ['*']
   pull_request:
+  schedule:
+    - cron: 0 8 * * 3
+
 concurrency:
   # Skip intermediate builds: always.
   # Cancel intermediate builds: only if it is a pull request build.
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 jobs:
   test:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    name: Kokkos ${{ matrix.kokkos }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         version:
-          - '1.7'
-          - '1.8'
           - '1.9'
+          - '1'
           - 'nightly'
         os:
           - ubuntu-latest
         arch:
           - x64
+        kokkos:
+          - '4-latest'
+        include:
+          - version: '1.9'
+            os: ubuntu-latest
+            arch: x64
+            kokkos: '3-latest'
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
@@ -37,6 +46,7 @@ jobs:
       - uses: julia-actions/julia-runtest@v1
         env:
           JULIA_NUM_THREADS: 2
+          TEST_KOKKOS_VERSION: ${{ matrix.kokkos }}
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v2
         with:
@@ -51,7 +61,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1
         with:
-          version: '1.8'
+          version: '1.9'
       - name: Install dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy

.gitignore

@@ -4,3 +4,5 @@ LocalPreferences.toml
 /docs/build/
 /lib/kokkos_wrapper/.idea/
 /lib/kokkos_wrapper/cmake-build-*/
+.vscode
+.idea

Project.toml

@@ -1,45 +1,53 @@
 name = "Kokkos"
 uuid = "3296cea9-b0de-4b57-aba0-ce554b517c3b"
 authors = ["Keluaa <[email protected]> and contributors"]
-version = "0.4.1"
+version = "0.5.3"
 
 [deps]
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CxxWrap = "1f15a43c-97ca-5a2a-ae31-89f07a497df4"
+FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
-MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+Pidfile = "fa939f87-e72e-5be4-a000-7fc836dbe307"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 libcxxwrap_julia_jll = "3eaa8342-bff7-56a5-9981-c04077f7cee7"
 
 [compat]
-CxxWrap = "0.13"
+AMDGPU = "0.7"
+CUDA = "4.1, 5"
+CxxWrap = "0.14, 0.15"
 MPI = "0.20"
-CUDA = "4"
+Pidfile = "1.3.0"
 Preferences = "1"
 ProgressMeter = "1"
 Requires = "1"
 Scratch = "1"
-julia = "1.7"
-libcxxwrap_julia_jll = "0.9.7"
+julia = "1.9"
+libcxxwrap_julia_jll = "≥ 0.9.7"
 
 [extensions]
-CUDAExt = "CUDA"
-MPIExt = "MPI"
+KokkosAMDGPU = "AMDGPU"
+KokkosCUDA = "CUDA"
+KokkosMPI = "MPI"
 
 [extras]
+AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Preferences", "Logging", "MPI", "CUDA"]
+test = ["Test", "Preferences", "Logging", "MPI", "CUDA", "AMDGPU"]
 
 [weakdeps]
-MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"

README.md

@@ -1,7 +1,7 @@
-# Kokkos.jl: A Kokkos wrapper for Julia 
+# Kokkos.jl: A Kokkos wrapper for Julia
 
-[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://keluaa.github.io/Kokkos.jl/stable)
-[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://keluaa.github.io/Kokkos.jl/dev)
+[![Documentation](https://img.shields.io/badge/docs-stable-blue.svg)](https://keluaa.github.io/Kokkos.jl/stable)
+[![Dev documentation](https://img.shields.io/badge/docs-dev-blue.svg)](https://keluaa.github.io/Kokkos.jl/dev)
 [![Build Status](https://github.com/Keluaa/Kokkos.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/Keluaa/Kokkos.jl/actions/workflows/CI.yml?query=branch%3Amain)
 [![Coverage](https://codecov.io/gh/Keluaa/Kokkos.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/Keluaa/Kokkos.jl)
 
@@ -20,34 +20,52 @@ written in a separate C++ shared library.
 If the library you want to use is configured with CMake, it is possible to configure the project
 with `Kokkos.jl`.
 
-This package uses a wrapper library which is compiled before initializing Kokkos.
+This package relies on a wrapper library which is compiled when initializing Kokkos, which is
+configured with the CMake and Kokkos options set in the configuration options.
 Because it is not pre-compiled as an artifact, this maximizes the flexibility of usage of `Kokkos.jl`.
+However, most Kokkos functions (and views) are compiled separately on demand, when their respective
+Julia method is called for the first time.
+The resulting shared library is then cached for the next session.
 
-`Kokkos.jl` currently supports Kokkos v3.7.0, v3.7.1, v4.0.0 and above.
+`Kokkos.jl` currently supports Kokkos v3.7, v4.0 and above.
 All Kokkos backends should be supported by this package, but not all of them were tested (yet).
 
-### Supported functionalities
- * :heavy_check_mark: `Kokkos::initialize`, `Kokkos::finalize` and `Kokkos::InitializationSettings`
- * :heavy_check_mark: `Kokkos::View`, `Kokkos::View<T, MyLayout, SomeMemorySpace>` and `Kokkos::view_alloc`
- * :x: `Kokkos::MemoryTraits` (planned)
- * :heavy_check_mark: `Kokkos::create_mirror`, `Kokkos::create_mirror_view`
- * :heavy_check_mark: `Kokkos::deep_copy`
- * :heavy_check_mark: `Kokkos::subview`
- * :x: `Kokkos::resize`, `Kokkos::realloc` (planned)
- * :heavy_check_mark: `Kokkos::fence`
- * :heavy_check_mark: All execution spaces (`Kokkos::OpenMP`, `Kokkos::Cuda`...) and memory spaces (`Kokkos::HostSpace`, `Kokkos::CudaSpace`...)
- * :x: All parallel patterns (`Kokkos::parallel_for`, `Kokkos::parallel_reduce`, `Kokkos::parallel_scan`), reducers, execution policies and tasking
- * :x: Atomics
- * :x: All containers (`Kokkos::DualView`, `Kokkos::ScatterView`...) (planned)
- * :x: SIMD
-
-### Tested backends
- * :heavy_check_mark: `Kokkos::Serial`
- * :heavy_check_mark: `Kokkos::OpenMP`
- * :x: `Kokkos::Threads`
- * :x: `Kokkos::HPX`
- * :x: `Kokkos::OpenMPTarget`
- * :heavy_check_mark: `Kokkos::Cuda` + interop with [CUDA.jl](https://github.com/JuliaGPU/CUDA.jl)
- * :x: `Kokkos::HIP`
- * :x: `Kokkos::SYCL`
- * :x: `Kokkos::OpenACC`
+## Supported functionalities
+
+* :white_check_mark: `Kokkos::initialize`, `Kokkos::finalize` and `Kokkos::InitializationSettings`
+* :white_check_mark: `Kokkos::View`, `Kokkos::View<T, MyLayout, SomeMemorySpace>` and `Kokkos::view_alloc`
+* :x: `Kokkos::MemoryTraits` (planned)
+* :white_check_mark: `Kokkos::create_mirror`, `Kokkos::create_mirror_view`
+* :white_check_mark: `Kokkos::deep_copy`
+* :white_check_mark: `Kokkos::subview`
+* :x: `Kokkos::resize`, `Kokkos::realloc` (planned)
+* :white_check_mark: `Kokkos::fence`
+* :white_check_mark: All execution spaces (`Kokkos::OpenMP`, `Kokkos::Cuda`...) and memory spaces (`Kokkos::HostSpace`, `Kokkos::CudaSpace`...)
+* :x: All parallel patterns (`Kokkos::parallel_for`, `Kokkos::parallel_reduce`, `Kokkos::parallel_scan`), reducers, execution policies and tasking
+* :x: Atomics
+* :x: All containers (`Kokkos::DualView`, `Kokkos::ScatterView`...) (planned)
+* :x: SIMD
+* :x: View hooks
+
+## Tested backends
+
+* :white_check_mark: `Kokkos::Serial`
+* :white_check_mark: `Kokkos::OpenMP`
+* :x: `Kokkos::Threads`
+* :x: `Kokkos::HPX`
+* :x: `Kokkos::OpenMPTarget`
+* :white_check_mark: `Kokkos::Cuda` + interop with [CUDA.jl](https://github.com/JuliaGPU/CUDA.jl)
+* :white_check_mark: `Kokkos::HIP` + interop with [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl)
+* :x: `Kokkos::SYCL`
+* :x: `Kokkos::OpenACC`
+
+### Known issues
+
+* The NVCC compiler is unable to compile the wrapper library. Use Clang instead (Clang-11 is the
+version used to test this backend).
+* `Kokkos::Cuda ERROR: Failed to call Kokkos::Cuda::finalize()` message when exiting Julia:
+`Kokkos.finalize()` is not yet called automatically upon quitting Julia, as it would require to call
+the finalizers of all views before doing so.
+* Memory leaks on GPU: this is a side effect of Julia's GC which cannot manage device memory. From
+Julia's POV, a `Kokkos.View` is only a pointer in the host memory. Calling `GC.gc(true)` will fix the
+issue.

docs/Project.toml

@@ -1,6 +1,10 @@
 [deps]
+AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Kokkos = "3296cea9-b0de-4b57-aba0-ce554b517c3b"
+MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
+MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [compat]
-Documenter = "0.27"
+Documenter = "1"

docs/make.jl

@@ -3,14 +3,22 @@ push!(LOAD_PATH, joinpath(@__DIR__, "../"))
 
 using Kokkos
 using Documenter
-using CUDA  # For the extension documentation
+
+# For the extensions documentation
+using CUDA
+using AMDGPU
+using MPI
+
+KokkosAMDGPU = Base.get_extension(Kokkos, :KokkosAMDGPU)
+KokkosCUDA = Base.get_extension(Kokkos, :KokkosCUDA)
+KokkosMPI = Base.get_extension(Kokkos, :KokkosMPI)
 
 ci = get(ENV, "CI", "") == "true"
 
 DocMeta.setdocmeta!(Kokkos, :DocTestSetup, :(using Kokkos); recursive=true)
 
 makedocs(;
-    modules=[Kokkos],
+    modules=[Kokkos, KokkosAMDGPU, KokkosCUDA, KokkosMPI],
     authors="Keluaa <[email protected]> and contributors",
     repo="https://github.com/Keluaa/Kokkos.jl/blob/{commit}{path}#{line}",
     sitename="Kokkos.jl",
@@ -24,15 +32,18 @@ makedocs(;
         "Usage" => [
             "Calling a Kokkos library" => "calling_c.md",
             "Using views in an inaccessible memory space" => "inaccessible_views.md",
-            "Interoperability with CUDA.jl" => "interop.md",
+            "Interoperability with CUDA.jl and AMDGPU.jl" => "interop.md",
             "MPI" => "MPI.md"
         ],
         "Environment" => "environment.md",
         "Execution & Memory Spaces" => "spaces.md",
         "Views" => "views.md",
         "Compilation" => "compilation.md",
         "Library Management" => "library_management.md",
-        "Configuration options" => "config_options.md"
+        "Configuration options" => "config_options.md",
+        "Internals" => [
+            "Dynamic Compilation" => "dynamic_compilation.md"
+        ]
     ],
 )
 

docs/src/MPI.md

@@ -65,6 +65,10 @@ MPI.Barrier(MPI.COMM_WORLD)
 rank != 0 && Kokkos.load_wrapper_lib(; no_compilation=true, no_git=true)
 ```
 
+### Dynamic Compilation and MPI
+
+See [`DynamicCompilation.compilation_lock`](@ref)
+
 
 ## Passing views to MPI