Skip to content

Commit 512e8b8

Browse files
HowardHuang1HowardHuang1
committed
Merge remote-tracking branch 'upstream/release/26.10' into HH-migrate-merge-dataset-to-take-inputs-concat-buffer-and-offset-26_10
2 parents 1cede24 + 23d4e42 commit 512e8b8

81 files changed

Lines changed: 1373 additions & 4462 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yaml‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -61,34 +61,54 @@ jobs:
6161
contents: read
6262
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@release/26.10
6363
with:
64-
build_type: pull-request
64+
build_type: ${{ inputs.build_type || 'branch' }}
6565
matrix_name: conda-cpp-build
6666
rocky8-clib-standalone-build:
67-
needs: [build-details, rocky8-clib-standalone-build-matrix]
67+
needs: [rocky8-clib-standalone-build-matrix]
68+
runs-on: linux-${{ matrix.ARCH }}-cpu16
69+
env:
70+
RAPIDS_BUILD_TYPE: ${{ inputs.build_type || 'branch' }}
6871
permissions:
6972
actions: read
7073
contents: read
7174
id-token: write
7275
packages: read
7376
pull-requests: read
74-
secrets: inherit # zizmor: ignore[secrets-inherit]
75-
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
7677
strategy:
7778
fail-fast: false
7879
matrix: ${{ fromJSON(needs.rocky8-clib-standalone-build-matrix.outputs.matrix) }}
79-
with:
80-
build_type: ${{ inputs.build_type || 'branch' }}
81-
build-datetime: ${{ needs.build-details.outputs.build-datetime }}
82-
branch: ${{ inputs.branch }}
83-
arch: "${{matrix.ARCH}}"
84-
date: ${{ inputs.date }}
85-
container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
86-
node_type: "cpu16"
87-
requires_license_builder: true
88-
script: "ci/build_standalone_c.sh"
89-
artifact-name: "libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz"
90-
file_to_upload: "libcuvs_c.tar.gz"
91-
sha: ${{ inputs.sha }}
80+
steps:
81+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
82+
with:
83+
ref: ${{ inputs.sha || github.sha }}
84+
fetch-depth: 0
85+
persist-credentials: false
86+
- name: Standardize repository information
87+
uses: rapidsai/shared-actions/rapids-github-info@main
88+
with:
89+
branch: ${{ inputs.branch }}
90+
date: ${{ inputs.date }}
91+
sha: ${{ inputs.sha }}
92+
- uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6.2.4
93+
with:
94+
role-to-assume: ${{ vars.AWS_ROLE_ARN }}
95+
aws-region: ${{ vars.AWS_REGION }}
96+
role-duration-seconds: 10800
97+
- name: Setup sccache-dist
98+
uses: rapidsai/shared-actions/setup-sccache-dist@main
99+
with:
100+
request-timeout: 7140
101+
- name: Build standalone C tarball
102+
env:
103+
CUVS_TARBALL_CUDA_VERSION: ${{ matrix.CUDA_VER }}
104+
CUVS_TARBALL_PYTHON_VERSION: ${{ matrix.PY_VER }}
105+
run: |
106+
./build.sh tarball --tarball-build-tests
107+
- name: Upload artifacts
108+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
109+
with:
110+
name: libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz
111+
path: libcuvs_c.tar.gz
92112
rust-build-matrix:
93113
needs: cpp-build
94114
permissions:
@@ -128,7 +148,7 @@ jobs:
128148
contents: read
129149
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@release/26.10
130150
with:
131-
build_type: pull-request
151+
build_type: ${{ inputs.build_type || 'branch' }}
132152
matrix_name: conda-cpp-build
133153
matrix_filter: map(select(.ARCH == "amd64"))
134154
go-build:
@@ -161,7 +181,7 @@ jobs:
161181
contents: read
162182
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@release/26.10
163183
with:
164-
build_type: pull-request
184+
build_type: ${{ inputs.build_type || 'branch' }}
165185
matrix_name: conda-cpp-build
166186
matrix_filter: map(select(.ARCH == "amd64"))
167187
java-build:
@@ -190,7 +210,8 @@ jobs:
190210
file_to_upload: "java/cuvs-java/target/"
191211
sha: ${{ inputs.sha }}
192212
lucene-build:
193-
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against.
213+
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and
214+
# reuses its matrix so the two always agree on the set of CUDA versions.
194215
needs: [java-build-matrix, java-build]
195216
permissions:
196217
actions: read

‎.github/workflows/pr.yaml‎

Lines changed: 89 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,10 @@ jobs:
2727
- rocky8-clib-tests
2828
- conda-java-build-and-tests-matrix
2929
- conda-java-build-and-tests
30+
- conda-java-tests-other-arch-matrix
31+
- conda-java-tests-other-arch
3032
- conda-lucene-build-and-tests
33+
- conda-lucene-tests-other-arch
3134
- rust-build-matrix
3235
- rust-build
3336
- go-build-matrix
@@ -563,30 +566,45 @@ jobs:
563566
build_type: pull-request
564567
matrix_name: conda-cpp-build
565568
rocky8-clib-standalone-build:
566-
needs: [build-details, rocky8-clib-standalone-build-matrix]
569+
needs: [rocky8-clib-standalone-build-matrix]
570+
runs-on: linux-${{ matrix.ARCH }}-cpu16
571+
env:
572+
RAPIDS_BUILD_TYPE: pull-request
567573
permissions:
568574
actions: read
569575
contents: read
570576
id-token: write
571577
packages: read
572578
pull-requests: read
573-
secrets: inherit # zizmor: ignore[secrets-inherit]
574-
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
575579
strategy:
576580
fail-fast: false
577581
matrix: ${{ fromJSON(needs.rocky8-clib-standalone-build-matrix.outputs.matrix) }}
578-
with:
579-
build_type: pull-request
580-
build-datetime: ${{ needs.build-details.outputs.build-datetime }}
581-
arch: "${{matrix.arch}}"
582-
date: ${{ inputs.date }}_c
583-
container_image: "rapidsai/ci-wheel:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
584-
node_type: "cpu16"
585-
requires_license_builder: true
586-
script: "ci/build_standalone_c.sh --build-tests"
587-
artifact-name: "libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz"
588-
file_to_upload: "libcuvs_c.tar.gz"
589-
sha: ${{ inputs.sha }}
582+
steps:
583+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
584+
with:
585+
ref: ${{ inputs.sha || github.sha }}
586+
fetch-depth: 0
587+
persist-credentials: false
588+
- uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6.2.4
589+
with:
590+
role-to-assume: ${{ vars.AWS_ROLE_ARN }}
591+
aws-region: ${{ vars.AWS_REGION }}
592+
role-duration-seconds: 10800
593+
- name: Setup sccache-dist
594+
uses: rapidsai/shared-actions/setup-sccache-dist@main
595+
with:
596+
request-timeout: 7140
597+
- name: Build standalone C tarball
598+
env:
599+
CUVS_TARBALL_CUDA_VERSION: ${{ matrix.CUDA_VER }}
600+
CUVS_TARBALL_PYTHON_VERSION: ${{ matrix.PY_VER }}
601+
run: |
602+
./build.sh tarball --tarball-build-tests
603+
- name: Upload artifacts
604+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
605+
with:
606+
name: libcuvs_c_${{ matrix.CUDA_VER }}_${{ matrix.ARCH }}.tar.gz
607+
path: libcuvs_c.tar.gz
590608
rocky8-clib-tests-matrix:
591609
needs: [rocky8-clib-standalone-build, changed-files]
592610
permissions:
@@ -650,6 +668,39 @@ jobs:
650668
script: "ci/test_java.sh"
651669
artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}"
652670
file_to_upload: "java/cuvs-java/target/"
671+
conda-java-tests-other-arch-matrix:
672+
# All architectures the C++ core builds for, except amd64.
673+
needs: [conda-cpp-build, changed-files]
674+
permissions:
675+
contents: read
676+
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@release/26.10
677+
if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
678+
with:
679+
build_type: pull-request
680+
matrix_name: conda-cpp-build
681+
matrix_filter: map(select(.ARCH != "amd64"))
682+
conda-java-tests-other-arch:
683+
# Verifies that the amd64-built cuvs-java jar (jextract-generated Panama bindings and
684+
# all) also works correctly on every non-amd64 architecture, by running its own IT test
685+
# suite against the amd64-built classes without recompiling -- see ci/test_java_prebuilt.sh.
686+
needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests]
687+
permissions:
688+
actions: read
689+
contents: read
690+
id-token: write
691+
packages: read
692+
pull-requests: read
693+
secrets: inherit # zizmor: ignore[secrets-inherit]
694+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
695+
strategy:
696+
fail-fast: false
697+
matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }}
698+
with:
699+
build_type: pull-request
700+
node_type: "gpu-l4-latest-1"
701+
arch: "${{ matrix.ARCH }}"
702+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
703+
script: "ci/test_java_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }}"
653704
conda-lucene-build-and-tests:
654705
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and
655706
# reuses its matrix so the two always agree on the set of CUDA versions.
@@ -675,6 +726,29 @@ jobs:
675726
script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}"
676727
artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
677728
file_to_upload: "java/cuvs-lucene/target/"
729+
conda-lucene-tests-other-arch:
730+
# Verifies that cuvs-lucene, built against the amd64-built cuvs-java jar, also works
731+
# correctly on every non-amd64 architecture, by running its test suite against the
732+
# amd64-built classes without recompiling -- see ci/test_lucene_prebuilt.sh. Reuses
733+
# conda-java-tests-other-arch-matrix so the two always agree on which architectures to test.
734+
needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests, conda-lucene-build-and-tests]
735+
permissions:
736+
actions: read
737+
contents: read
738+
id-token: write
739+
packages: read
740+
pull-requests: read
741+
secrets: inherit # zizmor: ignore[secrets-inherit]
742+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
743+
strategy:
744+
fail-fast: false
745+
matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }}
746+
with:
747+
build_type: pull-request
748+
node_type: "gpu-l4-latest-1"
749+
arch: "${{ matrix.ARCH }}"
750+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
751+
script: "ci/test_lucene_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }} cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
678752
rust-build-matrix:
679753
needs: [conda-cpp-build, changed-files]
680754
permissions:

‎.github/workflows/test.yaml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,41 @@ jobs:
104104
script: "ci/test_java.sh"
105105
artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}"
106106
file_to_upload: "java/cuvs-java/target/"
107+
conda-java-tests-other-arch-matrix:
108+
# All architectures the C++ core builds for, except amd64 -- so a newly added
109+
# architecture is automatically picked up here without a workflow change.
110+
permissions:
111+
contents: read
112+
uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@release/26.10
113+
with:
114+
build_type: ${{ inputs.build_type }}
115+
matrix_name: conda-cpp-build
116+
matrix_filter: map(select(.ARCH != "amd64"))
117+
conda-java-tests-other-arch:
118+
# Verifies that the amd64-built cuvs-java jar (jextract-generated Panama bindings and
119+
# all) also works correctly on every non-amd64 architecture, by running its own IT test
120+
# suite against the amd64-built classes without recompiling -- see ci/test_java_prebuilt.sh.
121+
needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests]
122+
permissions:
123+
actions: read
124+
contents: read
125+
id-token: write
126+
packages: read
127+
pull-requests: read
128+
secrets: inherit # zizmor: ignore[secrets-inherit]
129+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
130+
strategy:
131+
fail-fast: false
132+
matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }}
133+
with:
134+
build_type: ${{ inputs.build_type }}
135+
branch: ${{ inputs.branch }}
136+
date: ${{ inputs.date }}
137+
sha: ${{ inputs.sha }}
138+
node_type: "gpu-l4-latest-1"
139+
arch: "${{ matrix.ARCH }}"
140+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
141+
script: "ci/test_java_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }}"
107142
conda-lucene-build-and-tests:
108143
# Depends on the Java job for the cuvs-java artifact that cuvs-lucene builds against, and
109144
# reuses its matrix so the two always agree on the set of CUDA versions.
@@ -132,6 +167,32 @@ jobs:
132167
script: "ci/test_lucene.sh cuvs-java-cuda${{ matrix.CUDA_VER }}"
133168
artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
134169
file_to_upload: "java/cuvs-lucene/target/"
170+
conda-lucene-tests-other-arch:
171+
# Verifies that cuvs-lucene, built against the amd64-built cuvs-java jar, also works
172+
# correctly on every non-amd64 architecture, by running its test suite against the
173+
# amd64-built classes without recompiling -- see ci/test_lucene_prebuilt.sh. Reuses
174+
# conda-java-tests-other-arch-matrix so the two always agree on which architectures to test.
175+
needs: [conda-java-tests-other-arch-matrix, conda-java-build-and-tests, conda-lucene-build-and-tests]
176+
permissions:
177+
actions: read
178+
contents: read
179+
id-token: write
180+
packages: read
181+
pull-requests: read
182+
secrets: inherit # zizmor: ignore[secrets-inherit]
183+
uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.10
184+
strategy:
185+
fail-fast: false
186+
matrix: ${{ fromJSON(needs.conda-java-tests-other-arch-matrix.outputs.matrix) }}
187+
with:
188+
build_type: ${{ inputs.build_type }}
189+
branch: ${{ inputs.branch }}
190+
date: ${{ inputs.date }}
191+
sha: ${{ inputs.sha }}
192+
node_type: "gpu-l4-latest-1"
193+
arch: "${{ matrix.ARCH }}"
194+
container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}"
195+
script: "ci/test_lucene_prebuilt.sh ${{ matrix.ARCH }} cuvs-java-cuda${{ matrix.CUDA_VER }} cuvs-lucene-cuda${{ matrix.CUDA_VER }}"
135196
wheel-tests-cuvs:
136197
permissions:
137198
actions: read

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ dist/
1919
python/**/**/*.cpp
2020
python/cuvs/record.txt
2121
log
22+
infrequent_licenses/
2223
.ipynb_checkpoints
2324
.DS_Store
2425
dask-worker-space/
@@ -29,6 +30,7 @@ temporary_*.json
2930
rust/target/
3031
rust/Cargo.lock
3132
rmm_log.txt
33+
*.tar.gz
3234

3335
## example notebooks
3436
notebooks/simplewiki-2020-11-01-nq-distilbert-base-v1.pt

‎Dockerfile.standalone‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Environment and runner for the standalone C build.
5+
#
6+
# End-user instructions: https://docs.nvidia.com/cuvs/installation#build-the-standalone-c-library-with-docker
7+
8+
ARG CUDA_VERSION=notset
9+
ARG PYTHON_VERSION=notset
10+
ARG RAPIDS_VERSION=notset
11+
FROM rapidsai/ci-wheel:${RAPIDS_VERSION}-cuda${CUDA_VERSION}-rockylinux8-py${PYTHON_VERSION}
12+
13+
# Output directory for the standalone archive. Bind-mount a host folder here
14+
# (e.g. -v $(pwd)/build:/build) so libcuvs_c.tar.gz is written to the host.
15+
ENV CUVS_TARBALL_BUILD_OUTPUT_DIR=/build
16+
ENV CUVS_TARBALL_IN_CONTAINER=1
17+
18+
# Run from repo root; the repo is expected to be bind-mounted at /workspace.
19+
WORKDIR /workspace
20+
21+
# Run the build script, which writes the standalone tarball to the mounted output dir.
22+
# Example: docker run -v $(pwd):/workspace -v $(pwd)/build:/build cuvs-standalone-c [--tarball-build-tests]
23+
ENTRYPOINT ["/bin/bash", "-c", "ci/standalone_c/build_standalone_c.sh \"$@\"", "--"]

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ cuVS comes with pre-built packages that can be installed through [conda](https:/
8484
8585
Please see the [Build and Install Guide](https://docs.rapids.ai/api/cuvs/nightly/build/) for more information on installing the available cuVS packages and building from source.
8686

87+
### Standalone C library (Docker)
88+
89+
To build the standalone C library tarball (`libcuvs_c.tar.gz`) for use in your own C/C++ projects, see **[Building the standalone C library with Docker](https://docs.nvidia.com/cuvs/installation#build-the-standalone-c-library-with-docker)**.
90+
8791
## Getting Started
8892

8993
The following code snippets train an approximate nearest neighbors index for the CAGRA algorithm in the various different languages supported by cuVS.

0 commit comments

Comments
 (0)