diff --git a/Jenkinsfile b/Jenkinsfile
index a06e94d5a7ab..9eef7a7bc4aa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -179,10 +179,10 @@ def BuildCPUARM64() {
     ${dockerRun} ${container_type} ${docker_binary} bash -c "cd build && ctest --extra-verbose"
     ${dockerRun} ${container_type} ${docker_binary} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
     ${dockerRun} ${container_type} ${docker_binary} python tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} ${wheel_tag}
-    ${dockerRun} ${container_type} ${docker_binary} auditwheel repair --plat ${wheel_tag} python-package/dist/*.whl
+    ${dockerRun} ${container_type} ${docker_binary} bash -c "auditwheel repair --plat ${wheel_tag} python-package/dist/*.whl && python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}"
     mv -v wheelhouse/*.whl python-package/dist/
     # Make sure that libgomp.so is vendored in the wheel
-    ${dockerRun} ${container_type} ${docker_binary} bash -c "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
+    ${dockerRun} ${container_type} ${docker_binary} bash -c "unzip -l python-package/dist/*.whl | grep libgomp || exit -1"
     """
     echo 'Stashing Python wheel...'
     stash name: "xgboost_whl_arm64_cpu", includes: 'python-package/dist/*.whl'
@@ -231,6 +231,7 @@ def BuildCUDA(args) {
     if (args.cuda_version == ref_cuda_ver) {
       sh """
       ${dockerRun} auditwheel_x86_64 ${docker_binary} auditwheel repair --plat ${wheel_tag} python-package/dist/*.whl
+      ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}
       mv -v wheelhouse/*.whl python-package/dist/
       # Make sure that libgomp.so is vendored in the wheel
       ${dockerRun} auditwheel_x86_64 ${docker_binary} bash -c "unzip -l python-package/dist/*.whl | grep libgomp  || exit -1"
diff --git a/tests/ci_build/insert_vcomp140.py b/tests/ci_build/insert_vcomp140.py
index 9e40ae307b99..dce593501ff8 100644
--- a/tests/ci_build/insert_vcomp140.py
+++ b/tests/ci_build/insert_vcomp140.py
@@ -11,7 +11,7 @@
 
 for wheel_path in sorted(glob.glob(sys.argv[1])):
     m = re.search(r'xgboost-(.*)-py3', wheel_path)
-    assert m
+    assert m, f'wheel_path = {wheel_path}'
     version = m.group(1)
 
     with zipfile.ZipFile(wheel_path, 'a') as f:
diff --git a/tests/ci_build/rename_whl.py b/tests/ci_build/rename_whl.py
index 7b9db877c1c2..c8d2bf812eda 100644
--- a/tests/ci_build/rename_whl.py
+++ b/tests/ci_build/rename_whl.py
@@ -26,8 +26,9 @@ def cd(path):
 with cd(dirname):
     tokens = basename.split('-')
     assert len(tokens) == 5
+    version = tokens[1].split('+')[0]
     keywords = {'pkg_name': tokens[0],
-                'version': tokens[1],
+                'version': version,
                 'commit_id': commit_id,
                 'platform_tag': platform_tag}
     new_name = '{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl'.format(**keywords)