Merge branch 'refs/heads/main' into Cjian/cg

microsoft · Sep 17, 2024 · d948315 · d948315
2 parents 0ee4354 + b94ba09
commit d948315
Show file tree

Hide file tree

Showing 643 changed files with 18,279 additions and 28,844 deletions.
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -58,6 +58,90 @@ jobs:
           --use_xnnpack \
           --use_binskim_compliant_compile_flags
 
+  Vcpkg:
+    runs-on: macos-13
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.python_version }}
+
+    - name: "Run vcpkg(x64-osx)"
+      uses: lukka/run-vcpkg@v11
+      with:
+        vcpkgDirectory: "${{ runner.temp }}/vcpkg"
+        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
+        runVcpkgInstall: true
+        vcpkgJsonGlob: "cmake/vcpkg.json"
+        vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
+      env:
+        VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
+        VCPKG_DEFAULT_TRIPLET: "x64-osx"
+        # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
+
+    - name: "Run compile_schema.py"
+      run: |
+        # Runner's host triplet should be x64-osx or arm64-osx
+        export FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/flatbuffers"
+        export PATH="$FLATC_DIR:$PATH"
+        flatc --version
+        python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$(which flatc)"
+
+    - name: "Detect protoc"
+      id: protoc-detect
+      run: |
+        export PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-osx/tools/protobuf"
+        export PATH="$PROTOC_DIR:$PATH"
+        protoc --version
+        echo "protoc_path=$(which protoc)" >> "$GITHUB_OUTPUT"
+
+    - name: "Run build.py(x64-osx)"
+      run: |
+        python ./tools/ci_build/build.py \
+          --build_dir "build/x64-osx" \
+          --skip_submodule_sync \
+          --skip_tests \
+          --compile_no_warning_as_error \
+          --parallel \
+          --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
+          --osx_arch x86_64 \
+          --use_vcpkg \
+          --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
+          --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-osx" \
+          --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
+          --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      shell: bash
+
+    - name: "Run vcpkg(arm64-osx)"
+      uses: lukka/run-vcpkg@v11
+      with:
+        vcpkgDirectory: "${{ runner.temp }}/vcpkg"
+        vcpkgGitCommitId: "1de2026f28ead93ff1773e6e680387643e914ea1" # 2024.07.12
+        runVcpkgInstall: true
+        vcpkgJsonGlob: "cmake/vcpkg.json"
+        vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
+      env:
+        VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
+        VCPKG_DEFAULT_TRIPLET: "arm64-osx"
+        # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
+
+    - name: "Run build.py(arm64-osx)"
+      run: |
+        python ./tools/ci_build/build.py \
+          --build_dir "build/arm64-osx" \
+          --skip_submodule_sync \
+          --skip_tests \
+          --compile_no_warning_as_error \
+          --parallel \
+          --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" \
+          --osx_arch arm64 \
+          --use_vcpkg \
+          --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
+          --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-osx" \
+          --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" \
+          --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+      shell: bash
+
   Objective-C-StaticAnalysis:
     runs-on: macos-14
 

diff --git a/.github/workflows/publish-c-apidocs.yml b/.github/workflows/publish-c-apidocs.yml
@@ -49,4 +49,4 @@ jobs:
         with:
           name: onnxruntime-c-apidocs
           path: _site
-          retention-days: 60
+          retention-days: 30
diff --git a/.github/workflows/publish-csharp-apidocs.yml b/.github/workflows/publish-csharp-apidocs.yml
@@ -28,21 +28,25 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 6.0.x
+        dotnet-version: 8.0.x
+    - name: Install DocFX
+      run: |
+        dotnet tool update -g docfx
+    # NOTE: We need to restore Microsoft.ML.OnnxRuntime.csproj manually to set IncludeMobileTargets=false
+    # docfx doesn't seem to be able to do that properly resulting in build errors
     - name: Restore dependencies
-      run: dotnet restore csharp/ApiDocs/ApiDocs.csproj
-    - name: Download DocFX
       run: |
-        mkdir -p build/docfx
-        wget https://github.com/dotnet/docfx/releases/download/v${DOCFXVERSION}/docfx-linux-x64-v${DOCFXVERSION}.zip -O build/docfx/docfx.zip
-        unzip build/docfx/docfx.zip -d build/docfx
-    - name: Install NuGet
-      uses: nuget/setup-nuget@v2
-    - name: Build Documentation
+        dotnet restore csharp/ApiDocs/ApiDocs.csproj
+        dotnet restore /p:IncludeMobileTargets=false csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
+    - name: docfx metadata
+      run: |
+        docfx metadata csharp/ApiDocs/docfx.json
+    - name: Build ApiDocs project
       run: |
-        build/docfx/docfx metadata csharp/ApiDocs/docfx.json
         dotnet build csharp/ApiDocs/ApiDocs.csproj --no-restore
-        build/docfx/docfx build csharp/ApiDocs/docfx.json
+    - name: docfx build
+      run: |
+        docfx build csharp/ApiDocs/docfx.json
     - name: Log source commit
       run: git rev-parse --short HEAD > csharp/ApiDocs/csharp/source-version.txt
     - name: Move C# docs into site
@@ -55,4 +59,4 @@ jobs:
       with:
         name: onnxruntime-csharp-apidocs
         path: _site
-        retention-days: 60
+        retention-days: 30
diff --git a/.github/workflows/publish-java-apidocs.yml b/.github/workflows/publish-java-apidocs.yml
@@ -47,4 +47,4 @@ jobs:
         with:
           name: onnxruntime-java-apidocs
           path: _site
-          retention-days: 60
+          retention-days: 30
diff --git a/.github/workflows/publish-js-apidocs.yml b/.github/workflows/publish-js-apidocs.yml
@@ -47,4 +47,4 @@ jobs:
         with:
           name: onnxruntime-node-apidocs
           path: _site
-          retention-days: 60
+          retention-days: 30
diff --git a/.github/workflows/publish-objectivec-apidocs.yml b/.github/workflows/publish-objectivec-apidocs.yml
@@ -48,4 +48,4 @@ jobs:
       with:
         name: onnxruntime-objectivec-apidocs
         path: ./_site
-        retention-days: 60
+        retention-days: 30
diff --git a/.github/workflows/publish-python-apidocs.yml b/.github/workflows/publish-python-apidocs.yml
@@ -53,4 +53,4 @@ jobs:
         with:
           name: onnxruntime-python-apidocs
           path: _site
-          retention-days: 60
+          retention-days: 30
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -42,3 +42,90 @@ jobs:
       # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
       - name: Build code
         run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling  --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75
+
+  Vcpkg:
+    runs-on: "windows-latest"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11.x'
+          architecture: 'x64'
+
+      - name: "Run vcpkg(x64-windows)"
+        uses: lukka/run-vcpkg@v11
+        with:
+          vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
+          doNotUpdateVcpkg: true
+          runVcpkgInstall: true
+          vcpkgJsonGlob: "cmake/vcpkg.json"
+          vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
+        env:
+          VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
+          VCPKG_DEFAULT_TRIPLET: "x64-windows"
+          # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
+
+      - name: "Run compile_schema.py"
+        run: |
+          # Runner's host triplet should be x64-windows or arm64-windows
+          $FLATC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/flatbuffers"
+          $env:PATH="$FLATC_DIR;$env:PATH"
+          flatc --version
+          $FLATC_PATH = Join-Path "$FLATC_DIR" "flatc.exe"
+          python onnxruntime/core/flatbuffers/schema/compile_schema.py --flatc "$FLATC_PATH"
+        shell: pwsh
+
+      - name: "Detect protoc"
+        id: protoc-detect
+        run: |
+          $PROTOC_DIR="${{ github.workspace }}/.build/${{ runner.arch }}-windows/tools/protobuf"
+          $env:PATH="$PROTOC_DIR;$env:PATH"
+          protoc --version
+          $PROTOC_PATH = Join-Path "$PROTOC_DIR" "protoc.exe"
+          "protoc_path=$PROTOC_PATH" >> $env:GITHUB_OUTPUT
+        shell: pwsh
+
+      - name: "Run build.py(x64-windows)"
+        run: |
+          python tools\ci_build\build.py `
+            --build_dir "cmake_build/x64-windows" `
+            --skip_submodule_sync `
+            --skip_tests `
+            --compile_no_warning_as_error `
+            --parallel `
+            --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
+            --use_vcpkg `
+            --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
+            --cmake_extra_defines "VCPKG_TARGET_TRIPLET=x64-windows" `
+            --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
+            --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+        shell: pwsh
+
+      - name: "Run vcpkg(arm64-windows)"
+        uses: lukka/run-vcpkg@v11
+        with:
+          vcpkgDirectory: "C:/vcpkg" # use VCPKG_INSTALLATION_ROOT of the image
+          doNotUpdateVcpkg: true
+          runVcpkgInstall: true
+          vcpkgJsonGlob: "cmake/vcpkg.json"
+          vcpkgConfigurationJsonGlob: "cmake/vcpkg-configuration.json"
+        env:
+          VCPKG_INSTALLED_DIR: "${{ github.workspace }}/.build"
+          VCPKG_DEFAULT_TRIPLET: "arm64-windows"
+          # VCPKG_BINARY_SOURCES: "default" # https://learn.microsoft.com/en-us/vcpkg/reference/binarycaching
+
+      - name: "Run build.py(arm64-windows)"
+        run: |
+          python tools\ci_build\build.py `
+            --build_dir "cmake_build/arm64-windows" --arm64 `
+            --skip_submodule_sync `
+            --skip_tests `
+            --compile_no_warning_as_error `
+            --parallel `
+            --path_to_protoc_exe "${{ steps.protoc-detect.outputs.protoc_path }}" `
+            --use_vcpkg `
+            --cmake_extra_defines "CMAKE_TOOLCHAIN_FILE:FILEPATH=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" `
+            --cmake_extra_defines "VCPKG_TARGET_TRIPLET=arm64-windows" `
+            --cmake_extra_defines "VCPKG_INSTALLED_DIR:PATH=${{ github.workspace }}/.build" `
+            --cmake_extra_defines "VCPKG_INSTALL_OPTIONS=--x-feature=tests"
+        shell: pwsh
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -22,7 +22,5 @@
         "-build/include_subdir",
         "-runtime/references"
     ],
-    "files.associations": {
-        "span": "cpp"
-    }
+    "C_Cpp.autoAddFileAssociations": false
 }
diff --git a/NuGet.config b/NuGet.config
@@ -6,6 +6,8 @@
   <packageSources>
     <clear />
     <add key="NuGet Official" value="https://api.nuget.org/v3/index.json" />
+    <add key="MAUI DeviceRunners" value="https://pkgs.dev.azure.com/mattleibow/OpenSource/_packaging/test-device-runners/nuget/v3/index.json" />
+    <add key="XHarness" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />

diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json
@@ -469,7 +469,7 @@
             "type": "pip",
             "pip": {
                "Name": "transformers",
-               "Version": "4.36.0"
+               "Version": "4.38.0"
             },
             "comments": "Installed in the training docker image"
          }

diff --git a/cgmanifests/generated/cgmanifest.json b/cgmanifests/generated/cgmanifest.json
@@ -146,7 +146,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "0da379fc4808f9601faef392352018c741c0f297",
+          "commitHash": "309b75c9e56e0a674bf78d59872ce131f814dfb6",
           "repositoryUrl": "https://github.com/google/XNNPACK.git"
         },
         "comments": "googlexnnpack"
@@ -361,6 +361,16 @@
         },
         "comments": "cudnn_frontend"
       }
+    },
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "commitHash": "511eb80847afe6bded34ec491a38d5d78ba2d604",
+          "repositoryUrl": "https://github.com/google/dawn.git"
+        },
+        "comments": "dawn"
+      }
     }
   ]
 }