Merge branch 'main' into kvaishnavi/logit-softcapping

.github/policies/issueLabeler.yml

@@ -1,86 +1,148 @@
 id:
 name: Issue Triage
-description: Assign label to issues 
+description: Assign label to issues
 owner:
 resource: repository
 where:
 configuration:
   resourceManagementConfiguration:
     eventResponderTasks:
-      - if:
+      - description: Add api:java label to new issues
+        if:
           - payloadType: Issues
-          - and:
-              - isOpen
-              - not:
-                  and:
-                    - isAssignedToSomeone
-                    - isLabeled
+          - isOpen
+          - not:
+              labelRemoved:
+                label: api:java
+          - or:
+              - titleContains:
+                  pattern: (?i)\bjava\b
+                  isRegex: True
+              - bodyContains:
+                  pattern: (?i)\bjava\b
+                  isRegex: True
         then:
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/\bcuda\b/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/\bcuda\b/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: ep:CUDA
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/\bjava\b/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/\bjava\b/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: api:Java
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/(\bdirect\s*ml\b|\bdml\b)/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/(\bdirect\s*ml\b|\bdml\b)/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: ep:DML
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/(\bobj(?:ective)?-?c\b|\bnnapi\b|\bmobile\b|\bandroid\b|\bios\b|\bxamarin\b|\bmaui\b)/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/(\bobj(?:ective)?-?c\b|\bnnapi\b|\bmobile\b|\bandroid\b|\bios\b|\bxamarin\b|\bmaui\b)/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: platform:mobile
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/(\bwindows\b|\bwinrt\b|\bwinml\b)/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/(\bwindows\b|\bwinrt\b|\bwinml\b)/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: platform:windows
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: '/\btransformers(?!\.js)\b/i'
-                      isRegex: True
-                  - bodyContains:
-                      pattern: '/\btransformers(?!\.js)\b/i'
-                      isRegex: True
-            then:
-              - addLabel:
-                  label: model:transformer
+          - addLabel:
+              label: api:java
+      - description: Add api:javascript label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: api:javascript
+          - or:
+              - titleContains:
+                  pattern: (?i)\bjavascript\b
+                  isRegex: True
+              - bodyContains:
+                  pattern: (?i)\bjavascript\b
+                  isRegex: True
+        then:
+          - addLabel:
+              label: api:javascript
+      - description: Add ep:CUDA label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: ep:CUDA
+          - titleContains:
+              pattern: (?i)\bcuda\b
+              isRegex: True
+        then:
+          - addLabel:
+              label: ep:CUDA
+      - description: Add ep:DML label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: ep:DML
+          - or:
+              - titleContains:
+                  pattern: (?i)(\bdirect\s*ml\b|\bdml\b)
+                  isRegex: True
+              - bodyContains:
+                  pattern: (?i)(\bdirect\s*ml\b|\bdml\b)
+                  isRegex: True
+        then:
+          - addLabel:
+              label: ep:DML 
+      - description: Add ep:TensorRT label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: ep:TensorRT
+          - titleContains:
+                  pattern: (?i)(\btensor\s*rt\b|\btrt\b)
+                  isRegex: True
+        then:
+          - addLabel:
+              label: ep:TensorRT
+      - description: Add platform:jetson label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: platform:jetson
+          - or:
+              - titleContains:
+                  pattern: (?i)(\bjetson\b|\bjetpack\b)
+                  isRegex: True
+              - bodyContains:
+                  pattern: (?i)(\bjetson\b|\bjetpack\b)
+                  isRegex: True
+        then:
+          - addLabel:
+              label: platform:jetson
+      - description: Add platform:mobile label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: platform:mobile
+          - or:
+              - titleContains:
+                  pattern: (?i)(\bobj(?:ective)?-?c\b|\bnnapi\b|\bmobile\b|\bandroid\b|\bios\b|\bxamarin\b|\bmaui\b)
+                  isRegex: True
+              - bodyContains:
+                  pattern: (?i)(\bobj(?:ective)?-?c\b|\bnnapi\b|\bmobile\b|\bandroid\b|\bios\b|\bxamarin\b|\bmaui\b)
+                  isRegex: True
+        then:
+          - addLabel:
+              label: platform:mobile
+      - description: Add platform:windows label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: platform:windows
+          - titleContains:
+              pattern: (?i)(\bwindows\b|\bwinrt\b|\bwinml\b)
+              isRegex: True
+        then:
+          - addLabel:
+              label: platform:windows
+      - description: Add quantization label to new issues
+        if:
+          - payloadType: Issues
+          - isOpen
+          - not:
+              labelRemoved:
+                label: quantization
+          - titleContains:
+              pattern: (?i)(quant|\bqdq\b)
+              isRegex: True
+        then:
+          - addLabel:
+              label: quantization
 onFailure:
 onSuccess:

.github/policies/test_issueLabeler.yml

@@ -7,18 +7,15 @@ where:
 configuration:
   resourceManagementConfiguration:
     eventResponderTasks:
-      - if:
+      - description: Add ep:CUDA label to new issues
+        if:
           - payloadType: Issues
           - isOpen
+          - titleContains:
+              pattern: (?i)\bcuda\b
+              isRegex: True
         then:
-          - if:
-              - or:
-                  - titleContains:
-                      pattern: shark
-                  - bodyContains:
-                      pattern: strawberry
-            then:
-              - addLabel:
-                  label: wontfix 
+          - addLabel:
+              label: ep:CUDA 
 onFailure:
 onSuccess:

.github/workflows/clang-format-lint.yml

@@ -11,8 +11,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11.x'
+          architecture: 'x64'
       - uses: DoozyX/[email protected]
         with:
           source: './src'
           extensions: 'h,cpp'
-          clangFormatVersion: 17
+          clangFormatVersion: 17

.github/workflows/linux-cpu-arm64-build.yml

@@ -92,4 +92,4 @@ jobs:
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/onnxruntime_src/build/cpu/ /onnxruntime_src/build/cpu/test/unit_tests"
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/onnxruntime_src/ort/lib/ /onnxruntime_src/build/cpu/unit_tests"

.github/workflows/linux-cpu-x64-build.yml

@@ -68,39 +68,37 @@ jobs:
       - name: Install the python wheel and test dependencies
         run: |
           python3 -m pip install -r test/python/requirements.txt --user
-          python3 -m pip install -r test/python/requirements-cpu.txt --user
-          python3 -m pip install build/cpu/wheel/onnxruntime_genai*.whl --user --no-deps
+          python3 -m pip install -r test/python/cpu/torch/requirements.txt --user
+          python3 -m pip install -r test/python/cpu/ort/requirements.txt --user
+          python3 -m pip install --user --no-index --no-deps --find-links build/cpu/wheel onnxruntime_genai
 
-      - name: Get HuggingFace Token
+      - name: Use Dummy HuggingFace Token
         run: |
-          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
-          HF_TOKEN=$(az keyvault secret show --vault-name anubissvcsecret --name ANUBIS-HUGGINGFACE-TOKEN --query value)
-          echo "::add-mask::$HF_TOKEN"
-          echo "HF_TOKEN=$HF_TOKEN" >> $GITHUB_ENV
+          echo "HF_TOKEN=12345" >> $GITHUB_ENV
 
-      - name: Remove the ort lib and header files
+      - name: Verify Build Artifacts
+        if: always()
+        continue-on-error: true
         run: |
-          rm -rf ort
+          ls -l ${{ github.workspace }}/build/cpu
+          ls -l ${{ github.workspace }}/build/cpu/wheel
 
       # This will also download all the test models to the test/test_models directory
       # These models are used by the python tests as well as C#, C++ and others.
       - name: Run the python tests
         run: |
+          export ORTGENAI_LOG_ORT_LIB=1
           python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models
 
       - name: Build the C# API and Run the C# Tests
         run: |
+          export ORTGENAI_LOG_ORT_LIB=1
           cd test/csharp
-          dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/"
-
-      - name: Verify Build Artifacts
-        if: always()
-        continue-on-error: true
-        run: |
-          ls -l ${{ github.workspace }}/build/cpu
+          dotnet test /p:Configuration=Release /p:NativeBuildOutputDir="../../build/cpu/" /p:OrtLibDir="../../ort/lib/"
 
       - name: Run tests
         run: |
           set -e -x
-          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/build/cpu/
-          ./build/cpu/test/unit_tests
+          export ORTGENAI_LOG_ORT_LIB=1
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/ort/lib
+          ./build/cpu/unit_tests

.github/workflows/linux-cpu-x64-nightly-build.yml

@@ -51,15 +51,13 @@ jobs:
       - name: Install the python wheel and test dependencies
         run: |
           python3 -m pip install -r test/python/requirements.txt --user
-          python3 -m pip install -r test/python/requirements-cpu.txt --user
+          python3 -m pip install -r test/python/cpu/torch/requirements.txt --user
+          python3 -m pip install -r test/python/cpu/ort/requirements.txt --user
           python3 -m pip install build/cpu/wheel/onnxruntime_genai*.whl --no-deps
 
-      - name: Get HuggingFace Token
+      - name: Use Dummy HuggingFace Token
         run: |
-          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
-          HF_TOKEN=$(az keyvault secret show --vault-name anubissvcsecret --name ANUBIS-HUGGINGFACE-TOKEN --query value)
-          echo "::add-mask::$HF_TOKEN"
-          echo "HF_TOKEN=$HF_TOKEN" >> $GITHUB_ENV
+          echo "HF_TOKEN=12345" >> $GITHUB_ENV
 
       - name: Run the python tests
         run: |

.github/workflows/linux-gpu-x64-build.yml

@@ -20,8 +20,8 @@ env:
 jobs:
   linux-cuda-x64-build:
     env :
-      PYTHON_EXECUTABLE: "/opt/python/cp38-cp38/bin/python3.8"
-    runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2004-T4" ]
+      PYTHON_EXECUTABLE: "/opt/python/cp310-cp310/bin/python3.10"
+    runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-A10" ]
     steps:
       - name: Checkout OnnxRuntime GenAI repo
         uses: actions/checkout@v4
@@ -109,16 +109,9 @@ jobs:
             bash -c " \
               /usr/bin/cmake --build --preset linux_gcc_cuda_release"
 
-      - name: Get HuggingFace Token
+      - name: Use Dummy HuggingFace Token
         run: |
-          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
-          HF_TOKEN=$(az keyvault secret show --vault-name anubissvcsecret --name ANUBIS-HUGGINGFACE-TOKEN --query value)
-          echo "::add-mask::$HF_TOKEN"
-          echo "HF_TOKEN=$HF_TOKEN" >> $GITHUB_ENV
-
-      - name: Remove the ort lib and header files
-        run: |
-          rm -rf ort
+          echo "HF_TOKEN=12345" >> $GITHUB_ENV
 
       - name: Install the onnxruntime-genai Python wheel and run python test
         run: |
@@ -131,7 +124,8 @@ jobs:
             -e HF_TOKEN=$HF_TOKEN \
             -w /ort_genai_src onnxruntimecudabuildx64 bash -c " \
               ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/requirements.txt --user && \
-              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/requirements-cuda.txt --user && \
+              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/cuda/torch/requirements.txt --user && \
+              ${{ env.PYTHON_EXECUTABLE }} -m pip install -r test/python/cuda/ort/requirements.txt --user && \
               ${{ env.PYTHON_EXECUTABLE }} -m pip install /ort_genai_src/build/cuda/wheel/onnxruntime_genai*manylinux*.whl --no-deps --user && \
               ${{ env.PYTHON_EXECUTABLE }} test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models --e2e"
 
@@ -154,4 +148,4 @@ jobs:
             --rm \
             --volume /data/ortgenai_pytorch_models:/data/ortgenai_pytorch_models \
             --volume $GITHUB_WORKSPACE:/ort_genai_src \
-            -w /ort_genai_src onnxruntimecudabuildx64 bash -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/ort_genai_src/build/cuda/ /ort_genai_src/build/cuda/test/unit_tests"
+            -w /ort_genai_src onnxruntimecudabuildx64 bash -c "ORTGENAI_LOG_ORT_LIB=1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/ort_genai_src/build/cuda/ /ort_genai_src/build/cuda/unit_tests"