Merge branch 'main' into improve_stuck_system_tests

Author: daniel-sanche

.github/workflows/django-spanner-django5.2_tests.yml

@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           persist-credentials: false
-      - uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
         id: filter
         with:
           filters: |

.github/workflows/django-spanner-django6.0_tests.yml

@@ -0,0 +1,91 @@
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    paths:
+      - 'packages/django-google-spanner/**'
+      - '.github/workflows/django-spanner-django6.0_tests.yml'
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/django-google-spanner/**'
+      - '.github/workflows/django-spanner-django6.0_tests.yml'
+
+defaults:
+  run:
+    working-directory: packages/django-google-spanner
+
+name: django-spanner-django6.0-tests
+jobs:
+  check_changes:
+    runs-on: ubuntu-latest
+    outputs:
+      run_django_spanner: ${{ steps.filter.outputs.django_spanner }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        id: filter
+        with:
+          filters: |
+            django_spanner:
+              - 'packages/django-google-spanner/**'
+              - '.github/workflows/django-spanner-django6.0_tests.yml'
+
+  system-tests:
+    needs: check_changes
+    if: ${{ needs.check_changes.outputs.run_django_spanner == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+        include:
+          - chunk: 0
+            apps: admin_changelist admin_ordering distinct_on_fields expressions_window fixtures_model_package datetimes custom_methods generic_inline_admin field_defaults datatypes empty m2o_recursive many_to_one_null migrate_signals model_forms.test_uuid view_tests update select_related_onetoone sessions_tests
+          - chunk: 1
+            apps: db_functions save_delete_hooks get_object_or_404 model_indexes custom_pk indexes transaction_hooks constraints schema custom_columns i18n from_db_value sites_tests mutually_referential model_package defer_regress update_only_fields backends redirects_tests expressions get_or_create foreign_object generic_relations_regress many_to_many select_related generic_relations queryset_pickle model_inheritance
+          - chunk: 2
+            apps: model_options known_related_objects m2m_signals delete_regress fixtures generic_views model_inheritance_regress nested_foreign_keys lookup delete model_formsets
+          - chunk: 3
+            apps: signals or_lookups m2m_through_regress filtered_relation servers m2m_through fixtures_regress timezones model_forms.tests
+          - chunk: 4
+            apps: introspection multiple_database null_fk_ordering ordering m2m_intermediary null_fk max_lengths dates force_insert_update test_client m2m_multiple test_client_regress sitemaps_tests admin_inlines transactions null_queries test_runner m2m_and_m2o prefetch_related m2m_regress file_uploads sites_framework auth_tests forms_tests inline_formsets order_with_respect_to contenttypes_tests defer
+          - chunk: 5
+            apps: file_storage m2m_recursive reverse_lookup managers_regress basic annotations unmanaged_models string_lookup aggregation_regress reserved_names select_for_update many_to_one cache select_related_regress flatpages_tests model_formsets_regress
+          - chunk: 6
+            apps: model_fields queries.test_bulk_update queries.test_explain
+          - chunk: 7
+            apps: queries.test_iterator queries.test_q queries.test_query queries.test_qs_combinators
+          - chunk: 8
+            apps: inspectdb custom_managers migrations validation get_earliest_or_latest proxy_model_inheritance one_to_one raw_query bulk_create
+          - chunk: 9
+            apps: queries.tests
+
+    services:
+      emulator:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest # zizmor: ignore[unpinned-images]
+        ports:
+          - 9010:9010
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+      - name: Setup Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.12"
+      - name: Run Django tests
+        run: sh django_test_suite_6.0.sh
+        env:
+          SPANNER_EMULATOR_HOST: localhost:9010
+          GOOGLE_CLOUD_PROJECT: emulator-test-project
+          GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true
+          RUNNING_SPANNER_BACKEND_TESTS: 1
+          SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests
+          DJANGO_TEST_APPS: ${{ matrix.apps }}

.github/workflows/django-spanner-foreign_keys.yaml

@@ -39,6 +39,10 @@ jobs:
     needs: check_changes
     if: ${{ needs.check_changes.outputs.run_django_spanner == 'true' }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
 
     services:
       emulator-0:
@@ -54,7 +58,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
       - name: Run Django foreign key test
         run: sh foreign_key_test.sh
         env:

.github/workflows/django-spanner-integration-tests-against-emulator-3.10.yml

@@ -39,6 +39,10 @@ jobs:
     needs: check_changes
     if: ${{ needs.check_changes.outputs.run_django_spanner == 'true' }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12"]
 
     services:
       emulator:
@@ -52,14 +56,14 @@ jobs:
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           persist-credentials: false
-      - name: Set up Python 3.10
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
       - name: Install nox
         run: python -m pip install nox
       - name: Run nox
-        run: nox -s unit-3.10
+        run: nox -s "unit-${{ matrix.python-version }}"
         env:
           SPANNER_EMULATOR_HOST: localhost:9010
           GOOGLE_CLOUD_PROJECT: emulator-test-project

.github/workflows/unittest.yml

@@ -40,15 +40,33 @@ jobs:
       # Each weight is roughly 1 minute of expected execution time
       # Default for unset packages is 1
       PACKAGE_WEIGHTS: |
-        google-ai-generativelanguage: 4
-        google-auth: 5
+        django-google-spanner: 2
+        gapic-generator: 3
+        google-ai-generativelanguage: 3
+        google-api-core: 2
+        google-auth: 2
+        google-cloud-bigquery: 4
+        google-cloud-bigquery-storage: 2
+        google-cloud-bigtable: 4
         google-cloud-compute: 12
         google-cloud-compute-v1beta: 12
-        google-cloud-dialogflow: 6
-        google-cloud-dialogflow-cx: 6
-        google-cloud-discoveryengine: 8
-        google-cloud-retail: 5
-        google-shopping-merchant-accounts: 4
+        google-cloud-datastore: 3
+        google-cloud-dialogflow: 4
+        google-cloud-dialogflow-cx: 4
+        google-cloud-discoveryengine: 4
+        google-cloud-firestore: 3
+        google-cloud-logging: 3
+        google-cloud-monitoring: 3
+        google-cloud-ndb: 2
+        google-cloud-pubsub: 2
+        google-cloud-retail: 3
+        google-cloud-spanner: 4
+        google-cloud-storage: 4
+        google-shopping-merchant-accounts: 3
+        pandas-gbq: 2
+        proto-plus: 1
+        sqlalchemy-bigquery: 2
+        sqlalchemy-spanner: 2
     steps:
     - name: Checkout
       uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -156,12 +174,52 @@ jobs:
         path: .coverage.${{ matrix.python }}.*
         include-hidden-files: true
 
+  core-deps:
+    needs: initialize
+    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: true
+      matrix:
+        python: ["3.14"]
+        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
+    name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
+      # See https://github.com/googleapis/google-cloud-python/issues/12013
+      # and https://github.com/actions/checkout#checkout-head.
+      with:
+        fetch-depth: 2
+        persist-credentials: false
+    - name: Setup Python
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+      with:
+        python-version: ${{ matrix.python }}
+        cache: 'pip'
+        allow-prereleases: true
+    - name: Install nox
+      run: |
+        pip install nox
+    - name: Run core_deps_from_source for ${{ matrix.package_shard.description }}
+      env:
+        BUILD_TYPE: presubmit
+        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
+        TEST_TYPE: core_deps_from_source
+        PY_VERSION: ${{ matrix.python }}
+        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
+        NOX_DEFAULT_VENV_BACKEND: "virtualenv"
+        NOXFORCEPYTHON: ${{ matrix.python }}
+      run: |
+        ci/run_conditional_tests.sh
+
   all-tests:
-    needs: [initialize, unit]
+    needs: [initialize, unit, core-deps]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - name: Check unit test results
+      - name: Check test results
         run: |
           # 1. Check initialize job
           if [[ "${{ needs.initialize.result }}" != "success" ]]; then
@@ -173,7 +231,12 @@ jobs:
             echo "Unit tests failed"
             exit 1
           fi
-          echo "All unit tests passed or were skipped"
+          # 3. Check core dependencies test shards
+          if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then
+            echo "Core dependencies tests failed"
+            exit 1
+          fi
+          echo "All unit and core dependencies tests passed or were skipped"
 
   cover:
     if: always() && !cancelled() && needs.all-tests.result == 'success'

.kokoro/presubmit/sqlalchemy_compliance.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run compliance nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "compliance"
+}

.kokoro/presubmit/system.cfg

@@ -5,3 +5,9 @@ env_vars: {
     key: "NOX_SESSION"
     value: "system-3.12"
 }
+
+# Skip compliance tests in system job since they run in dedicated compliance job
+env_vars: {
+    key: "RUN_COMPLIANCE_TESTS"
+    value: "false"
+}

.kokoro/system.sh

@@ -286,6 +286,12 @@ for path in `find 'packages' \
     files_to_check=("${package_path}")
   fi
 
+  # When running compliance tests, only test packages that have compliance suites
+  if [[ "${NOX_SESSION:-}" == "compliance"* && "${package_name:-}" != "sqlalchemy-"* ]]; then
+    printf "SKIP %-20s %-40s %s\n" "[not_applicable]" "${package_name}" "${commit_hash:-HEAD}"
+    continue
+  fi
+
   set +e
   # Passing the array expanded as arguments to git diff.
   package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check[@]}" | wc -l)

.librarian/generator-input/client-post-processing/add-dependency-google-cloud-common.yaml

@@ -19,14 +19,14 @@ replacements:
     ]
     before: |
       dependencies = \[
-          "google-api-core\[grpc\] >= 2.25.0, <3.0.0",
+          "google-api-core\[grpc\] >= 2.28.0, <3.0.0",
           # Exclude incompatible versions of `google-auth`
           # See https://github.com/googleapis/google-cloud-python/issues/12364
           "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",
           "grpcio >= 1.59.0, < 2.0.0",
     after: |
       dependencies = [
-          "google-api-core[grpc] >= 2.25.0, <3.0.0",
+          "google-api-core[grpc] >= 2.28.0, <3.0.0",
           # Exclude incompatible versions of `google-auth`
           # See https://github.com/googleapis/google-cloud-python/issues/12364
           "google-auth >= 2.14.1, <3.0.0,!=2.24.0,!=2.25.0",