From 345fd84551e3153c890ec4fbbf16f9704441b936 Mon Sep 17 00:00:00 2001 From: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> Date: Sat, 4 Jan 2025 13:57:25 -0700 Subject: [PATCH 1/3] Remove some unnecessary FAB test setup code (#45400) --- .../api_endpoints/test_dag_endpoint.py | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py index bd47271204816..853ba3e643606 100644 --- a/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py +++ b/providers/tests/fab/auth_manager/api_endpoints/test_dag_endpoint.py @@ -16,16 +16,11 @@ # under the License. from __future__ import annotations -import os -from datetime import datetime - import pendulum import pytest from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP -from airflow.models import DagBag, DagModel -from airflow.models.dag import DAG -from airflow.operators.empty import EmptyOperator +from airflow.models import DagModel from airflow.security import permissions from airflow.utils.session import provide_session @@ -74,27 +69,6 @@ def configured_app(minimal_app_for_auth_api): }, ) - with DAG( - DAG_ID, - schedule=None, - start_date=datetime(2020, 6, 15), - doc_md="details", - params={"foo": 1}, - tags=["example"], - ) as dag: - EmptyOperator(task_id=TASK_ID) - - with DAG(DAG2_ID, schedule=None, start_date=datetime(2020, 6, 15)) as dag2: # no doc_md - EmptyOperator(task_id=TASK_ID) - - with DAG(DAG3_ID, schedule=None) as dag3: # DAG start_date set to None - EmptyOperator(task_id=TASK_ID, start_date=datetime(2019, 6, 12)) - - dag_bag = DagBag(os.devnull, include_examples=False) - dag_bag.dags = {dag.dag_id: dag, dag2.dag_id: dag2, dag3.dag_id: dag3} - - app.dag_bag = dag_bag - yield app delete_user(app, username="test_granular_permissions") From de6d83a00fd7af30586e58461428c8d3c74d55d3 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 4 Jan 2025 22:36:45 +0100 Subject: [PATCH 2/3] Update uv, pip and pre-commit versions automatically in more places. (#45398) --- .github/actions/install-pre-commit/action.yml | 7 +- .github/workflows/basic-tests.yml | 3 +- .pre-commit-config.yaml | 8 +- contributing-docs/08_static_code_checks.rst | 2 +- dev/breeze/doc/ci/02_images.md | 56 ++--- .../doc/images/output_static-checks.svg | 2 +- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 2 +- scripts/ci/pre_commit/update_installers.py | 161 ------------- .../update_installers_and_pre_commit.py | 218 ++++++++++++++++++ 10 files changed, 260 insertions(+), 201 deletions(-) delete mode 100755 scripts/ci/pre_commit/update_installers.py create mode 100755 scripts/ci/pre_commit/update_installers_and_pre_commit.py diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index b4c6a6c9d546a..abdd3ea98ffc9 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -19,19 +19,18 @@ name: 'Install pre-commit' description: 'Installs pre-commit and related packages' inputs: - # TODO(potiuk): automate update of these versions python-version: description: 'Python version to use' default: "3.9" uv-version: description: 'uv version to use' - default: "0.5.14" + default: "0.5.14" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' - default: "4.0.1" + default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version pre-commit-uv-version: description: 'pre-commit-uv version to use' - default: "4.1.4" + default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version runs: using: "composite" steps: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 297a912ea8122..353f65d9a6c9c 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -307,11 +307,12 @@ jobs: run: > pre-commit run --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers || true + --hook-stage manual update-installers-and-pre-commit || true if: always() env: UPGRADE_UV: "true" UPGRADE_PIP: "false" + UPGRADE_PRE_COMMIT: "true" - name: "Run automated upgrade for pip" run: > pre-commit run diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09690edc5db6c..c5d0d154b88a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -197,12 +197,12 @@ repos: additional_dependencies: ['pyyaml'] pass_filenames: false require_serial: true - - id: update-installers - name: Update installers to latest (manual) - entry: ./scripts/ci/pre_commit/update_installers.py + - id: update-installers-and-pre-commit + name: Update installers and pre-commit to latest (manual) + entry: ./scripts/ci/pre_commit/update_installers_and_pre_commit.py stages: ['manual'] language: python - files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers.py$ + files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit.py$ pass_filenames: false require_serial: true additional_dependencies: ['pyyaml', 'rich>=12.4.4', 'requests'] diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index abfb737890460..78462afe3057f 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -374,7 +374,7 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-installed-providers-to-be-sorted | Sort and uniquify installed_providers.txt | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installers | Update installers to latest (manual) | | +| update-installers-and-pre-commit | Update installers and pre-commit to latest (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-local-yml-file | Update mounts in the local yml file | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index b613f67c72d04..3d1d7d8b53eb7 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -419,33 +419,35 @@ DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ The following build arguments (`--build-arg` in docker build command) can be used for CI images: -| Build argument | Default value | Description | -|-----------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | -| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | -| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | -| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | -| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | -| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | -| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | -| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | -| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | -| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | -| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | -| `AIRFLOW_CONSTRAINTS_REFERENCE` | | reference (branch or tag) from GitHub repository from which constraints are used. By default it is set to `constraints-main` but can be `constraints-2-X`. | -| `AIRFLOW_EXTRAS` | `all` | extras to install | -| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | -| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | -| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | -| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | -| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | -| `DEV_APT_DEPS` | | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) | -| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | -| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | -| `AIRFLOW_PIP_VERSION` | `24.3.1` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.5.14` | UV version used. | -| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | -| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | +| Build argument | Default value | Description | +|---------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------| +| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | +| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | +| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | +| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | +| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | +| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | +| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | +| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | +| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | +| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | +| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | +| `AIRFLOW_CONSTRAINTS_REFERENCE` | `constraints-main` | reference (branch or tag) from GitHub repository from which constraints are used. | +| `AIRFLOW_EXTRAS` | `all` | extras to install | +| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | +| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | +| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | +| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | +| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | +| `DEV_APT_DEPS` | | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) | +| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | +| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | +| `AIRFLOW_PIP_VERSION` | `24.3.1` | `pip` version used. | +| `AIRFLOW_UV_VERSION` | `0.5.14` | `uv` version used. | +| `AIRFLOW_PRE_COMMIT_VERSION` | `4.0.1` | `pre-commit` version used. | +| `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | +| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index a2422824dd981..bfea55c495fb5 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -371,7 +371,7 @@ │update-breeze-readme-config-hash | update-chart-dependencies |                   │ │update-common-sql-api-stubs | update-er-diagram | update-extras |                │ │update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-installers |                    │ +│update-installed-providers-to-be-sorted | update-installers-and-pre-commit |     │ │update-local-yml-file | update-migration-references |                            │ │update-openapi-spec-tags-to-be-sorted | update-providers-dependencies |          │ │update-providers-init-py | update-reproducible-source-date-epoch |               │ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index a685635825ff4..38529eb9753cf 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -1e545fdd89efbdd78a883519b6d93ce2 +6239e6a528459f731b6908ce668a8950 diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index a4495df607ecd..8667a2cc4b785 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -141,7 +141,7 @@ "update-in-the-wild-to-be-sorted", "update-inlined-dockerfile-scripts", "update-installed-providers-to-be-sorted", - "update-installers", + "update-installers-and-pre-commit", "update-local-yml-file", "update-migration-references", "update-openapi-spec-tags-to-be-sorted", diff --git a/scripts/ci/pre_commit/update_installers.py b/scripts/ci/pre_commit/update_installers.py deleted file mode 100755 index 28de198824064..0000000000000 --- a/scripts/ci/pre_commit/update_installers.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import os -import re -import sys -from pathlib import Path - -import requests - -sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported -from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console - -FILES_TO_UPDATE = [ - AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile", - AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", - AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", - AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "global_constants.py", - AIRFLOW_SOURCES_ROOT_PATH - / "dev" - / "breeze" - / "src" - / "airflow_breeze" - / "commands" - / "release_management_commands.py", -] - - -DOC_FILES_TO_UPDATE: list[Path] = [ - AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" / "02_images.md" -] - - -def get_latest_pypi_version(package_name: str) -> str: - response = requests.get(f"https://pypi.org/pypi/{package_name}/json") - response.raise_for_status() # Ensure we got a successful response - data = response.json() - latest_version = data["info"]["version"] # The version info is under the 'info' key - return latest_version - - -AIRFLOW_PIP_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)") -AIRFLOW_PIP_QUOTED_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")") -PIP_QUOTED_PATTERN = re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")") -PIP_QUOTED_PATTERN_NO_SPACES = re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")") -AIRFLOW_PIP_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)") -AIRFLOW_PIP_UPGRADE_PATTERN = re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)") - -AIRFLOW_UV_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)") -AIRFLOW_UV_QUOTED_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")") -UV_QUOTED_PATTERN = re.compile(r"(UV_VERSION = )(\"[0-9.]+\")") -UV_QUOTED_PATTERN_NO_SPACES = re.compile(r"(UV_VERSION=)(\"[0-9.]+\")") -AIRFLOW_UV_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)") -UV_GREATER_PATTERN = re.compile(r'"(uv>=)([0-9]+)"') - -UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" -UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" - - -def replace_group_2_while_keeping_total_length(pattern: re.Pattern[str], replacement: str, text: str) -> str: - def replacer(match): - original_length = len(match.group(2)) - padding = "" - if len(match.groups()) > 2: - padding = match.group(3) - new_length = len(replacement) - diff = new_length - original_length - if diff <= 0: - padding = " " * -diff + padding - else: - padding = padding[diff:] - padded_replacement = match.group(1) + replacement + padding - return padded_replacement.strip() - - return re.sub(pattern, replacer, text) - - -if __name__ == "__main__": - pip_version = get_latest_pypi_version("pip") - console.print(f"[bright_blue]Latest pip version: {pip_version}") - uv_version = get_latest_pypi_version("uv") - console.print(f"[bright_blue]Latest uv version: {uv_version}") - - changed = False - for file in FILES_TO_UPDATE: - console.print(f"[bright_blue]Updating {file}") - file_content = file.read_text() - new_content = file_content - if UPGRADE_PIP: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_PATTERN, pip_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_UPGRADE_PATTERN, pip_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - PIP_QUOTED_PATTERN_NO_SPACES, f'"{pip_version}"', new_content - ) - if UPGRADE_UV: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_PATTERN, uv_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_GREATER_PATTERN, uv_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_QUOTED_PATTERN, f'"{uv_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_QUOTED_PATTERN, f'"{uv_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_QUOTED_PATTERN_NO_SPACES, f'"{uv_version}"', new_content - ) - if new_content != file_content: - file.write_text(new_content) - console.print(f"[bright_blue]Updated {file}") - changed = True - for file in DOC_FILES_TO_UPDATE: - console.print(f"[bright_blue]Updating {file}") - file_content = file.read_text() - new_content = file_content - if UPGRADE_PIP: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_DOC_PATTERN, f"`{pip_version}`", new_content - ) - if UPGRADE_UV: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_DOC_PATTERN, f"`{uv_version}`", new_content - ) - if new_content != file_content: - file.write_text(new_content) - console.print(f"[bright_blue]Updated {file}") - changed = True - if changed: - sys.exit(1) diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py b/scripts/ci/pre_commit/update_installers_and_pre_commit.py new file mode 100755 index 0000000000000..9e5b2c68cf57e --- /dev/null +++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import re +import sys +from enum import Enum +from pathlib import Path + +import requests + +sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported +from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console + +# List of files to update and whether to keep total length of the original value when replacing. +FILES_TO_UPDATE: list[tuple[Path, bool]] = [ + (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile", False), + (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", False), + (AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", False), + (AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "global_constants.py", False), + ( + AIRFLOW_SOURCES_ROOT_PATH + / "dev" + / "breeze" + / "src" + / "airflow_breeze" + / "commands" + / "release_management_commands.py", + False, + ), + (AIRFLOW_SOURCES_ROOT_PATH / ".github" / "actions" / "install-pre-commit" / "action.yml", False), + (AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" / "02_images.md", True), +] + + +def get_latest_pypi_version(package_name: str) -> str: + response = requests.get(f"https://pypi.org/pypi/{package_name}/json") + response.raise_for_status() # Ensure we got a successful response + data = response.json() + latest_version = data["info"]["version"] # The version info is under the 'info' key + return latest_version + + +class Quoting(Enum): + UNQUOTED = 0 + SINGLE_QUOTED = 1 + DOUBLE_QUOTED = 2 + REVERSE_SINGLE_QUOTED = 3 + + +PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), +] + +UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(uv>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(UV_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to " + r"allow automatic replacement of uv version)" + ), + Quoting.UNQUOTED, + ), +] + +PRE_COMMIT_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(pre-commit>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(PRE_COMMIT_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PRE_COMMIT_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_VERSION` *\| *)(`[0-9.]+`)( *\|)"), + Quoting.REVERSE_SINGLE_QUOTED, + ), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to allow automatic " + r"replacement of pre-commit version)" + ), + Quoting.UNQUOTED, + ), +] + +PRE_COMMIT_UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(pre-commit-uv>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PRE_COMMIT_UV_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"), + Quoting.REVERSE_SINGLE_QUOTED, + ), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to allow automatic " + r"replacement of pre-commit-uv version)" + ), + Quoting.UNQUOTED, + ), +] + + +def get_replacement(value: str, quoting: Quoting) -> str: + if quoting == Quoting.DOUBLE_QUOTED: + return f'"{value}"' + elif quoting == Quoting.SINGLE_QUOTED: + return f"'{value}'" + elif quoting == Quoting.REVERSE_SINGLE_QUOTED: + return f"`{value}`" + return value + + +UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" +UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" +UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT", "true").lower() == "true" + + +def replace_version(pattern: re.Pattern[str], version: str, text: str, keep_total_length: bool = True) -> str: + # Assume that the pattern has up to 3 replacement groups: + # 1. Prefix + # 2. Original version + # 3. Suffix + # + # (prefix)(version)(suffix) + # In case "keep_total_length" is set to True, the replacement will be padded with spaces to match + # the original length + def replacer(match): + prefix = match.group(1) + postfix = match.group(3) if len(match.groups()) > 2 else "" + if not keep_total_length: + return prefix + version + postfix + original_length = len(match.group(2)) + new_length = len(version) + diff = new_length - original_length + if diff <= 0: + postfix = " " * -diff + postfix + else: + postfix = postfix[diff:] + padded_replacement = prefix + version + postfix + return padded_replacement.strip() + + return re.sub(pattern, replacer, text) + + +if __name__ == "__main__": + changed = False + for file, keep_length in FILES_TO_UPDATE: + console.print(f"[bright_blue]Updating {file}") + file_content = file.read_text() + new_content = file_content + if UPGRADE_PIP: + pip_version = get_latest_pypi_version("pip") + console.print(f"[bright_blue]Latest pip version: {pip_version}") + for line_pattern, quoting in PIP_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(pip_version, quoting), new_content, keep_length + ) + if UPGRADE_UV: + uv_version = get_latest_pypi_version("uv") + console.print(f"[bright_blue]Latest uv version: {uv_version}") + for line_pattern, quoting in UV_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(uv_version, quoting), new_content, keep_length + ) + if UPGRADE_PRE_COMMIT: + pre_commit_version = get_latest_pypi_version("pre-commit") + console.print(f"[bright_blue]Latest pre-commit version: {pre_commit_version}") + for line_pattern, quoting in PRE_COMMIT_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(pre_commit_version, quoting), new_content, keep_length + ) + if UPGRADE_UV: + pre_commit_uv_version = get_latest_pypi_version("pre-commit-uv") + console.print(f"[bright_blue]Latest pre-commit-uv version: {pre_commit_uv_version}") + for line_pattern, quoting in PRE_COMMIT_UV_PATTERNS: + new_content = replace_version( + line_pattern, + get_replacement(pre_commit_uv_version, quoting), + new_content, + keep_length, + ) + if new_content != file_content: + file.write_text(new_content) + console.print(f"[bright_blue]Updated {file}") + changed = True + if changed: + sys.exit(1) From 8a3d0f44152ab4453975840d20d42ec1d6c04f91 Mon Sep 17 00:00:00 2001 From: Shubham Raj <48172486+shubhamraj-git@users.noreply.github.com> Date: Sun, 5 Jan 2025 04:39:32 +0530 Subject: [PATCH 3/3] Allow empty value while variable creation (#45402) * fix * fix backend * added tests --- .../core_api/datamodels/variables.py | 4 +-- .../core_api/openapi/v1-generated.yaml | 8 ++--- .../ui/openapi-gen/requests/schemas.gen.ts | 18 ++--------- airflow/ui/openapi-gen/requests/types.gen.ts | 4 +-- .../ManageVariable/EditVariableButton.tsx | 2 +- .../Variables/ManageVariable/VariableForm.tsx | 10 ++---- .../core_api/routes/public/test_variables.py | 32 +++++++++++++++++++ 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/airflow/api_fastapi/core_api/datamodels/variables.py b/airflow/api_fastapi/core_api/datamodels/variables.py index 2e6f25993a55d..8307809bc5f5b 100644 --- a/airflow/api_fastapi/core_api/datamodels/variables.py +++ b/airflow/api_fastapi/core_api/datamodels/variables.py @@ -33,7 +33,7 @@ class VariableResponse(BaseModel): model_config = ConfigDict(populate_by_name=True, from_attributes=True) key: str - val: str | None = Field(alias="value") + val: str = Field(alias="value") description: str | None is_encrypted: bool @@ -56,7 +56,7 @@ class VariableBody(BaseModel): """Variable serializer for bodies.""" key: str = Field(max_length=ID_LEN) - value: str | None = Field(serialization_alias="val") + value: str = Field(serialization_alias="val") description: str | None = Field(default=None) diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index ceda67d98c131..ef127a5743984 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -9736,9 +9736,7 @@ components: maxLength: 250 title: Key value: - anyOf: - - type: string - - type: 'null' + type: string title: Value description: anyOf: @@ -9773,9 +9771,7 @@ components: type: string title: Key value: - anyOf: - - type: string - - type: 'null' + type: string title: Value description: anyOf: diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index b714cf0799306..6a8959465d7bc 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -5485,14 +5485,7 @@ export const $VariableBody = { title: "Key", }, value: { - anyOf: [ - { - type: "string", - }, - { - type: "null", - }, - ], + type: "string", title: "Value", }, description: { @@ -5540,14 +5533,7 @@ export const $VariableResponse = { title: "Key", }, value: { - anyOf: [ - { - type: "string", - }, - { - type: "null", - }, - ], + type: "string", title: "Value", }, description: { diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index 70c3f532429bf..4a78a65cef3a3 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1276,7 +1276,7 @@ export type ValidationError = { */ export type VariableBody = { key: string; - value: string | null; + value: string; description?: string | null; }; @@ -1293,7 +1293,7 @@ export type VariableCollectionResponse = { */ export type VariableResponse = { key: string; - value: string | null; + value: string; description: string | null; is_encrypted: boolean; }; diff --git a/airflow/ui/src/pages/Variables/ManageVariable/EditVariableButton.tsx b/airflow/ui/src/pages/Variables/ManageVariable/EditVariableButton.tsx index b0bce9f9f82da..3fee33d461e2f 100644 --- a/airflow/ui/src/pages/Variables/ManageVariable/EditVariableButton.tsx +++ b/airflow/ui/src/pages/Variables/ManageVariable/EditVariableButton.tsx @@ -36,7 +36,7 @@ const EditVariableButton = ({ variable }: Props) => { const initialVariableValue: VariableBody = { description: variable.description ?? "", key: variable.key, - value: variable.value ?? "", + value: variable.value, }; const { editVariable, error, isPending, setError } = useEditVariable(initialVariableValue, { onSuccessConfirm: onClose, diff --git a/airflow/ui/src/pages/Variables/ManageVariable/VariableForm.tsx b/airflow/ui/src/pages/Variables/ManageVariable/VariableForm.tsx index 639bed075bde8..8ae39c58210b5 100644 --- a/airflow/ui/src/pages/Variables/ManageVariable/VariableForm.tsx +++ b/airflow/ui/src/pages/Variables/ManageVariable/VariableForm.tsx @@ -80,18 +80,14 @@ const VariableForm = ({ error, initialVariable, isPending, manageMutate, setErro ( - + render={({ field }) => ( + Value -