Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ The `blueprint:` value is the snake_case form of the class name. `Extract` becom

```python
# dags/loader.py
from blueprint import build_all_dags
from blueprint import build_all_airflow_dags

build_all_dags()
build_all_airflow_dags()
```

### 4. Validate
Expand Down Expand Up @@ -452,13 +452,13 @@ The `on_dag_built` callback lets you modify each DAG after it's built from YAML.
# dags/loader.py
from pathlib import Path
from airflow import DAG
from blueprint import build_all_dags
from blueprint import build_all_airflow_dags

def post_process(dag: DAG, yaml_path: Path) -> None:
dag.tags = [*(dag.tags or []), "managed-by-blueprint"]
dag.access_control = {"data-team": {"can_read", "can_edit"}}

build_all_dags(on_dag_built=post_process)
build_all_airflow_dags(on_dag_built=post_process)
```

This is useful for applying cross-cutting concerns like access controls, tags, or custom metadata that shouldn't live in individual YAML files. The callback runs once per DAG, after all steps are wired up.
Expand Down
11 changes: 10 additions & 1 deletion blueprint/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,15 @@

__version__ = "0.3.0"

from .builder import Builder, DAGConfig, OnDagBuilt, StepConfig, build_all, build_all_dags
from .builder import (
Builder,
DAGConfig,
OnDagBuilt,
StepConfig,
build_all,
build_all_airflow_dags,
build_all_dags,
)
from .core import Blueprint, BlueprintDagArgs, DefaultDagArgs, TaskOrGroup
from .errors import (
BlueprintError,
Expand Down Expand Up @@ -73,6 +81,7 @@
"ValidationError",
"YAMLParseError",
"build_all",
"build_all_airflow_dags",
"build_all_dags",
"discover_blueprints",
"field_validator",
Expand Down
70 changes: 57 additions & 13 deletions blueprint/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ def _check_duplicate_dag_id(dag_id: str, yaml_path: Path, dag_id_to_file: dict[s
raise DuplicateDAGIdError(dag_id, [dag_id_to_file[dag_id], yaml_path])


def build_all_dags(
def build_all_airflow_dags(
search_path: str | Path | None = None,
register_globals: dict | None = None,
pattern: str = "*.dag.yaml",
Expand All @@ -509,6 +509,11 @@ def build_all_dags(
This is the top-level convenience function meant to be called from a
DAG loader file (e.g., loader.py in your dags/ directory).

The name intentionally contains both ``airflow`` and ``dag`` so that a
one-line loader of ``from blueprint import build_all_airflow_dags;
build_all_airflow_dags()`` satisfies Airflow's safe-mode DAG file scanner,
which only considers a file if its contents contain both substrings.

Args:
search_path: Directory to search for YAML files. Defaults to dags/
or the directory containing the calling file.
Expand All @@ -529,9 +534,9 @@ def build_all_dags(
Example:
```python
# In dags/loader.py
from blueprint import build_all_dags
from blueprint import build_all_airflow_dags

build_all_dags()
build_all_airflow_dags()
```
"""
from blueprint.loaders import render_yaml_template
Expand Down Expand Up @@ -595,7 +600,7 @@ def build_all_dags(
return dags


def build_all(
def build_all_dags(
search_path: str | Path | None = None,
register_globals: dict | None = None,
pattern: str = "*.dag.yaml",
Expand All @@ -604,22 +609,61 @@ def build_all(
bp_registry: BlueprintRegistry | None = None,
on_dag_built: OnDagBuilt | None = None,
) -> list["DAG"]:
"""Deprecated alias for ``build_all_dags``.
"""Deprecated alias for ``build_all_airflow_dags``.

Renamed so a one-line loader ``from blueprint import build_all_dags;
build_all_dags()`` carries the substring ``dag`` and satisfies Airflow's
safe-mode DAG file scanner.
A loader of ``from blueprint import build_all_dags; build_all_dags()``
carries the substring ``dag`` but not ``airflow``, so Airflow's safe-mode
scanner skips the file. Use ``build_all_airflow_dags`` instead, whose name
carries both required substrings.
"""
warnings.warn(
"blueprint.build_all_dags is deprecated and will be removed in a "
"future release; use blueprint.build_all_airflow_dags instead. Its "
"name carries both 'airflow' and 'dag' so a one-line loader satisfies "
"Airflow's safe-mode DAG file scanner.",
DeprecationWarning,
stacklevel=2,
)
if register_globals is None:
frame = inspect.currentframe()
try:
register_globals = frame.f_back.f_globals if frame and frame.f_back else {}
finally:
del frame
return build_all_airflow_dags(
search_path=search_path,
register_globals=register_globals,
pattern=pattern,
render_templates=render_templates,
template_context=template_context,
bp_registry=bp_registry,
on_dag_built=on_dag_built,
)


def build_all(
search_path: str | Path | None = None,
register_globals: dict | None = None,
pattern: str = "*.dag.yaml",
render_templates: bool = True,
template_context: dict[str, Any] | None = None,
bp_registry: BlueprintRegistry | None = None,
on_dag_built: OnDagBuilt | None = None,
) -> list["DAG"]:
"""Deprecated alias for ``build_all_airflow_dags``."""
warnings.warn(
"blueprint.build_all is deprecated and will be removed in a future "
"release; use blueprint.build_all_dags instead.",
"release; use blueprint.build_all_airflow_dags instead.",
DeprecationWarning,
stacklevel=2,
)
if register_globals is None:
frame = inspect.currentframe()
register_globals = frame.f_back.f_globals if frame and frame.f_back else {}
return build_all_dags(
try:
register_globals = frame.f_back.f_globals if frame and frame.f_back else {}
finally:
del frame
return build_all_airflow_dags(
search_path=search_path,
register_globals=register_globals,
pattern=pattern,
Expand All @@ -631,7 +675,7 @@ def build_all(


def _get_caller_file() -> str | None:
"""Return the __file__ of the module that called build_all_dags().
"""Return the __file__ of the module that called build_all_airflow_dags().

Walks the call stack to find the first frame outside of the blueprint
package, making this resilient to internal helper wrappers.
Expand All @@ -657,7 +701,7 @@ def _resolve_search_path(search_path: str | Path | None) -> Path:

Resolution order:
1. Explicit search_path argument
2. Directory of the file that called build_all_dags()
2. Directory of the file that called build_all_airflow_dags()
3. Current working directory
"""
if search_path is not None:
Expand Down
4 changes: 2 additions & 2 deletions blueprint/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -502,8 +502,8 @@ def new(template_dir: str | None, output_dir: str):

console.print(f"\n[green]Created {file_path}[/green]")
console.print("\nTo load this DAG, add a loader.py to your dags/ directory:")
console.print(" from blueprint import build_all_dags")
console.print(" build_all_dags()")
console.print(" from blueprint import build_all_airflow_dags")
console.print(" build_all_airflow_dags()")


def main():
Expand Down
2 changes: 1 addition & 1 deletion examples/advanced/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ Custom `BlueprintDagArgs` subclass that converts a `priority` field into a DAG t

### Loader (`dags/loader.py`)

`build_all()` with `on_dag_built` callback and `template_context`.
`build_all_airflow_dags()` with `on_dag_built` callback and `template_context`.

### Programmatic Building (`dags/programmatic_dags.py`)

Expand Down
4 changes: 2 additions & 2 deletions examples/advanced/dags/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

from airflow.models import DAG

from blueprint import build_all_dags
from blueprint import build_all_airflow_dags


def add_mission_tags(dag: DAG, config_path: Path) -> None:
"""Add the source YAML filename as a DAG tag."""
dag.tags = [*(dag.tags or []), f"source:{config_path.stem}"]


build_all_dags(
build_all_airflow_dags(
on_dag_built=add_mission_tags,
template_context={"agency": "Deep Space Network"},
)
2 changes: 1 addition & 1 deletion examples/simple/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Simple Example

One DAG with two concurrent extract steps followed by a load step. Demonstrates the basics: defining blueprints, composing them via YAML, and loading with `build_all()`.
One DAG with two concurrent extract steps followed by a load step. Demonstrates the basics: defining blueprints, composing them via YAML, and loading with `build_all_airflow_dags()`.

## Quick Start

Expand Down
4 changes: 2 additions & 2 deletions examples/simple/dags/loader.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
from blueprint import build_all_dags
from blueprint import build_all_airflow_dags

build_all_dags()
build_all_airflow_dags()
4 changes: 2 additions & 2 deletions tests/integration/project/dags/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@

from airflow import DAG

from blueprint import build_all_dags
from blueprint import build_all_airflow_dags


def post_process(dag: DAG, yaml_path: Path) -> None:
"""Post-process every built DAG by appending a tag."""
dag.tags = [*(dag.tags or []), "callback-verified"]


build_all_dags(on_dag_built=post_process)
build_all_airflow_dags(on_dag_built=post_process)
19 changes: 19 additions & 0 deletions tests/integration/project/dags/safe_mode_minimal/blueprints.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
"""Blueprint for the bare-minimum safe-mode discovery probe."""

from airflow.operators.bash import BashOperator

from blueprint import BaseModel, Blueprint


class SafeModeProbeConfig(BaseModel):
message: str = "safe-mode probe"


class SafeModeProbe(Blueprint[SafeModeProbeConfig]):
"""Single-task blueprint used to prove safe-mode discovery end-to-end."""

def render(self, config: SafeModeProbeConfig) -> BashOperator:
return BashOperator(
task_id=self.step_id,
bash_command=f"echo '{config.message}'",
)
8 changes: 8 additions & 0 deletions tests/integration/project/dags/safe_mode_minimal/loader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from blueprint import build_all_airflow_dags


def _tag(dag, yaml_path):
dag.tags = [*(dag.tags or []), "safe-mode-probe"]


build_all_airflow_dags(pattern="*.safe.yaml", on_dag_built=_tag)
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
dag_id: safe_mode_minimal_probe
steps:
probe:
blueprint: safe_mode_probe
message: "discovered via build_all_airflow_dags"
62 changes: 62 additions & 0 deletions tests/integration/test_safe_mode_discovery.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Tier 1: Airflow safe-mode DAG discovery.

Airflow's DAG file processor runs in *safe mode* by default: it only parses a
file as a potential DAG file when its contents contain both the ``airflow``
and ``dag`` substrings. A Blueprint loader is otherwise a plain function call,
so the entry-point name has to carry both substrings on its own -- otherwise a
minimal one-line loader is silently skipped and no DAGs appear.

This is the end-to-end regression guard for the
``build_all_dags`` -> ``build_all_airflow_dags`` rename. The project ships a
bare-minimum loader at ``dags/safe_mode_minimal/loader.py`` -- an import and a
call, with deliberately no ``from airflow import DAG`` -- and these tests run
against the live Airflow instance to assert that Airflow actually discovered
and parsed the DAG that loader builds.
"""

from __future__ import annotations

import time
from typing import TYPE_CHECKING

import pytest

from .conftest import DAG_PARSE_TIMEOUT, HEALTH_CHECK_INTERVAL

if TYPE_CHECKING:
from .conftest import AirflowAPI

pytestmark = pytest.mark.integration

PROBE_DAG_ID = "safe_mode_minimal_probe"


class TestSafeModeDiscovery:
"""Verify the bare-minimum loader is discovered by the live Airflow scanner."""

def test_bare_minimum_loader_dag_is_parsed(self, api_client: AirflowAPI):
"""Airflow parses the no-`import DAG` loader purely via the entry-point name."""
deadline = time.monotonic() + DAG_PARSE_TIMEOUT
dag_ids: set[str] = set()
while time.monotonic() < deadline:
dag_ids = api_client.get_dag_ids()
if PROBE_DAG_ID in dag_ids:
break
time.sleep(HEALTH_CHECK_INTERVAL)

assert PROBE_DAG_ID in dag_ids, (
f"Airflow did not discover '{PROBE_DAG_ID}'. Its loader contains no "
"'from airflow import DAG' — discovery relies solely on "
"build_all_airflow_dags carrying the 'airflow' substring for safe mode."
)

def test_no_import_errors_for_minimal_loader(self, api_client: AirflowAPI):
"""The minimal loader parses cleanly, with no import error recorded."""
resp = api_client.get("/importErrors")
assert resp.status_code == 200, resp.text
offending = [
e
for e in resp.json().get("import_errors", [])
if "safe_mode_minimal" in (e.get("filename") or "")
]
assert not offending, f"Import errors for the minimal loader: {offending}"
Loading