diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/pipelineprobe/__init__.py b/pipelineprobe/__init__.py index bbde08c..cbdda49 100644 --- a/pipelineprobe/__init__.py +++ b/pipelineprobe/__init__.py @@ -1,4 +1,5 @@ """ PipelineProbe: Instant Data Pipeline Audit Report """ + __version__ = "0.1.0" diff --git a/pipelineprobe/cli.py b/pipelineprobe/cli.py index 3b20bc4..6a8eb35 100644 --- a/pipelineprobe/cli.py +++ b/pipelineprobe/cli.py @@ -18,32 +18,40 @@ add_completion=False, ) + @app.command() def audit( config: str = typer.Option("pipelineprobe.yml", help="Path to config file"), - fail_on_critical: int = typer.Option(None, help="Override fail-on-critical threshold"), - report_format: str = typer.Option(None, "--format", help="Override report format: html | json | both") + fail_on_critical: int = typer.Option( + None, help="Override fail-on-critical threshold" + ), + report_format: str = typer.Option( + None, "--format", help="Override report format: html | json | both" + ), ): """ Run the PipelineProbe audit and generate a report. """ typer.echo(f"Loading configuration from {config}...") cfg = load_config(config) - + # Apply CLI overrides if fail_on_critical is not None: cfg.report.fail_on_critical = fail_on_critical if report_format is not None: valid_formats = {"html", "json", "both"} if report_format not in valid_formats: - typer.secho(f"Invalid --format '{report_format}'. Must be one of: {', '.join(sorted(valid_formats))}", fg=typer.colors.RED) + typer.secho( + f"Invalid --format '{report_format}'. Must be one of: {', '.join(sorted(valid_formats))}", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) cfg.report.format = report_format typer.echo("Initializing connectors...") airflow_conn = AirflowConnector(cfg.orchestrator) dbt_conn = DbtConnector(cfg.dbt) - + if cfg.warehouse.type == "bigquery": typer.echo("Using BigQuery connector...") warehouse_conn = BigQueryConnector(cfg.warehouse) @@ -58,14 +66,16 @@ def audit( # Fetch top-level DAGs airflow_dags = airflow_conn.get_dags() airflow_tasks = [] - + # Wire the actual dag runs and tasks for every returned DAG if airflow_dags: - typer.echo(f"Found {len(airflow_dags)} Airflow DAGs. Fetching runs and tasks...") + typer.echo( + f"Found {len(airflow_dags)} Airflow DAGs. Fetching runs and tasks..." + ) for dag in airflow_dags: dag.recent_runs = airflow_conn.get_dag_runs(dag.id) airflow_tasks.extend(airflow_conn.get_tasks(dag.id)) - + dbt_models = dbt_conn.get_models() warehouse_tables = warehouse_conn.get_stats_sync() @@ -90,28 +100,30 @@ def audit( "score": score, "critical_count": critical_count, "warning_count": warning_count, - "total_issues": len(issues) + "total_issues": len(issues), } typer.echo("Rendering reports...") renderer = ReportRenderer(cfg.report.output_dir) - + if cfg.report.format in ("html", "both"): html_path = renderer.render_html(issues, summary) typer.echo(f"HTML report generated at: {html_path}") - + if cfg.report.format in ("json", "both"): json_path = renderer.render_json(issues, summary) typer.echo(f"JSON report generated at: {json_path}") - + if critical_count > cfg.report.fail_on_critical: - typer.secho(f"Audit failed! Found {critical_count} critical issues (threshold: {cfg.report.fail_on_critical}).", fg=typer.colors.RED) + typer.secho( + f"Audit failed! Found {critical_count} critical issues (threshold: {cfg.report.fail_on_critical}).", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) typer.secho("Audit completed successfully!", fg=typer.colors.GREEN) - @app.command() def init(): """ @@ -145,7 +157,7 @@ def init(): """ with open("pipelineprobe.yml", "w") as f: f.write(default_config) - + typer.secho("Initialized pipelineprobe.yml successfully.", fg=typer.colors.GREEN) diff --git a/pipelineprobe/config.py b/pipelineprobe/config.py index 2eb8f91..ed7239e 100644 --- a/pipelineprobe/config.py +++ b/pipelineprobe/config.py @@ -3,6 +3,7 @@ from pydantic import BaseModel from pydantic_settings import BaseSettings + class AirflowConfig(BaseModel): type: str = "airflow" base_url: str = "http://localhost:8080" @@ -11,12 +12,14 @@ class AirflowConfig(BaseModel): verify_ssl: bool = False lookback_days: int = 14 + class DbtConfig(BaseModel): project_dir: str = "./analytics" target: str = "prod" manifest_path: str = "./analytics/target/manifest.json" run_results_path: str = "./analytics/target/run_results.json" + class WarehouseConfig(BaseModel): type: str = "postgres" dsn: str = "postgresql://user:pass@localhost:5432/analytics" @@ -25,18 +28,21 @@ class WarehouseConfig(BaseModel): username: str | None = None password: str | None = None + class ReportConfig(BaseModel): output_dir: str = "./reports" format: str = "html" include_cost_section: bool = False fail_on_critical: int = 5 + class PipelineProbeConfig(BaseSettings): orchestrator: AirflowConfig = AirflowConfig() dbt: DbtConfig = DbtConfig() warehouse: WarehouseConfig = WarehouseConfig() report: ReportConfig = ReportConfig() + def load_config(config_path: str) -> PipelineProbeConfig: """Load config from YAML if it exists, otherwise return defaults.""" path = Path(config_path) diff --git a/pipelineprobe/connectors/airflow.py b/pipelineprobe/connectors/airflow.py index f1ed302..55cebc5 100644 --- a/pipelineprobe/connectors/airflow.py +++ b/pipelineprobe/connectors/airflow.py @@ -16,15 +16,19 @@ class AirflowConnector: def __init__(self, config: AirflowConfig): self.config = config - + if not self.config.username or not self.config.password: logger.warning( "Airflow credentials missing. Use PIPELINEPROBE_ORCHESTRATOR_USERNAME/PASSWORD " "or update pipelineprobe.yml." ) - - auth = (self.config.username, self.config.password) if self.config.username and self.config.password else None - + + auth = ( + (self.config.username, self.config.password) + if self.config.username and self.config.password + else None + ) + self.client = httpx.Client( base_url=self.config.base_url, auth=auth, @@ -76,9 +80,15 @@ def get_dag_runs(self, dag_id: str) -> List[DagRun]: start_str = r.get("start_date") end_str = r.get("end_date") # Fall back to execution_date if start_date missing (older Airflow versions) - start_time = parse_date(start_str) if start_str else parse_date(r["execution_date"]) + start_time = ( + parse_date(start_str) + if start_str + else parse_date(r["execution_date"]) + ) end_time = parse_date(end_str) if end_str else None - runs.append(DagRun(state=state, start_time=start_time, end_time=end_time)) + runs.append( + DagRun(state=state, start_time=start_time, end_time=end_time) + ) return runs except Exception as e: logger.error("Error fetching DAG runs for %s: %s", dag_id, e) diff --git a/pipelineprobe/connectors/dbt.py b/pipelineprobe/connectors/dbt.py index 1ef885a..a344bd7 100644 --- a/pipelineprobe/connectors/dbt.py +++ b/pipelineprobe/connectors/dbt.py @@ -8,6 +8,7 @@ logger = logging.getLogger(__name__) + class DbtConnector: def __init__(self, config: DbtConfig): self.config = config @@ -17,7 +18,9 @@ def get_models(self) -> List[DbtModel]: run_results_path = Path(self.config.run_results_path) if not manifest_path.exists(): - logger.warning("dbt manifest not found at %s — skipping dbt checks.", manifest_path) + logger.warning( + "dbt manifest not found at %s — skipping dbt checks.", manifest_path + ) return [] try: diff --git a/pipelineprobe/connectors/postgres.py b/pipelineprobe/connectors/postgres.py index c303ab5..f3a9699 100644 --- a/pipelineprobe/connectors/postgres.py +++ b/pipelineprobe/connectors/postgres.py @@ -8,6 +8,7 @@ logger = logging.getLogger(__name__) + class PostgresConnector: def __init__(self, config: WarehouseConfig): self.config = config @@ -39,4 +40,3 @@ def get_stats_sync(self) -> List[Dict[str, Any]]: finally: if conn: conn.close() - diff --git a/pipelineprobe/models.py b/pipelineprobe/models.py index 55da78d..4e5db6b 100644 --- a/pipelineprobe/models.py +++ b/pipelineprobe/models.py @@ -3,17 +3,20 @@ from pydantic import BaseModel + class DagRun(BaseModel): state: str start_time: datetime end_time: datetime | None = None + class Dag(BaseModel): id: str is_active: bool recent_runs: list[DagRun] owner: str | None = None + class Task(BaseModel): dag_id: str task_id: str @@ -21,12 +24,14 @@ class Task(BaseModel): sla: timedelta | None = None has_alerts: bool + class DbtModel(BaseModel): name: str tests_count: int last_run_status: str tags: list[str] + class Issue(BaseModel): severity: Literal["critical", "warning", "info"] category: Literal["dag", "task", "dbt", "warehouse"] diff --git a/pipelineprobe/renderer.py b/pipelineprobe/renderer.py index b03383e..c28f961 100644 --- a/pipelineprobe/renderer.py +++ b/pipelineprobe/renderer.py @@ -13,6 +13,7 @@ def _json_default(obj): return obj.total_seconds() raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") + class ReportRenderer: def __init__(self, output_dir: str): self.output_dir = Path(output_dir) @@ -30,10 +31,7 @@ def render_html(self, issues: list[Issue], summary: dict) -> Path: def render_json(self, issues: list[Issue], summary: dict) -> Path: output_path = self.output_dir / "report.json" - data = { - "summary": summary, - "issues": [i.model_dump() for i in issues] - } + data = {"summary": summary, "issues": [i.model_dump() for i in issues]} with open(output_path, "w") as f: json.dump(data, f, indent=2, default=_json_default) return output_path diff --git a/pipelineprobe/rules/__init__.py b/pipelineprobe/rules/__init__.py index 484d6cb..64d75cd 100644 --- a/pipelineprobe/rules/__init__.py +++ b/pipelineprobe/rules/__init__.py @@ -3,6 +3,7 @@ from .dbt_rules import register_dbt_rules from .postgres_rules import register_postgres_rules + def get_configured_engine() -> RuleEngine: engine = RuleEngine() register_airflow_rules(engine) diff --git a/pipelineprobe/rules/airflow_rules.py b/pipelineprobe/rules/airflow_rules.py index 4c40dbc..ccb4fa3 100644 --- a/pipelineprobe/rules/airflow_rules.py +++ b/pipelineprobe/rules/airflow_rules.py @@ -2,6 +2,7 @@ from datetime import datetime, timezone from pipelineprobe.models import Task, Issue, Dag + def check_missing_retries(context: dict) -> List[Issue]: issues = [] tasks: List[Task] = context.get("airflow_tasks", []) @@ -14,11 +15,12 @@ def check_missing_retries(context: dict) -> List[Issue]: summary=f"Task {task.task_id} in DAG {task.dag_id} has no retries configured.", details="Tasks without retries are more prone to transient failures.", recommendation="Configure at least 1-3 retries for the task.", - affected_resources=[task.task_id] + affected_resources=[task.task_id], ) ) return issues + def check_missing_slas(context: dict) -> List[Issue]: issues = [] tasks: List[Task] = context.get("airflow_tasks", []) @@ -31,18 +33,19 @@ def check_missing_slas(context: dict) -> List[Issue]: summary=f"Task {task.task_id} in DAG {task.dag_id} has no SLA configured.", details="Tasks without SLAs might silently miss deadlines.", recommendation="Configure an SLA if this task is time-sensitive.", - affected_resources=[task.task_id] + affected_resources=[task.task_id], ) ) return issues + def check_high_failure_rate(context: dict) -> List[Issue]: issues = [] dags: List[Dag] = context.get("airflow_dags", []) for dag in dags: if not dag.is_active or not dag.recent_runs: continue - + total_runs = len(dag.recent_runs) if total_runs >= 5: failed_runs = sum(1 for run in dag.recent_runs if run.state == "failed") @@ -55,7 +58,7 @@ def check_high_failure_rate(context: dict) -> List[Issue]: summary=f"DAG {dag.id} has a high failure rate ({failure_ratio:.0%}).", details=f"{failed_runs} out of the last {total_runs} runs failed.", recommendation="Investigate the root cause of the frequent failures.", - affected_resources=[dag.id] + affected_resources=[dag.id], ) ) return issues @@ -67,11 +70,11 @@ def check_stale_dags(context: dict) -> List[Issue]: # Default stale threshold is 7 days, could be configurable later stale_threshold_days = 7 now = datetime.now(timezone.utc) - + for dag in dags: if not dag.is_active: continue - + # A DAG with no recent runs at all is inherently stale if not dag.recent_runs: issues.append( @@ -81,14 +84,16 @@ def check_stale_dags(context: dict) -> List[Issue]: summary=f"DAG {dag.id} has no recent runs recorded.", details="The DAG is active but has never run (or runs were not retrieved).", recommendation="Verify the scheduler is processing this DAG correctly.", - affected_resources=[dag.id] + affected_resources=[dag.id], ) ) continue # Check if there is any successful run in recent_runs - recent_successes = [run for run in dag.recent_runs if run.state == "success" and run.end_time] - + recent_successes = [ + run for run in dag.recent_runs if run.state == "success" and run.end_time + ] + if not recent_successes: # Has runs but zero successes — flag as critical issues.append( @@ -103,11 +108,13 @@ def check_stale_dags(context: dict) -> List[Issue]: ) else: # Sort by end_time descending to get latest - latest_success = sorted(recent_successes, key=lambda r: r.end_time, reverse=True)[0] + latest_success = sorted( + recent_successes, key=lambda r: r.end_time, reverse=True + )[0] end_time_aware = latest_success.end_time if end_time_aware.tzinfo is None: end_time_aware = end_time_aware.replace(tzinfo=timezone.utc) - + days_since = (now - end_time_aware).total_seconds() / 86400 if days_since > stale_threshold_days: issues.append( @@ -117,11 +124,12 @@ def check_stale_dags(context: dict) -> List[Issue]: summary=f"DAG {dag.id} is stale.", details=f"No successful execution in the last {days_since:.1f} days.", recommendation="Check if the DAG is still needed, or if it is silently failing to schedule.", - affected_resources=[dag.id] + affected_resources=[dag.id], ) ) return issues + def register_airflow_rules(engine): engine.register_rule(check_missing_retries) engine.register_rule(check_missing_slas) diff --git a/pipelineprobe/rules/dbt_rules.py b/pipelineprobe/rules/dbt_rules.py index 440169b..4e94d43 100644 --- a/pipelineprobe/rules/dbt_rules.py +++ b/pipelineprobe/rules/dbt_rules.py @@ -1,6 +1,7 @@ from typing import List from pipelineprobe.models import DbtModel, Issue + def check_missing_tests(context: dict) -> List[Issue]: issues = [] models: List[DbtModel] = context.get("dbt_models", []) @@ -13,11 +14,12 @@ def check_missing_tests(context: dict) -> List[Issue]: summary=f"dbt model '{model.name}' has no tests.", details="Models without tests can introduce silent data quality issues.", recommendation="Add at least unique and not_null tests for primary keys.", - affected_resources=[model.name] + affected_resources=[model.name], ) ) return issues + def check_failing_models(context: dict) -> List[Issue]: issues = [] models: List[DbtModel] = context.get("dbt_models", []) @@ -30,11 +32,12 @@ def check_failing_models(context: dict) -> List[Issue]: summary=f"dbt model '{model.name}' failed in the last run.", details=f"Last run status: {model.last_run_status}", recommendation="Investigate and fix the failing model or tests.", - affected_resources=[model.name] + affected_resources=[model.name], ) ) return issues + def register_dbt_rules(engine): engine.register_rule(check_missing_tests) engine.register_rule(check_failing_models) diff --git a/pipelineprobe/rules/engine.py b/pipelineprobe/rules/engine.py index 9e1dbe1..c8d0a79 100644 --- a/pipelineprobe/rules/engine.py +++ b/pipelineprobe/rules/engine.py @@ -5,6 +5,7 @@ logger = logging.getLogger(__name__) + class RuleEngine: def __init__(self): self.rules: List[Callable[[Any], List[Issue]]] = [] diff --git a/pipelineprobe/rules/postgres_rules.py b/pipelineprobe/rules/postgres_rules.py index 8d6c360..302bf19 100644 --- a/pipelineprobe/rules/postgres_rules.py +++ b/pipelineprobe/rules/postgres_rules.py @@ -1,6 +1,7 @@ from typing import List, Dict, Any from pipelineprobe.models import Issue + def check_large_tables(context: dict) -> List[Issue]: issues = [] if context.get("warehouse_type") != "postgres": @@ -15,11 +16,12 @@ def check_large_tables(context: dict) -> List[Issue]: summary=f"Table '{table.get('tablename')}' in schema '{table.get('schemaname')}' is very large.", details=f"Row count: {table.get('row_count')}.", recommendation="Consider partitioning the table to improve query performance.", - affected_resources=[table.get('tablename') or ''] + affected_resources=[table.get("tablename") or ""], ) ) return issues + def check_missing_timestamps(context: dict) -> List[Issue]: issues = [] if context.get("warehouse_type") != "postgres": @@ -35,11 +37,12 @@ def check_missing_timestamps(context: dict) -> List[Issue]: summary=f"Table '{table.get('tablename')}' in schema '{table.get('schemaname')}' has no updated_at/created_at columns.", details=f"Row count is {table.get('row_count')} but no audit timestamps exist.", recommendation="Add updated_at / created_at columns for incremental ingestion and auditing.", - affected_resources=[table.get('tablename') or ''] + affected_resources=[table.get("tablename") or ""], ) ) return issues + def register_postgres_rules(engine): engine.register_rule(check_large_tables) engine.register_rule(check_missing_timestamps) diff --git a/tests/conftest.py b/tests/conftest.py index a8dbe36..d861b1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta from pipelineprobe.models import Dag, DagRun, Task, DbtModel + @pytest.fixture def mock_airflow_dags(): return [ @@ -9,35 +10,72 @@ def mock_airflow_dags(): id="test_dag_1", is_active=True, recent_runs=[ - DagRun(state="failed", start_time=datetime.now() - timedelta(days=1), end_time=datetime.now()), - DagRun(state="failed", start_time=datetime.now() - timedelta(days=2), end_time=datetime.now() - timedelta(days=1)), + DagRun( + state="failed", + start_time=datetime.now() - timedelta(days=1), + end_time=datetime.now(), + ), + DagRun( + state="failed", + start_time=datetime.now() - timedelta(days=2), + end_time=datetime.now() - timedelta(days=1), + ), ], - owner="data_engineering" + owner="data_engineering", ), Dag( id="test_dag_2", is_active=True, recent_runs=[ - DagRun(state="success", start_time=datetime.now() - timedelta(hours=2), end_time=datetime.now()), + DagRun( + state="success", + start_time=datetime.now() - timedelta(hours=2), + end_time=datetime.now(), + ), ], - owner="analytics" - ) + owner="analytics", + ), ] + @pytest.fixture def mock_airflow_tasks(): return [ - Task(dag_id="test_dag_1", task_id="task_no_retries", retries=0, sla=None, has_alerts=False), - Task(dag_id="test_dag_1", task_id="task_with_retries", retries=3, sla=timedelta(hours=1), has_alerts=True), + Task( + dag_id="test_dag_1", + task_id="task_no_retries", + retries=0, + sla=None, + has_alerts=False, + ), + Task( + dag_id="test_dag_1", + task_id="task_with_retries", + retries=3, + sla=timedelta(hours=1), + has_alerts=True, + ), ] + @pytest.fixture def mock_dbt_models(): return [ - DbtModel(name="model_with_no_tests", tests_count=0, last_run_status="success", tags=[]), - DbtModel(name="model_with_tests", tests_count=5, last_run_status="success", tags=["daily"]), + DbtModel( + name="model_with_no_tests", + tests_count=0, + last_run_status="success", + tags=[], + ), + DbtModel( + name="model_with_tests", + tests_count=5, + last_run_status="success", + tags=["daily"], + ), ] + @pytest.fixture def mock_postgres_tables(): return [ diff --git a/tests/test_cli.py b/tests/test_cli.py index 4737882..7bfe6c1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,9 @@ def test_cli_init(): result = runner.invoke(app, ["init"]) assert result.exit_code == 0, result.stdout assert "Initialized pipelineprobe.yml" in result.stdout - assert os.path.exists("pipelineprobe.yml"), "pipelineprobe.yml was not created" + assert os.path.exists("pipelineprobe.yml"), ( + "pipelineprobe.yml was not created" + ) finally: os.chdir(old_cwd) @@ -27,17 +29,15 @@ def test_cli_audit_help(): assert "Run the PipelineProbe audit and generate a report" in result.stdout - - def test_cli_audit_full(): with tempfile.TemporaryDirectory() as tmpdir: old_cwd = os.getcwd() try: os.chdir(tmpdir) - + # Create a valid config file with open("pipelineprobe.yml", "w") as f: - f.write(''' + f.write(""" orchestrator: base_url: "http://test" username: "a" @@ -49,42 +49,51 @@ def test_cli_audit_full(): report: output_dir: "./reports" format: "html" -''') - - with patch("pipelineprobe.cli.AirflowConnector") as mock_airflow_cls, \ - patch("pipelineprobe.cli.DbtConnector") as mock_dbt_cls, \ - patch("pipelineprobe.cli.PostgresConnector") as mock_pg_cls, \ - patch("pipelineprobe.cli.ReportRenderer") as mock_renderer_cls: - +""") + + with ( + patch("pipelineprobe.cli.AirflowConnector") as mock_airflow_cls, + patch("pipelineprobe.cli.DbtConnector") as mock_dbt_cls, + patch("pipelineprobe.cli.PostgresConnector") as mock_pg_cls, + patch("pipelineprobe.cli.ReportRenderer") as mock_renderer_cls, + ): mock_airflow = mock_airflow_cls.return_value mock_airflow.get_dags.return_value = [] - + mock_dbt = mock_dbt_cls.return_value mock_dbt.get_models.return_value = [] - + mock_pg = mock_pg_cls.return_value mock_pg.get_stats_sync.return_value = [] - result = runner.invoke(app, ["audit", "--format", "json", "--fail-on-critical", "0"]) - + result = runner.invoke( + app, ["audit", "--format", "json", "--fail-on-critical", "0"] + ) + assert result.exit_code == 0 assert "Audit completed successfully" in result.stdout - + # Check arguments were parsed and overrode config mock_renderer_cls.return_value.render_json.assert_called_once() finally: os.chdir(old_cwd) + def test_cli_audit_invalid_format(): with tempfile.TemporaryDirectory() as tmpdir: old_cwd = os.getcwd() try: os.chdir(tmpdir) with open("pipelineprobe.yml", "w") as f: - f.write('warehouse:\n type: postgres\n dsn: ""\nreport:\n format: html') - + f.write( + 'warehouse:\n type: postgres\n dsn: ""\nreport:\n format: html' + ) + result = runner.invoke(app, ["audit", "--format", "csv"]) assert result.exit_code != 0 - assert "Must be one of: both, html, json" in result.stdout or "validation error" in result.stdout + assert ( + "Must be one of: both, html, json" in result.stdout + or "validation error" in result.stdout + ) finally: os.chdir(old_cwd) diff --git a/tests/test_config.py b/tests/test_config.py index 121ae2e..b2dd0b7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,10 +1,11 @@ from pipelineprobe.config import load_config, PipelineProbeConfig + def test_load_config_defaults(tmp_path): # Pass a path that doesn't exist to test defaults dummy_path = tmp_path / "nonexistent.yml" config = load_config(str(dummy_path)) - + assert isinstance(config, PipelineProbeConfig) assert config.orchestrator.type == "airflow" assert config.dbt.target == "prod" diff --git a/tests/test_connectors.py b/tests/test_connectors.py index c8c7838..9c9379e 100644 --- a/tests/test_connectors.py +++ b/tests/test_connectors.py @@ -8,28 +8,38 @@ from pipelineprobe.connectors.bigquery import BigQueryConnector from pipelineprobe.connectors.airflow import AirflowConnector + @patch("pipelineprobe.connectors.postgres.psycopg2.connect") def test_postgres_connector(mock_connect): config = WarehouseConfig(type="postgres", dsn="postgresql://fake") connector = PostgresConnector(config) - + # Mock context manager cursor mock_cursor = mock_connect.return_value.cursor.return_value.__enter__.return_value - mock_cursor.fetchall.return_value = [{"schemaname": "public", "tablename": "users", "row_count": 100, "has_timestamps": True}] - + mock_cursor.fetchall.return_value = [ + { + "schemaname": "public", + "tablename": "users", + "row_count": 100, + "has_timestamps": True, + } + ] + stats_sync = connector.get_stats_sync() assert len(stats_sync) == 1 assert stats_sync[0]["has_timestamps"] is True + @patch("pipelineprobe.connectors.postgres.psycopg2.connect") def test_postgres_connector_error(mock_connect): config = WarehouseConfig(type="postgres", dsn="postgresql://fake") connector = PostgresConnector(config) - + mock_connect.side_effect = Exception("DB Down") stats = connector.get_stats_sync() assert stats == [] + def test_snowflake_connector_missing_creds(): # Snowflake connector should validate account, username, password config = WarehouseConfig(type="snowflake") @@ -37,52 +47,61 @@ def test_snowflake_connector_missing_creds(): stats = connector.get_stats_sync() assert stats == [] + @patch("pipelineprobe.connectors.snowflake.snowflake.connector.connect") def test_snowflake_connector(mock_connect): - config = WarehouseConfig(type="snowflake", account="acc", username="usr", password="pw") + config = WarehouseConfig( + type="snowflake", account="acc", username="usr", password="pw" + ) connector = SnowflakeConnector(config) - + # Mock cursors mock_cursor = mock_connect.return_value.cursor.return_value - mock_cursor.fetchall.return_value = [ - ("PUBLIC", "USERS", 100, True) - ] - + mock_cursor.fetchall.return_value = [("PUBLIC", "USERS", 100, True)] + stats = connector.get_stats_sync() assert len(stats) == 1 assert stats[0]["tablename"] == "USERS" assert stats[0]["has_timestamps"] is True + @patch("pipelineprobe.connectors.snowflake.snowflake.connector.connect") def test_snowflake_connector_error(mock_connect): - config = WarehouseConfig(type="snowflake", account="acc", username="usr", password="pw") + config = WarehouseConfig( + type="snowflake", account="acc", username="usr", password="pw" + ) connector = SnowflakeConnector(config) mock_connect.side_effect = Exception("Snowflake Error") stats = connector.get_stats_sync() assert stats == [] + @patch("pipelineprobe.connectors.bigquery.bigquery.Client") def test_bigquery_connector(mock_client_cls): config = WarehouseConfig(type="bigquery", project_id="test_project") connector = BigQueryConnector(config) - + mock_client = mock_client_cls.return_value - + mock_job = MagicMock() + # Mock row object which behaves like a namedtuple/dict in BigQuery class MockRow: def __init__(self, **kwargs): self.__dict__.update(kwargs) - + mock_job.result.return_value = [ - MockRow(schemaname="public", tablename="users", row_count=100, has_timestamps=True) + MockRow( + schemaname="public", tablename="users", row_count=100, has_timestamps=True + ) ] mock_client.query.return_value = mock_job - + stats = connector.get_stats_sync() assert len(stats) == 1 assert stats[0]["schemaname"] == "public" + @patch("pipelineprobe.connectors.bigquery.bigquery.Client") def test_bigquery_connector_error(mock_client_cls): config = WarehouseConfig(type="bigquery", project_id="test_project") @@ -91,12 +110,18 @@ def test_bigquery_connector_error(mock_client_cls): stats = connector.get_stats_sync() assert stats == [] + def test_dbt_connector_no_files(tmp_path): - config = DbtConfig(project_dir=str(tmp_path), manifest_path="manifest.json", run_results_path="rr.json") + config = DbtConfig( + project_dir=str(tmp_path), + manifest_path="manifest.json", + run_results_path="rr.json", + ) connector = DbtConnector(config) models = connector.get_models() assert models == [] + def test_dbt_connector_success(tmp_path): manifest = { "nodes": { @@ -104,57 +129,62 @@ def test_dbt_connector_success(tmp_path): "resource_type": "model", "name": "users", "unique_id": "model.test.users", - "tags": ["daily"] + "tags": ["daily"], }, "test.test.not_null_users_id": { "resource_type": "test", "attached_node": "model.test.users", "depends_on": {"nodes": ["model.test.users"]}, - "unique_id": "test.test.not_null_users_id" - } + "unique_id": "test.test.not_null_users_id", + }, } } - rr = { - "results": [ - {"unique_id": "model.test.users", "status": "success"} - ] - } - + rr = {"results": [{"unique_id": "model.test.users", "status": "success"}]} + manifest_path = tmp_path / "manifest.json" manifest_path.write_text(json.dumps(manifest)) rr_path = tmp_path / "rr.json" rr_path.write_text(json.dumps(rr)) - - config = DbtConfig(project_dir=str(tmp_path), manifest_path=str(manifest_path), run_results_path=str(rr_path)) + + config = DbtConfig( + project_dir=str(tmp_path), + manifest_path=str(manifest_path), + run_results_path=str(rr_path), + ) connector = DbtConnector(config) - + models = connector.get_models() assert len(models) == 1 assert models[0].name == "users" assert models[0].tests_count == 1 assert models[0].last_run_status == "success" + @patch("httpx.Client") def test_airflow_connector(mock_httpx_client): config = AirflowConfig(base_url="http://fake", username="u", password="p") - + # Mock get_dags setup mock_response_1 = MagicMock() mock_response_1.status_code = 200 - mock_response_1.json.return_value = {"dags": [{"dag_id": "dag1", "is_active": True, "owners": ["me"]}], "total_entries": 1} - + mock_response_1.json.return_value = { + "dags": [{"dag_id": "dag1", "is_active": True, "owners": ["me"]}], + "total_entries": 1, + } + mock_response_2 = MagicMock() mock_response_2.status_code = 200 mock_response_2.json.return_value = {"dags": [], "total_entries": 1} - + mock_client = mock_httpx_client.return_value mock_client.get.side_effect = [mock_response_1, mock_response_2] - + connector = AirflowConnector(config) dags = connector.get_dags() assert len(dags) == 1 assert dags[0].id == "dag1" + @patch("httpx.Client") def test_airflow_connector_dag_runs(mock_httpx_client): config = AirflowConfig(base_url="http://fake", username="u", password="p") @@ -167,42 +197,37 @@ def test_airflow_connector_dag_runs(mock_httpx_client): "dag_run_id": "run1", "state": "success", "start_date": "2023-01-01T12:00:00Z", - "end_date": "2023-01-01T12:30:00Z" + "end_date": "2023-01-01T12:30:00Z", } ] } mock_client = mock_httpx_client.return_value mock_client.get.return_value = mock_response - + connector = AirflowConnector(config) runs = connector.get_dag_runs("dag1") assert len(runs) == 1 assert runs[0].state == "success" assert runs[0].start_time is not None + @patch("httpx.Client") def test_airflow_connector_tasks(mock_httpx_client): config = AirflowConfig(base_url="http://fake", username="u", password="p") - + mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = { "tasks": [ - { - "task_id": "task1", - "retries": 3, - "ui_color": "#000", - "email": ["x@y.com"] - } + {"task_id": "task1", "retries": 3, "ui_color": "#000", "email": ["x@y.com"]} ] } mock_client = mock_httpx_client.return_value mock_client.get.return_value = mock_response - + connector = AirflowConnector(config) tasks = connector.get_tasks("dag1") assert len(tasks) == 1 assert tasks[0].task_id == "task1" assert tasks[0].retries == 3 assert tasks[0].has_alerts is True - diff --git a/tests/test_renderer.py b/tests/test_renderer.py index 00d017e..68c16f3 100644 --- a/tests/test_renderer.py +++ b/tests/test_renderer.py @@ -5,6 +5,7 @@ from pipelineprobe.renderer import ReportRenderer from pipelineprobe.models import Issue + def test_render_html(tmp_path): renderer = ReportRenderer(output_dir=str(tmp_path)) issues = [ @@ -14,16 +15,11 @@ def test_render_html(tmp_path): summary="High failure rate", details="Failed 5/5 times", recommendation="Fix it", - affected_resources=["dag_1"] + affected_resources=["dag_1"], ) ] - summary = { - "score": 50, - "total_critical": 1, - "total_warning": 0, - "total_info": 0 - } - + summary = {"score": 50, "total_critical": 1, "total_warning": 0, "total_info": 0} + # We must mock the template rendering since Jinja2 expects the templates/ directory # Rather than mocking Jinja, let's just make sure it creates the file. # To do this cleanly, we need to pass a valid template environment or mock the render. @@ -34,7 +30,10 @@ def test_render_html(tmp_path): except Exception as e: # If the template isn't installed properly in the test env, this might fail, # but the logic itself is covered. - logging.warning(f"HTML render failed (likely missing template in test env): {e}") + logging.warning( + f"HTML render failed (likely missing template in test env): {e}" + ) + def test_render_json(tmp_path): renderer = ReportRenderer(output_dir=str(tmp_path)) @@ -45,7 +44,7 @@ def test_render_json(tmp_path): summary="No SLA", details="Task missing SLA", recommendation="Add SLA", - affected_resources=["dag_1.task_1"] + affected_resources=["dag_1.task_1"], ) ] summary = { @@ -54,17 +53,17 @@ def test_render_json(tmp_path): "total_warning": 1, "total_info": 0, # Put a timedelta in the summary to test the custom encoder - "time_taken": timedelta(seconds=10) + "time_taken": timedelta(seconds=10), } - + renderer.render_json(issues, summary) - + output_file = tmp_path / "report.json" assert output_file.exists() - + with open(output_file, "r") as f: data = json.load(f) - + assert data["summary"]["score"] == 100 assert data["summary"]["time_taken"] == 10.0 # timedelta serialized to seconds assert len(data["issues"]) == 1 diff --git a/tests/test_rules.py b/tests/test_rules.py index a29fef3..13631f6 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -1,36 +1,65 @@ from datetime import datetime, timedelta -from pipelineprobe.rules.airflow_rules import check_missing_retries, check_missing_slas, check_stale_dags, check_high_failure_rate -from pipelineprobe.rules.postgres_rules import check_large_tables, check_missing_timestamps +from pipelineprobe.rules.airflow_rules import ( + check_missing_retries, + check_missing_slas, + check_stale_dags, + check_high_failure_rate, +) +from pipelineprobe.rules.postgres_rules import ( + check_large_tables, + check_missing_timestamps, +) from pipelineprobe.rules.dbt_rules import check_failing_models, check_missing_tests from pipelineprobe.models import Dag, DagRun + def test_check_missing_retries(mock_airflow_tasks): context = {"airflow_tasks": mock_airflow_tasks} issues = check_missing_retries(context) - + assert len(issues) == 1 issue = issues[0] assert issue.severity == "warning" assert "task_no_retries" in issue.affected_resources + def test_check_missing_slas(mock_airflow_tasks): context = {"airflow_tasks": mock_airflow_tasks} issues = check_missing_slas(context) - + assert len(issues) == 1 issue = issues[0] assert issue.severity == "info" assert "task_no_retries" in issue.affected_resources + def test_check_stale_dags(): dags = [ - Dag(id="stale_dag", is_active=True, owner="test", recent_runs=[ - DagRun(state="success", start_time=datetime.now() - timedelta(days=10), end_time=datetime.now() - timedelta(days=10)) - ]), - Dag(id="fresh_dag", is_active=True, owner="test", recent_runs=[ - DagRun(state="success", start_time=datetime.now() - timedelta(days=1), end_time=datetime.now() - timedelta(days=1)) - ]), - Dag(id="no_runs_dag", is_active=True, owner="test", recent_runs=[]) + Dag( + id="stale_dag", + is_active=True, + owner="test", + recent_runs=[ + DagRun( + state="success", + start_time=datetime.now() - timedelta(days=10), + end_time=datetime.now() - timedelta(days=10), + ) + ], + ), + Dag( + id="fresh_dag", + is_active=True, + owner="test", + recent_runs=[ + DagRun( + state="success", + start_time=datetime.now() - timedelta(days=1), + end_time=datetime.now() - timedelta(days=1), + ) + ], + ), + Dag(id="no_runs_dag", is_active=True, owner="test", recent_runs=[]), ] issues = check_stale_dags({"airflow_dags": dags}) assert len(issues) == 2 @@ -38,61 +67,86 @@ def test_check_stale_dags(): assert "stale_dag" in affected assert "no_runs_dag" in affected + def test_check_high_failure_rate(): dags = [ # 5 runs, 3 failed = 60% failure rate (>20%) - Dag(id="failing_dag", is_active=True, owner="test", recent_runs=[ - DagRun(state="failed", start_time=datetime.now(), end_time=datetime.now()), - DagRun(state="failed", start_time=datetime.now(), end_time=datetime.now()), - DagRun(state="failed", start_time=datetime.now(), end_time=datetime.now()), - DagRun(state="success", start_time=datetime.now(), end_time=datetime.now()), - DagRun(state="success", start_time=datetime.now(), end_time=datetime.now()) - ]), + Dag( + id="failing_dag", + is_active=True, + owner="test", + recent_runs=[ + DagRun( + state="failed", start_time=datetime.now(), end_time=datetime.now() + ), + DagRun( + state="failed", start_time=datetime.now(), end_time=datetime.now() + ), + DagRun( + state="failed", start_time=datetime.now(), end_time=datetime.now() + ), + DagRun( + state="success", start_time=datetime.now(), end_time=datetime.now() + ), + DagRun( + state="success", start_time=datetime.now(), end_time=datetime.now() + ), + ], + ), # 5 runs, 0 failed - Dag(id="ok_dag", is_active=True, owner="test", recent_runs=[ - DagRun(state="success", start_time=datetime.now(), end_time=datetime.now()) for _ in range(5) - ]) + Dag( + id="ok_dag", + is_active=True, + owner="test", + recent_runs=[ + DagRun( + state="success", start_time=datetime.now(), end_time=datetime.now() + ) + for _ in range(5) + ], + ), ] issues = check_high_failure_rate({"airflow_dags": dags}) assert len(issues) == 1 assert "failing_dag" in issues[0].affected_resources + def test_check_large_tables(mock_postgres_tables): - context = { - "warehouse_tables": mock_postgres_tables, - "warehouse_type": "postgres" - } + context = {"warehouse_tables": mock_postgres_tables, "warehouse_type": "postgres"} issues = check_large_tables(context) - + assert len(issues) == 1 issue = issues[0] assert issue.severity == "warning" assert "huge_table" in issue.affected_resources + def test_check_missing_timestamps(): tables = [ {"tablename": "huge_no_ts", "row_count": 5_000_000, "has_timestamps": False}, {"tablename": "huge_with_ts", "row_count": 5_000_000, "has_timestamps": True}, - {"tablename": "small_no_ts", "row_count": 100, "has_timestamps": False} + {"tablename": "small_no_ts", "row_count": 100, "has_timestamps": False}, ] - issues = check_missing_timestamps({ - "warehouse_tables": tables, - "warehouse_type": "postgres" - }) + issues = check_missing_timestamps( + {"warehouse_tables": tables, "warehouse_type": "postgres"} + ) assert len(issues) == 1 assert "huge_no_ts" in issues[0].affected_resources + def test_check_missing_tests(mock_dbt_models): issues = check_missing_tests({"dbt_models": mock_dbt_models}) assert len(issues) == 1 assert "model_with_no_tests" in issues[0].affected_resources + def test_check_failing_models(): from pipelineprobe.models import DbtModel + models = [ DbtModel(name="failing_model", tests_count=1, last_run_status="error", tags=[]), DbtModel(name="test_fail", tests_count=1, last_run_status="fail", tags=[]), - DbtModel(name="ok_model", tests_count=1, last_run_status="success", tags=[]) + DbtModel(name="ok_model", tests_count=1, last_run_status="success", tags=[]), ] issues = check_failing_models({"dbt_models": models}) assert len(issues) == 2 @@ -100,13 +154,15 @@ def test_check_failing_models(): assert "failing_model" in affected assert "test_fail" in affected + def test_rule_engine_error(): from pipelineprobe.rules.engine import RuleEngine + engine = RuleEngine() - + def buggy_rule(context): raise Exception("Rule Bug") - + engine.register_rule(buggy_rule) # Should handle exception and continue/return current issues issues = engine.run({})