Add dp lint command

Author: swtwsk

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- `dp lint` command that uses [SQLFluff](https://github.com/sqlfluff/sqlfluff) to lint models and tests
+
 ## [0.14.0] - 2022-02-02
 
 ## [0.13.0] - 2022-02-01

data_pipelines_cli/cli.py

@@ -7,6 +7,7 @@
 from .cli_commands.create import create_command
 from .cli_commands.deploy import deploy_command
 from .cli_commands.init import init_command
+from .cli_commands.lint import lint_command
 from .cli_commands.prepare_env import prepare_env_command
 from .cli_commands.publish import publish_command
 from .cli_commands.run import run_command
@@ -36,6 +37,7 @@ def cli() -> None:
 _cli.add_command(create_command)
 _cli.add_command(deploy_command)
 _cli.add_command(init_command)
+_cli.add_command(lint_command)
 _cli.add_command(prepare_env_command)
 _cli.add_command(publish_command)
 _cli.add_command(run_command)

data_pipelines_cli/cli_commands/lint.py

@@ -0,0 +1,144 @@
+import pathlib
+import tempfile
+from configparser import ConfigParser
+from typing import List
+
+import click
+import yaml
+
+from ..cli_constants import BUILD_DIR
+from ..cli_utils import echo_info, echo_subinfo, echo_warning, subprocess_run
+from ..config_generation import (
+    generate_profiles_yml,
+    read_dictionary_from_config_directory,
+)
+from ..errors import SQLLintError, SubprocessNonZeroExitError
+
+SQLFLUFF_FIX_NOT_EVERYTHING_ERROR = 1
+SQLFLUFF_LINT_ERROR = 65  # according to `sqlfluff.core.linter.LintingResult.stats`
+SQLFLUFF_DIALECT_LOADING_ERROR = 66  # according to `sqlfluff.cli.commands.get_config`
+
+
+def _get_dialect_or_default() -> str:
+    """Read ``dbt.yml`` config file and return its ``target_type`` or just the ``ansi``."""
+    env, dbt_filename = "base", "dbt.yml"
+    dbt_env_config = read_dictionary_from_config_directory(
+        BUILD_DIR.joinpath("dag"), env, dbt_filename
+    ) or read_dictionary_from_config_directory(pathlib.Path.cwd(), env, dbt_filename)
+    try:
+        dialect = dbt_env_config["target_type"]
+        echo_subinfo(f'Found target_type "{dialect}", attempting to use it as the SQL dialect.')
+    except KeyError:
+        dialect = "ansi"
+        echo_warning(
+            'Could not find `target_type` in `dbt.yml`. Using the default SQL dialect ("ansi").'
+        )
+    return dialect
+
+
+def _get_source_tests_paths() -> List[pathlib.Path]:
+    with open(pathlib.Path.cwd().joinpath("dbt_project.yml"), "r") as f:
+        dbt_project_config = yaml.safe_load(f)
+    dir_names: List[str] = (
+        dbt_project_config.get("source-paths", [])
+        + dbt_project_config.get("model-paths", [])
+        + dbt_project_config.get("test-paths", [])
+    )
+    return list(map(lambda dir_name: pathlib.Path.cwd().joinpath(dir_name), dir_names))
+
+
+def _create_temporary_sqlfluff_config(env: str) -> ConfigParser:
+    config = ConfigParser()
+    config["sqlfluff"] = {"templater": "dbt"}
+    config["sqlfluff:templater:dbt"] = {
+        "profiles_dir": str(generate_profiles_yml(env, copy_config_dir=True).absolute())
+    }
+    return config
+
+
+def _run_sqlfluff(command: str, dialect: str, env: str, additional_args: List[str]) -> None:
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_config_path = pathlib.Path(tmp_dir).joinpath("sqlfluff.config")
+        with open(tmp_config_path, "w") as tmp_config:
+            _create_temporary_sqlfluff_config(env).write(tmp_config)
+
+        def sqlfluff_args(sql_dialect: str) -> List[str]:
+            return [
+                "sqlfluff",
+                command,
+                "--dialect",
+                sql_dialect,
+                "--config",
+                str(tmp_config_path),
+                *additional_args,
+                *map(str, _get_source_tests_paths()),
+            ]
+
+        try:
+            subprocess_run(sqlfluff_args(dialect))
+        except SubprocessNonZeroExitError as err:
+            if err.exit_code == SQLFLUFF_DIALECT_LOADING_ERROR and dialect != "ansi":
+                subprocess_run(sqlfluff_args("ansi"))
+            else:
+                raise err
+
+
+def _run_fix_sqlfluff(dialect: str, env: str) -> None:
+    try:
+        echo_subinfo("Attempting to fix SQLs. Not every error can be automatically fixed.")
+        _run_sqlfluff("fix", dialect, env, ["--force"])
+    except SubprocessNonZeroExitError as err:
+        if err.exit_code != SQLFLUFF_FIX_NOT_EVERYTHING_ERROR:
+            raise err
+
+
+def _run_lint_sqlfluff(dialect: str, env: str) -> None:
+    try:
+        echo_subinfo("Linting SQLs.")
+        _run_sqlfluff("lint", dialect, env, [])
+    except SubprocessNonZeroExitError as err:
+        if err.exit_code == SQLFLUFF_LINT_ERROR:
+            raise SQLLintError
+        else:
+            raise err
+
+
+def lint(fix: bool, env: str) -> None:
+    """
+    Lint and format SQL.
+
+    :param fix: Whether to lint and fix linting errors, or just lint.
+    :type fix: bool
+    :param env: Name of the environment
+    :type env: str
+    """
+    echo_info("Linting SQLs:")
+    dialect = _get_dialect_or_default()
+    if fix:
+        _run_fix_sqlfluff(dialect, env)
+    _run_lint_sqlfluff(dialect, env)
+
+
+@click.command(
+    name="lint",
+    short_help="Lint and format SQL",
+    help="Lint and format SQL using SQLFluff.\n\n"
+    "For more information on rules and the workings of SQLFluff, "
+    "refer to https://docs.sqlfluff.com/",
+)
+@click.option(
+    "--no-fix",
+    is_flag=True,
+    default=False,
+    type=bool,
+    help="Whether to lint and fix linting errors, or just lint.",
+)
+@click.option(
+    "--env",
+    default="local",
+    type=str,
+    show_default=True,
+    help="Name of the environment",
+)
+def lint_command(no_fix: bool, env: str) -> None:
+    lint(not no_fix, env)

data_pipelines_cli/errors.py

@@ -38,8 +38,11 @@ def __init__(self, project_path: str) -> None:
 class SubprocessNonZeroExitError(DataPipelinesError):
     """Exception raised if subprocess exits with non-zero exit code"""
 
+    exit_code: int
+
     def __init__(self, subprocess_name: str, exit_code: int) -> None:
         self.message = f"{subprocess_name} has exited with non-zero exit code: {exit_code}"
+        self.exit_code = exit_code
 
 
 class SubprocessNotFound(DataPipelinesError):
@@ -80,3 +83,10 @@ class DockerErrorResponseError(DataPipelinesError):
 
     def __init__(self, error_msg: str) -> None:
         self.message = "Error raised when using Docker.\n" + error_msg
+
+
+class SQLLintError(DataPipelinesError):
+    """Exception raised if there are linting problems in some files."""
+
+    def __init__(self) -> None:
+        self.message = "Fix SQL linting errors."

docs/conf.py

@@ -9,7 +9,7 @@
 import os
 import sys
 
-sys.path.insert(0, os.path.abspath("../data_pipelines_cli"))
+sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------

docs/source/data_pipelines_cli.cli_commands.rst

@@ -49,6 +49,14 @@ data\_pipelines\_cli.cli\_commands.init module
    :undoc-members:
    :show-inheritance:
 
+data\_pipelines\_cli.cli\_commands.lint module
+----------------------------------------------
+
+.. automodule:: data_pipelines_cli.cli_commands.lint
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 data\_pipelines\_cli.cli\_commands.prepare\_env module
 ------------------------------------------------------
 
@@ -96,4 +104,3 @@ data\_pipelines\_cli.cli\_commands.update module
    :members:
    :undoc-members:
    :show-inheritance:
-

setup.py

@@ -10,10 +10,10 @@
     "questionary==1.10.0",
     "pyyaml>=5.1, <6.0",
     "types-PyYAML>=6.0",
-    "copier==5.1.0",
-    "dbt>=0.21, <0.22",
+    "dbt==0.21.0",
     "Jinja2>=2.11,<2.12",
     "fsspec",
+    "copier==5.1.0",
 ]
 
 EXTRA_FILESYSTEMS_REQUIRE = {
@@ -25,6 +25,10 @@
     "docker": ["docker>=5.0"],
     "datahub": ["acryl-datahub>=0.8.17, <0.8.18"],
     "git": ["GitPython==3.1.26"],
+    "lint": [
+        "sqlfluff==0.9.0",
+        "sqlfluff-templater-dbt==0.9.0",
+    ],
     "tests": [
         "pytest>=6.2.2, <7.0.0",
         "pytest-cov>=2.8.0, <3.0.0",