From b2c2d522188d5875a9069d05e0251e20f89f4b4d Mon Sep 17 00:00:00 2001 From: Ben King <9087625+benfdking@users.noreply.github.com> Date: Thu, 28 Nov 2024 17:21:34 +0000 Subject: [PATCH] temp [ci skip] --- crates/lib/src/templaters/jinja_templater.py | 831 ++-- .../jinja_templater_builtins_common.py | 31 + .../jinja_templater_builtins_dbt.py | 60 + .../src/templaters/jinja_templater_test.py | 3675 +++++++++-------- .../src/templaters/jinja_templater_tracers.py | 2 +- pyproject.toml | 3 +- 6 files changed, 2354 insertions(+), 2248 deletions(-) create mode 100644 crates/lib/src/templaters/jinja_templater_builtins_common.py create mode 100644 crates/lib/src/templaters/jinja_templater_builtins_dbt.py diff --git a/crates/lib/src/templaters/jinja_templater.py b/crates/lib/src/templaters/jinja_templater.py index 7acbb5519..32e1a3d40 100644 --- a/crates/lib/src/templaters/jinja_templater.py +++ b/crates/lib/src/templaters/jinja_templater.py @@ -1,13 +1,8 @@ """Defines the templaters.""" import copy -import importlib -import importlib.util import logging import os.path -import pkgutil -import sys -from functools import reduce from typing import ( TYPE_CHECKING, Any, @@ -37,19 +32,8 @@ from jinja2.ext import Extension from jinja2.sandbox import SandboxedEnvironment -from sqlfluff.core.config import FluffConfig -from sqlfluff.core.errors import SQLFluffUserError, SQLTemplaterError -from sqlfluff.core.formatter import FormatterInterface -from sqlfluff.core.helpers.slice import is_zero_slice, slice_length -from sqlfluff.core.templaters.base import ( - RawFileSlice, - TemplatedFile, - TemplatedFileSlice, - large_file_check, -) -from sqlfluff.core.templaters.builtins.dbt import DBT_BUILTINS -from sqlfluff.core.templaters.python import PythonTemplater -from sqlfluff.core.templaters.slicers.tracer import JinjaAnalyzer, JinjaTrace +from templaters.python_templater import PythonTemplater, SQLTemplaterError, TemplatedFile, RawFileSlice, \ + TemplatedFileSlice if TYPE_CHECKING: # pragma: no cover from jinja2.runtime import Macro @@ -58,6 +42,11 @@ templater_logger = logging.getLogger("sqlfluff.templater") +def is_zero_slice(s: slice) -> bool: + """Return true if this is a zero slice.""" + is_zero: bool = s.stop == s.start + return is_zero + class UndefinedRecorder: """Similar to jinja2.StrictUndefined, but remembers, not fails.""" @@ -104,8 +93,6 @@ class JinjaTemplater(PythonTemplater): See: https://jinja.palletsprojects.com/ """ - name = "jinja" - class Libraries: """Mock namespace for user-defined Jinja library.""" @@ -212,112 +199,115 @@ def _extract_macros_from_path( ) return macro_ctx - def _extract_macros_from_config( - self, config: FluffConfig, env: Environment, ctx: Dict[str, Any] - ) -> Dict[str, "Macro"]: - """Take a config and load any macros from it. - - Args: - config: The config to extract macros from. - env: The environment. - ctx: The context. - - Returns: - dict: A dictionary containing the extracted macros. - """ - if config: - loaded_context = ( - config.get_section((self.templater_selector, self.name, "macros")) or {} - ) - else: # pragma: no cover TODO? - loaded_context = {} - - # Iterate to load macros - macro_ctx: Dict[str, "Macro"] = {} - for value in loaded_context.values(): - try: - macro_ctx.update( - self._extract_macros_from_template(value, env=env, ctx=ctx) - ) - except TemplateSyntaxError as err: - raise SQLFluffUserError( - f"Error loading user provided macro:\n`{value}`\n> {err}." - ) - return macro_ctx - - def _extract_libraries_from_config(self, config: FluffConfig) -> Dict[str, Any]: - """Extracts libraries from the given configuration. - - This function iterates over the modules in the library path and - imports them dynamically. The imported modules are then added to a 'Libraries' - object, which is returned as a dictionary excluding magic methods. - - Args: - config: The configuration object. - - Returns: - dict: A dictionary containing the extracted libraries. - """ - # If a more global library_path is set, let that take precedence. - library_path = config.get("library_path") or config.get_section(( - self.templater_selector, - self.name, - "library_path", - )) - if not library_path: - return {} - - libraries = JinjaTemplater.Libraries() - - # If library_path has __init__.py we parse it as one module, else we parse it - # a set of modules - is_library_module = os.path.exists(os.path.join(library_path, "__init__.py")) - library_module_name = os.path.basename(library_path) - - # Need to go one level up to parse as a module correctly - walk_path = ( - os.path.join(library_path, "..") if is_library_module else library_path - ) - - for module_finder, module_name, _ in pkgutil.walk_packages([walk_path]): - # skip other modules that can be near module_dir - if is_library_module and not module_name.startswith(library_module_name): - continue - - # import_module is deprecated as of python 3.4. This follows roughly - # the guidance of the python docs: - # https://docs.python.org/3/library/importlib.html#approximating-importlib-import-module - spec = module_finder.find_spec(module_name, None) - assert spec, ( - f"Module {module_name} failed to be found despite being listed." - ) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - assert spec.loader, f"Module {module_name} missing expected loader." - spec.loader.exec_module(module) - - if "." in module_name: # nested modules have `.` in module_name - *module_path, last_module_name = module_name.split(".") - # find parent module recursively - parent_module = reduce( - lambda res, path_part: getattr(res, path_part), - module_path, - libraries, - ) - - # set attribute on module object to make jinja working correctly - setattr(parent_module, last_module_name, module) - else: - # set attr on `libraries` obj to make it work in jinja nicely - setattr(libraries, module_name, module) - - if is_library_module: - # when library is module we have one more root module in hierarchy and we - # remove it - libraries = getattr(libraries, library_module_name) - - # remove magic methods from result - return {k: v for k, v in libraries.__dict__.items() if not k.startswith("__")} + # TODO Potentially reimplement + # def _extract_macros_from_config( + # self, config: FluffConfig, env: Environment, ctx: Dict[str, Any] + # ) -> Dict[str, "Macro"]: + # """Take a config and load any macros from it. + # + # Args: + # config: The config to extract macros from. + # env: The environment. + # ctx: The context. + # + # Returns: + # dict: A dictionary containing the extracted macros. + # """ + # if config: + # loaded_context = ( + # config.get_section((self.templater_selector, self.name, "macros")) or {} + # ) + # else: # pragma: no cover TODO? + # loaded_context = {} + # + # # Iterate to load macros + # macro_ctx: Dict[str, "Macro"] = {} + # for value in loaded_context.values(): + # try: + # macro_ctx.update( + # self._extract_macros_from_template(value, env=env, ctx=ctx) + # ) + # except TemplateSyntaxError as err: + # raise SQLFluffUserError( + # f"Error loading user provided macro:\n`{value}`\n> {err}." + # ) + # return macro_ctx + + # def _extract_libraries_from_config(self, config: FluffConfig) -> Dict[str, Any]: + # """Extracts libraries from the given configuration. + # + # This function iterates over the modules in the library path and + # imports them dynamically. The imported modules are then added to a 'Libraries' + # object, which is returned as a dictionary excluding magic methods. + # + # Args: + # config: The configuration object. + # + # Returns: + # dict: A dictionary containing the extracted libraries. + # """ + # # If a more global library_path is set, let that take precedence. + # library_path = config.get("library_path") or config.get_section( + # ( + # self.templater_selector, + # self.name, + # "library_path", + # ) + # ) + # if not library_path: + # return {} + # + # libraries = JinjaTemplater.Libraries() + # + # # If library_path has __init__.py we parse it as one module, else we parse it + # # a set of modules + # is_library_module = os.path.exists(os.path.join(library_path, "__init__.py")) + # library_module_name = os.path.basename(library_path) + # + # # Need to go one level up to parse as a module correctly + # walk_path = ( + # os.path.join(library_path, "..") if is_library_module else library_path + # ) + # + # for module_finder, module_name, _ in pkgutil.walk_packages([walk_path]): + # # skip other modules that can be near module_dir + # if is_library_module and not module_name.startswith(library_module_name): + # continue + # + # # import_module is deprecated as of python 3.4. This follows roughly + # # the guidance of the python docs: + # # https://docs.python.org/3/library/importlib.html#approximating-importlib-import-module + # spec = module_finder.find_spec(module_name, None) + # assert ( + # spec + # ), f"Module {module_name} failed to be found despite being listed." + # module = importlib.util.module_from_spec(spec) + # sys.modules[module_name] = module + # assert spec.loader, f"Module {module_name} missing expected loader." + # spec.loader.exec_module(module) + # + # if "." in module_name: # nested modules have `.` in module_name + # *module_path, last_module_name = module_name.split(".") + # # find parent module recursively + # parent_module = reduce( + # lambda res, path_part: getattr(res, path_part), + # module_path, + # libraries, + # ) + # + # # set attribute on module object to make jinja working correctly + # setattr(parent_module, last_module_name, module) + # else: + # # set attr on `libraries` obj to make it work in jinja nicely + # setattr(libraries, module_name, module) + # + # if is_library_module: + # # when library is module we have one more root module in hierarchy and we + # # remove it + # libraries = getattr(libraries, library_module_name) + # + # # remove magic methods from result + # return {k: v for k, v in libraries.__dict__.items() if not k.startswith("__")} @classmethod def _crawl_tree( @@ -342,278 +332,283 @@ def _crawl_tree( line_pos=pos, ) - def _get_jinja_env(self, config: Optional[FluffConfig] = None) -> Environment: - """Get a properly configured jinja environment. - - This method returns a properly configured jinja environment. It - first checks if the 'ignore' key is present in the config dictionary and - if it contains the value 'templating'. If so, it creates a subclass of - FileSystemLoader called SafeFileSystemLoader that overrides the - get_source method to handle missing templates when templating is ignored. - If 'ignore' is not present or does not contain 'templating', it uses the - regular FileSystemLoader. It then sets the extensions to ['jinja2.ext.do'] - and adds the DBTTestExtension if the _apply_dbt_builtins method returns - True. Finally, it returns a SandboxedEnvironment object with the - specified settings. - - Args: - config (dict, optional): A dictionary containing configuration settings. - - Returns: - jinja2.Environment: A properly configured jinja environment. - """ - loader: Optional[FileSystemLoader] - macros_path = self._get_macros_path(config, "load_macros_from_path") - loader_search_path = self._get_loader_search_path(config) - final_search_path = (loader_search_path or []) + (macros_path or []) - - ignore_templating = config and "templating" in config.get("ignore") - if ignore_templating: - - class SafeFileSystemLoader(FileSystemLoader): - def get_source( - self, environment: Environment, name: str - ) -> Tuple[str, str, Callable[..., Any]]: - try: - if not isinstance(name, DummyUndefined): - return super().get_source(environment, name) - raise TemplateNotFound(str(name)) - except TemplateNotFound: - # When ignore=templating is set, treat missing files - # or attempts to load an "Undefined" file as the first - # 'base' part of the name / filename rather than failing. - templater_logger.debug( - "Providing dummy contents for Jinja macro file: %s", name - ) - value = os.path.splitext(os.path.basename(str(name)))[0] - return value, f"{value}.sql", lambda: False - - loader = SafeFileSystemLoader(final_search_path or []) - else: - loader = FileSystemLoader(final_search_path) if final_search_path else None - extensions: List[Union[str, Type[Extension]]] = ["jinja2.ext.do"] - if self._apply_dbt_builtins(config): - extensions.append(DBTTestExtension) - - return SandboxedEnvironment( - # We explicitly want to preserve newlines. - keep_trailing_newline=True, - # The do extension allows the "do" directive - autoescape=False, - extensions=extensions, - loader=loader, - ) - - def _get_macros_path( - self, config: Optional[FluffConfig], key: str - ) -> Optional[List[str]]: - """Get the list of macros paths from the provided config object. - - This method searches for a config section specified by the - templater_selector, name, and key specified. If the section is - found, it retrieves the value associated with that section and splits it into - a list of strings using a comma as the delimiter. The resulting list is - stripped of whitespace and empty strings and returned. If the section is not - found or the resulting list is empty, it returns None. - - Args: - config (FluffConfig): The config object to search for the macros path - section. - key (str): Key to load the macros path from the config file. - Also used for loading the excluding macros path from config. - - Returns: - Optional[List[str]]: The list of macros paths if found, None otherwise. - """ - if config: - macros_path = config.get_section((self.templater_selector, self.name, key)) - if macros_path: - result = [s.strip() for s in macros_path.split(",") if s.strip()] - if result: - return result - return None - - def _get_loader_search_path( - self, config: Optional[FluffConfig] - ) -> Optional[List[str]]: - """Get the list of Jinja loader search paths from the provided config object. - - This method searches for a config section specified by the - templater_selector, name, and 'loader_search_path' keys. If the section is - found, it retrieves the value associated with that section and splits it into - a list of strings using a comma as the delimiter. The resulting list is - stripped of whitespace and empty strings and returned. If the section is not - found or the resulting list is empty, it returns None. - - Args: - config (FluffConfig): The config object to search for the loader search - path section. - - Returns: - Optional[List[str]]: The list of loader search paths if found, None - otherwise. - """ - if config: - loader_search_path = config.get_section(( - self.templater_selector, - self.name, - "loader_search_path", - )) - if loader_search_path: - result = [s.strip() for s in loader_search_path.split(",") if s.strip()] - if result: - return result - return None - - def _get_jinja_analyzer(self, raw_str: str, env: Environment) -> JinjaAnalyzer: - """Creates a new object derived from JinjaAnalyzer. - - Derived classes can provide their own analyzers (e.g. to support custom Jinja - tags). - """ - return JinjaAnalyzer(raw_str, env) - - def _apply_dbt_builtins(self, config: Optional[FluffConfig]) -> bool: - """Check if dbt builtins should be applied from the provided config object. - - This method searches for a config section specified by the - templater_selector, name, and 'apply_dbt_builtins' keys. If the section - is found, it returns the value associated with that section. If the - section is not found, it returns False. - - Args: - config (FluffConfig): The config object to search for the apply_dbt_builtins - section. - - Returns: - bool: True if dbt builtins should be applied, False otherwise. - """ - if config: - apply_dbt_builtins = config.get_section(( - self.templater_selector, - self.name, - "apply_dbt_builtins", - )) - # If the config is totally absent for this templater, default to False, - # but for any other value that isn't boolean, throw an error. - if apply_dbt_builtins is None: - apply_dbt_builtins = False - assert isinstance(apply_dbt_builtins, bool), ( - f"`apply_dbt_builtins` for {self.templater_selector}.{self.name} " - f"must be True/False, not {apply_dbt_builtins!r}" - ) - return apply_dbt_builtins - return False - - def _get_env_context( - self, - fname: Optional[str], - config: Optional[FluffConfig], - env: Environment, - ) -> Dict[str, Any]: - """Get the templating context from the config. - - NOTE: This closely mirrors the `get_context` method which we inherit from the - python templater, but extends the signature. For that reason we define a new - method here, which internally refers to `get_context`. - - Args: - fname (str, optional): The name of the file. - config (dict, optional): The configuration. - env: The Jinja Environment. - - Returns: - dict: The templating context. - """ - # Load the context - live_context = self.get_context(fname, config) - # Apply dbt builtin functions if we're allowed. - if config: - # first make libraries available in the context - # so they can be used by the macros too - libraries = self._extract_libraries_from_config(config=config) - live_context.update(libraries) - - jinja_filters = libraries.get("SQLFLUFF_JINJA_FILTERS") - if jinja_filters: - env.filters.update(jinja_filters) - - if self._apply_dbt_builtins(config): - for name in DBT_BUILTINS: - # Only apply if it hasn't already been set at this stage. - if name not in live_context: - live_context[name] = DBT_BUILTINS[name] - - # Load macros from path (if applicable) - if config: - macros_path = self._get_macros_path(config, "load_macros_from_path") - exclude_macros_path = self._get_macros_path( - config, "exclude_macros_from_path" - ) - if macros_path: - live_context.update( - self._extract_macros_from_path( - macros_path, - env=env, - ctx=live_context, - exclude_paths=exclude_macros_path, - ) - ) - - # Load config macros, these will take precedence over macros from the path - live_context.update( - self._extract_macros_from_config( - config=config, env=env, ctx=live_context - ) - ) - - return live_context - - def construct_render_func( - self, fname: Optional[str] = None, config: Optional[FluffConfig] = None - ) -> Tuple[Environment, Dict[str, Any], Callable[[str], str]]: - """Builds and returns objects needed to create and run templates. - - Args: - fname (Optional[str]): The name of the file. - config (Optional[dict]): The configuration settings. - - Returns: - Tuple[Environment, dict, Callable[[str], str]]: A tuple - containing the following: - - env (Environment): An instance of the 'Environment' class. - - live_context (dict): A dictionary containing the live context. - - render_func (Callable[[str], str]): A callable function - that is used to instantiate templates. - """ - # Load the context - env = self._get_jinja_env(config) - live_context = self._get_env_context(fname, config, env) - - def render_func(in_str: str) -> str: - """Used by JinjaTracer to instantiate templates. - - This function is a closure capturing internal state from process(). - Note that creating templates involves quite a bit of state known to - _this_ function but not to JinjaTracer. - - https://www.programiz.com/python-programming/closure - """ - try: - template = env.from_string(in_str, globals=live_context) - except TemplateSyntaxError as err: # pragma: no cover - # NOTE: If the template fails to parse, then this clause - # will be triggered. However in normal that should never - # happen because the template should already have been - # validated by the point this is called. Typically that - # happens when searching for undefined variables. - raise SQLTemplaterError( - f"Late failure to parse jinja template: {err}.", - line_no=err.lineno, - ) - return template.render() - - return env, live_context, render_func + # def _get_jinja_env(self, config: Optional[FluffConfig] = None) -> Environment: + # """Get a properly configured jinja environment. + # + # This method returns a properly configured jinja environment. It + # first checks if the 'ignore' key is present in the config dictionary and + # if it contains the value 'templating'. If so, it creates a subclass of + # FileSystemLoader called SafeFileSystemLoader that overrides the + # get_source method to handle missing templates when templating is ignored. + # If 'ignore' is not present or does not contain 'templating', it uses the + # regular FileSystemLoader. It then sets the extensions to ['jinja2.ext.do'] + # and adds the DBTTestExtension if the _apply_dbt_builtins method returns + # True. Finally, it returns a SandboxedEnvironment object with the + # specified settings. + # + # Args: + # config (dict, optional): A dictionary containing configuration settings. + # + # Returns: + # jinja2.Environment: A properly configured jinja environment. + # """ + # loader: Optional[FileSystemLoader] + # macros_path = self._get_macros_path(config, "load_macros_from_path") + # loader_search_path = self._get_loader_search_path(config) + # final_search_path = (loader_search_path or []) + (macros_path or []) + # + # ignore_templating = config and "templating" in config.get("ignore") + # if ignore_templating: + # + # class SafeFileSystemLoader(FileSystemLoader): + # def get_source( + # self, environment: Environment, name: str + # ) -> Tuple[str, str, Callable[..., Any]]: + # try: + # if not isinstance(name, DummyUndefined): + # return super().get_source(environment, name) + # raise TemplateNotFound(str(name)) + # except TemplateNotFound: + # # When ignore=templating is set, treat missing files + # # or attempts to load an "Undefined" file as the first + # # 'base' part of the name / filename rather than failing. + # templater_logger.debug( + # "Providing dummy contents for Jinja macro file: %s", name + # ) + # value = os.path.splitext(os.path.basename(str(name)))[0] + # return value, f"{value}.sql", lambda: False + # + # loader = SafeFileSystemLoader(final_search_path or []) + # else: + # loader = FileSystemLoader(final_search_path) if final_search_path else None + # extensions: List[Union[str, Type[Extension]]] = ["jinja2.ext.do"] + # if self._apply_dbt_builtins(config): + # extensions.append(DBTTestExtension) + # + # return SandboxedEnvironment( + # # We explicitly want to preserve newlines. + # keep_trailing_newline=True, + # # The do extension allows the "do" directive + # autoescape=False, + # extensions=extensions, + # loader=loader, + # ) + + # def _get_macros_path( + # self, config: Optional[FluffConfig], key: str + # ) -> Optional[List[str]]: + # """Get the list of macros paths from the provided config object. + # + # This method searches for a config section specified by the + # templater_selector, name, and key specified. If the section is + # found, it retrieves the value associated with that section and splits it into + # a list of strings using a comma as the delimiter. The resulting list is + # stripped of whitespace and empty strings and returned. If the section is not + # found or the resulting list is empty, it returns None. + # + # Args: + # config (FluffConfig): The config object to search for the macros path + # section. + # key (str): Key to load the macros path from the config file. + # Also used for loading the excluding macros path from config. + # + # Returns: + # Optional[List[str]]: The list of macros paths if found, None otherwise. + # """ + # if config: + # macros_path = config.get_section((self.templater_selector, self.name, key)) + # if macros_path: + # result = [s.strip() for s in macros_path.split(",") if s.strip()] + # if result: + # return result + # return None + + # def _get_loader_search_path( + # self, config: Optional[FluffConfig] + # ) -> Optional[List[str]]: + # """Get the list of Jinja loader search paths from the provided config object. + # + # This method searches for a config section specified by the + # templater_selector, name, and 'loader_search_path' keys. If the section is + # found, it retrieves the value associated with that section and splits it into + # a list of strings using a comma as the delimiter. The resulting list is + # stripped of whitespace and empty strings and returned. If the section is not + # found or the resulting list is empty, it returns None. + # + # Args: + # config (FluffConfig): The config object to search for the loader search + # path section. + # + # Returns: + # Optional[List[str]]: The list of loader search paths if found, None + # otherwise. + # """ + # if config: + # loader_search_path = config.get_section( + # ( + # self.templater_selector, + # self.name, + # "loader_search_path", + # ) + # ) + # if loader_search_path: + # result = [s.strip() for s in loader_search_path.split(",") if s.strip()] + # if result: + # return result + # return None + + # def _get_jinja_analyzer(self, raw_str: str, env: Environment) -> JinjaAnalyzer: + # """Creates a new object derived from JinjaAnalyzer. + # + # Derived classes can provide their own analyzers (e.g. to support custom Jinja + # tags). + # """ + # return JinjaAnalyzer(raw_str, env) + + # def _apply_dbt_builtins(self, config: Optional[FluffConfig]) -> bool: + # """Check if dbt builtins should be applied from the provided config object. + # + # This method searches for a config section specified by the + # templater_selector, name, and 'apply_dbt_builtins' keys. If the section + # is found, it returns the value associated with that section. If the + # section is not found, it returns False. + # + # Args: + # config (FluffConfig): The config object to search for the apply_dbt_builtins + # section. + # + # Returns: + # bool: True if dbt builtins should be applied, False otherwise. + # """ + # if config: + # apply_dbt_builtins = config.get_section( + # ( + # self.templater_selector, + # self.name, + # "apply_dbt_builtins", + # ) + # ) + # # If the config is totally absent for this templater, default to False, + # # but for any other value that isn't boolean, throw an error. + # if apply_dbt_builtins is None: + # apply_dbt_builtins = False + # assert isinstance(apply_dbt_builtins, bool), ( + # f"`apply_dbt_builtins` for {self.templater_selector}.{self.name} " + # f"must be True/False, not {apply_dbt_builtins!r}" + # ) + # return apply_dbt_builtins + # return False + + # def _get_env_context( + # self, + # fname: Optional[str], + # config: Optional[FluffConfig], + # env: Environment, + # ) -> Dict[str, Any]: + # """Get the templating context from the config. + # + # NOTE: This closely mirrors the `get_context` method which we inherit from the + # python templater, but extends the signature. For that reason we define a new + # method here, which internally refers to `get_context`. + # + # Args: + # fname (str, optional): The name of the file. + # config (dict, optional): The configuration. + # env: The Jinja Environment. + # + # Returns: + # dict: The templating context. + # """ + # # Load the context + # live_context = self.get_context(fname, config) + # # Apply dbt builtin functions if we're allowed. + # if config: + # # first make libraries available in the context + # # so they can be used by the macros too + # libraries = self._extract_libraries_from_config(config=config) + # live_context.update(libraries) + # + # jinja_filters = libraries.get("SQLFLUFF_JINJA_FILTERS") + # if jinja_filters: + # env.filters.update(jinja_filters) + # + # if self._apply_dbt_builtins(config): + # for name in DBT_BUILTINS: + # # Only apply if it hasn't already been set at this stage. + # if name not in live_context: + # live_context[name] = DBT_BUILTINS[name] + # + # # Load macros from path (if applicable) + # if config: + # macros_path = self._get_macros_path(config, "load_macros_from_path") + # exclude_macros_path = self._get_macros_path( + # config, "exclude_macros_from_path" + # ) + # if macros_path: + # live_context.update( + # self._extract_macros_from_path( + # macros_path, + # env=env, + # ctx=live_context, + # exclude_paths=exclude_macros_path, + # ) + # ) + # + # # Load config macros, these will take precedence over macros from the path + # live_context.update( + # self._extract_macros_from_config( + # config=config, env=env, ctx=live_context + # ) + # ) + # + # return live_context + + # def construct_render_func( + # self, fname: Optional[str] = None, + # # config: Optional[FluffConfig] = None + # ) -> Tuple[Environment, Dict[str, Any], Callable[[str], str]]: + # """Builds and returns objects needed to create and run templates. + # + # Args: + # fname (Optional[str]): The name of the file. + # config (Optional[dict]): The configuration settings. + # + # Returns: + # Tuple[Environment, dict, Callable[[str], str]]: A tuple + # containing the following: + # - env (Environment): An instance of the 'Environment' class. + # - live_context (dict): A dictionary containing the live context. + # - render_func (Callable[[str], str]): A callable function + # that is used to instantiate templates. + # """ + # # Load the context + # env = self._get_jinja_env(config) + # live_context = self._get_env_context(fname, config, env) + # + # def render_func(in_str: str) -> str: + # """Used by JinjaTracer to instantiate templates. + # + # This function is a closure capturing internal state from process(). + # Note that creating templates involves quite a bit of state known to + # _this_ function but not to JinjaTracer. + # + # https://www.programiz.com/python-programming/closure + # """ + # try: + # template = env.from_string(in_str, globals=live_context) + # except TemplateSyntaxError as err: # pragma: no cover + # # NOTE: If the template fails to parse, then this clause + # # will be triggered. However in normal that should never + # # happen because the template should already have been + # # validated by the point this is called. Typically that + # # happens when searching for undefined variables. + # raise SQLTemplaterError( + # f"Late failure to parse jinja template: {err}.", + # line_no=err.lineno, + # ) + # return template.render() + # + # return env, live_context, render_func def _generate_violations_for_undefined_variables( self, @@ -624,7 +619,7 @@ def _generate_violations_for_undefined_variables( """Generates violations for any undefined variables.""" violations: List[SQLTemplaterError] = [] if undefined_variables: - # Lets go through and find out where they are: + # Go through and find out where they are: for template_err_val in self._crawl_tree( syntax_tree, undefined_variables, in_str ): @@ -654,14 +649,14 @@ def _init_undefined_tracking( return undefined_variables - @large_file_check def process( self, *, in_str: str, fname: str, - config: Optional[FluffConfig] = None, - formatter: Optional[FormatterInterface] = None, + context: Optional[Dict[str, Any]] = None, + # config: Optional[FluffConfig] = None, + # formatter: Optional[FormatterInterface] = None, ) -> Tuple[TemplatedFile, List[SQLTemplaterError]]: """Process a string and return the new string. @@ -690,11 +685,11 @@ def process( Tuple[TemplatedFile, List[SQLTemplaterError]]: A tuple containing the templated file and a list of violations. """ - if not config: # pragma: no cover - raise ValueError( - "For the jinja templater, the `process()` method requires a config " - "object." - ) + # if not config: # pragma: no cover + # raise ValueError( + # "For the jinja templater, the `process()` method requires a config " + # "object." + # ) env, live_context, render_func = self.construct_render_func( fname=fname, config=config @@ -996,14 +991,13 @@ def _handle_unreached_code( trace.templated_str, ) - @large_file_check def process_with_variants( self, *, in_str: str, fname: str, - config: Optional[FluffConfig] = None, - formatter: Optional[FormatterInterface] = None, + # config: Optional[FluffConfig] = None, + # formatter: Optional[FormatterInterface] = None, ) -> Iterator[Tuple[TemplatedFile, List[SQLTemplaterError]]]: """Process a string and return one or more variant renderings. @@ -1214,3 +1208,20 @@ def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.Macro: node.name = f"test_{test_name}" node.body = parser.parse_statements(("name:endtest",), drop_needle=True) return node + + +def process_from_rust( + string: str, + fname: str, + live_context: Dict[str, Any], +) -> TemplatedFile: + """Process the call from the rust side.""" + templater = JinjaTemplater(override_context=live_context) + (output, errors) = templater.process( + in_str=string, + fname=fname, + context=live_context, + ) + if errors != []: + raise ValueError + return output diff --git a/crates/lib/src/templaters/jinja_templater_builtins_common.py b/crates/lib/src/templaters/jinja_templater_builtins_common.py new file mode 100644 index 000000000..2c09ab05c --- /dev/null +++ b/crates/lib/src/templaters/jinja_templater_builtins_common.py @@ -0,0 +1,31 @@ +"""Common classes and functions for defining templating builtins.""" + +from typing import Any, Callable + +from .python_templater import SQLTemplaterError + + +class FunctionWrapper: + """Class to wrap a callable, for better error handling. + + When called, it just delegates to the provided callable, but if + it is rendered as a string directly, it generates a templating + error. + """ + + def __init__(self, name: str, callable: Callable[..., Any]): + self._name = name + self._callable = callable + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """When the wrapper is called, call the internal function.""" + return self._callable(*args, **kwargs) + + def __str__(self) -> str: + """If we try and render the wrapper directly, throw an error.""" + raise SQLTemplaterError( + f"Unable to render builtin callable {self._name!r} as a " + "variable because it is defined as a function. To remove " + "this function from the context, set `apply_dbt_builtins` " + "to False." + ) \ No newline at end of file diff --git a/crates/lib/src/templaters/jinja_templater_builtins_dbt.py b/crates/lib/src/templaters/jinja_templater_builtins_dbt.py new file mode 100644 index 000000000..ba4d03e9b --- /dev/null +++ b/crates/lib/src/templaters/jinja_templater_builtins_dbt.py @@ -0,0 +1,60 @@ +"""Defines the jinja builtins for dbt.""" + +from typing import Any, Union + +from templaters.jinja_templater_builtins_common import FunctionWrapper + + +class RelationEmulator: + """A class which emulates the `this` class from dbt.""" + + # Tell Jinja this object is safe to call and does not alter data. + # https://jinja.palletsprojects.com/en/3.0.x/sandbox/#jinja2.sandbox.SandboxedEnvironment.is_safe_callable + unsafe_callable = False + alters_data = False + + identifier = "this_model" + schema = "this_schema" + database = "this_database" + + def __init__(self, identifier: str = "this_model") -> None: + self.identifier = identifier + + def __call__(self, *args: Any, **kwargs: Any) -> "RelationEmulator": + """When relation(*) is called return self as another relation.""" + return self + + def __getattr__(self, name: str) -> Union["RelationEmulator", bool]: + """When relation.attribute is called return self as another relation. + + NOTE: If the attribute begins with `is_`, then return a boolean True. + """ + if name[0:3] == "is_": + return True + return self + + def __str__(self) -> str: + return self.identifier + + +# NOTE: we use `FunctionWrapper` on all of the callable builtins here +# so that there's a sensible error message if someone tries to render +# them directly. +DBT_BUILTINS = { + "ref": FunctionWrapper("ref", lambda *args, **kwargs: RelationEmulator(args[-1])), + # In case of a cross project ref in dbt, model_ref is the second + # argument. Otherwise it is the only argument. + "source": FunctionWrapper( + "source", + lambda source_name, table: RelationEmulator(f"{source_name}_{table}"), + ), + "config": FunctionWrapper("config", lambda **kwargs: ""), + "var": FunctionWrapper("var", lambda variable, default="": "item"), + # `is_incremental()` renders as True, always in this case. + # TODO: This means we'll never parse other parts of the query, + # that are only reachable when `is_incremental()` returns False. + # We should try to find a solution to that. Perhaps forcing the file + # to be parsed TWICE if it uses this variable. + "is_incremental": FunctionWrapper("is_incremental", lambda: True), + "this": RelationEmulator(), +} \ No newline at end of file diff --git a/crates/lib/src/templaters/jinja_templater_test.py b/crates/lib/src/templaters/jinja_templater_test.py index 5aaa35d08..fd560cf49 100644 --- a/crates/lib/src/templaters/jinja_templater_test.py +++ b/crates/lib/src/templaters/jinja_templater_test.py @@ -6,23 +6,26 @@ loops and placeholders. """ -import logging -from collections import defaultdict -from pathlib import Path -from typing import List, NamedTuple, Union - +# import logging +# from collections import defaultdict +# from pathlib import Path +# from typing import List, NamedTuple, Union +# import pytest -from jinja2 import Environment, nodes -from jinja2.exceptions import UndefinedError -from jinja2.ext import Extension -from jinja2.nodes import Node -from jinja2.parser import Parser -from .python_templater import FluffConfig, Linter -from .python_templater import SQLFluffSkipFile, SQLFluffUserError, SQLTemplaterError -from .python_templater import BaseSegment, RawFileSlice, TemplatedFile -from .jinja_templater import JinjaTemplater, DummyUndefined -from .jinja_templater_tracers import JinjaAnalyzer, JinjaTagConfiguration +from templaters.jinja_templater import process_from_rust + +# from jinja2 import Environment, nodes +# from jinja2.exceptions import UndefinedError +# from jinja2.ext import Extension +# from jinja2.nodes import Node +# from jinja2.parser import Parser +# +# from .python_templater import FluffConfig, Linter +# from .python_templater import SQLFluffSkipFile, SQLFluffUserError, SQLTemplaterError +# from .python_templater import BaseSegment, RawFileSlice, TemplatedFile +# from .jinja_templater import JinjaTemplater, DummyUndefined +# from .jinja_templater_tracers import JinjaAnalyzer, JinjaTagConfiguration JINJA_STRING = ( "SELECT * FROM {% for c in blah %}{{c}}{% if not loop.last %}, " @@ -41,13 +44,13 @@ ) -def get_parsed(path: str) -> BaseSegment: - """Testing helper to parse paths.""" - linter = Linter() - # Get the first file matching the path string - first_path = next(linter.parse_path(path)) - # Delegate parse assertions to the `.tree` property - return first_path.tree +# def get_parsed(path: str) -> BaseSegment: +# """Testing helper to parse paths.""" +# linter = Linter() +# # Get the first file matching the path string +# first_path = next(linter.parse_path(path)) +# # Delegate parse assertions to the `.tree` property +# return first_path.tree @pytest.mark.parametrize( @@ -77,1822 +80,1822 @@ def get_parsed(path: str) -> BaseSegment: ], ids=["simple", "unboundlocal_bugfix"], ) -def test__templater_jinja(instr, expected_outstr): +def test__templater_jinja(instr: str, expected_outstr): """Test jinja templating and the treatment of whitespace.""" - t = JinjaTemplater(override_context=dict(blah="foo", condition="a < 10")) - outstr, _ = t.process( - in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) - ) - assert str(outstr) == expected_outstr - - -class RawTemplatedTestCase(NamedTuple): - """Instances of this object are test cases for test__templater_jinja_slices.""" - - name: str - instr: str - templated_str: str - - # These fields are used to check TemplatedFile.sliced_file. - expected_templated_sliced__source_list: List[str] - expected_templated_sliced__templated_list: List[str] - - # This field is used to check TemplatedFile.raw_sliced. - expected_raw_sliced__source_list: List[str] - - -@pytest.mark.parametrize( - "case", - [ - RawTemplatedTestCase( - name="basic_block", - instr="\n\n{% set x = 42 %}\nSELECT 1, 2\n", - templated_str="\n\n\nSELECT 1, 2\n", - expected_templated_sliced__source_list=[ - "\n\n", - "{% set x = 42 %}", - "\nSELECT 1, 2\n", - ], - expected_templated_sliced__templated_list=[ - "\n\n", - "", - "\nSELECT 1, 2\n", - ], - expected_raw_sliced__source_list=[ - "\n\n", - "{% set x = 42 %}", - "\nSELECT 1, 2\n", - ], - ), - RawTemplatedTestCase( - name="strip_left_block", - instr="\n\n{%- set x = 42 %}\nSELECT 1, 2\n", - templated_str="\nSELECT 1, 2\n", - expected_templated_sliced__source_list=[ - "\n\n", - "{%- set x = 42 %}", - "\nSELECT 1, 2\n", - ], - expected_templated_sliced__templated_list=[ - "", - "", - "\nSELECT 1, 2\n", - ], - expected_raw_sliced__source_list=[ - "\n\n", - "{%- set x = 42 %}", - "\nSELECT 1, 2\n", - ], - ), - RawTemplatedTestCase( - name="strip_both_block", - instr="\n\n{%- set x = 42 -%}\nSELECT 1, 2\n", - templated_str="SELECT 1, 2\n", - expected_templated_sliced__source_list=[ - "\n\n", - "{%- set x = 42 -%}", - "\n", - "SELECT 1, 2\n", - ], - expected_templated_sliced__templated_list=[ - "", - "", - "", - "SELECT 1, 2\n", - ], - expected_raw_sliced__source_list=[ - "\n\n", - "{%- set x = 42 -%}", - "\n", - "SELECT 1, 2\n", - ], - ), - RawTemplatedTestCase( - name="strip_and_templated_whitespace", - instr="SELECT {{- ' ' -}} 1{{ ' , 2' -}}\n", - templated_str="SELECT 1 , 2", - expected_templated_sliced__source_list=[ - "SELECT", - " ", - "{{- ' ' -}}", - " ", - "1", - "{{ ' , 2' -}}", - "\n", - ], - expected_templated_sliced__templated_list=[ - "SELECT", - "", # Placeholder for consumed whitespace - " ", # Placeholder for templated whitespace - "", # Placeholder for consumed whitespace - "1", - " , 2", - "", # Placeholder for consumed newline - ], - expected_raw_sliced__source_list=[ - "SELECT", - " ", - "{{- ' ' -}}", - " ", - "1", - "{{ ' , 2' -}}", - "\n", - ], - ), - RawTemplatedTestCase( - name="strip_both_block_hard", - instr="SELECT {%- set x = 42 %} 1 {%- if true -%} , 2{% endif -%}\n", - templated_str="SELECT 1, 2", - expected_templated_sliced__source_list=[ - "SELECT", - # NB: Even though the jinja tag consumes whitespace, we still - # get it here as a placeholder. - " ", - "{%- set x = 42 %}", - " 1", - # This whitespace is a separate from the 1 because it's consumed. - " ", - "{%- if true -%}", - " ", - ", 2", - "{% endif -%}", - "\n", - ], - expected_templated_sliced__templated_list=[ - "SELECT", - "", # Consumed whitespace placeholder - "", # Jinja block placeholder - " 1", - "", # Consumed whitespace - "", # Jinja block placeholder - "", # More consumed whitespace - ", 2", - "", # Jinja block - "", # Consumed final newline. - ], - expected_raw_sliced__source_list=[ - "SELECT", - " ", - "{%- set x = 42 %}", - " 1", - " ", - "{%- if true -%}", - " ", - ", 2", - "{% endif -%}", - "\n", - ], - ), - RawTemplatedTestCase( - name="basic_data", - instr="""select - c1, - {{ 'c' }}2 as user_id -""", - templated_str="""select - c1, - c2 as user_id -""", - expected_templated_sliced__source_list=[ - "select\n c1,\n ", - "{{ 'c' }}", - "2 as user_id\n", - ], - expected_templated_sliced__templated_list=[ - "select\n c1,\n ", - "c", - "2 as user_id\n", - ], - expected_raw_sliced__source_list=[ - "select\n c1,\n ", - "{{ 'c' }}", - "2 as user_id\n", - ], - ), - # Note this is basically identical to the "basic_data" case above. - # "Right strip" is not actually a thing in Jinja. - RawTemplatedTestCase( - name="strip_right_data", - instr="""SELECT - {{ 'col1,' -}} - col2 -""", - templated_str="""SELECT - col1,col2 -""", - expected_templated_sliced__source_list=[ - "SELECT\n ", - "{{ 'col1,' -}}", - "\n ", - "col2\n", - ], - expected_templated_sliced__templated_list=[ - "SELECT\n ", - "col1,", - "", - "col2\n", - ], - expected_raw_sliced__source_list=[ - "SELECT\n ", - "{{ 'col1,' -}}", - "\n ", - "col2\n", - ], - ), - RawTemplatedTestCase( - name="strip_both_data", - instr="""select - c1, - {{- 'c' -}} -2 as user_id -""", - templated_str="""select - c1,c2 as user_id -""", - expected_templated_sliced__source_list=[ - "select\n c1,", - "\n ", - "{{- 'c' -}}", - "\n", - "2 as user_id\n", - ], - expected_templated_sliced__templated_list=[ - "select\n c1,", - "", - "c", - "", - "2 as user_id\n", - ], - expected_raw_sliced__source_list=[ - "select\n c1,", - "\n ", - "{{- 'c' -}}", - "\n", - "2 as user_id\n", - ], - ), - RawTemplatedTestCase( - name="strip_both_comment", - instr="""select - c1, - {#- Column 2 -#} c2 as user_id -""", - templated_str="""select - c1,c2 as user_id -""", - expected_templated_sliced__source_list=[ - "select\n c1,", - "\n ", - "{#- Column 2 -#}", - " ", - "c2 as user_id\n", - ], - expected_templated_sliced__templated_list=[ - "select\n c1,", - "", - "", - "", - "c2 as user_id\n", - ], - expected_raw_sliced__source_list=[ - "select\n c1,", - "\n ", - "{#- Column 2 -#}", - " ", - "c2 as user_id\n", - ], - ), - RawTemplatedTestCase( - name="union_all_loop1", - instr="""{% set products = [ - 'table1', - 'table2', - ] %} - -{% for product in products %} -SELECT - brand -FROM - {{ product }} -{% if not loop.last -%} UNION ALL {%- endif %} -{% endfor %} -""", - templated_str=( - "\n\n\nSELECT\n brand\nFROM\n table1\nUNION ALL\n\nSELECT\n " - "brand\nFROM\n table2\n\n\n" - ), - expected_templated_sliced__source_list=[ - "{% set products = [\n 'table1',\n 'table2',\n ] %}", - "\n\n", - "{% for product in products %}", - "\nSELECT\n brand\nFROM\n ", - "{{ product }}", - "\n", - "{% if not loop.last -%}", - " ", - "UNION ALL", - " ", - "{%- endif %}", - "\n", - "{% endfor %}", - "\nSELECT\n brand\nFROM\n ", - "{{ product }}", - "\n", - "{% if not loop.last -%}", - "{%- endif %}", - "\n", - "{% endfor %}", - "\n", - ], - expected_templated_sliced__templated_list=[ - "", - "\n\n", - "", - "\nSELECT\n brand\nFROM\n ", - "table1", - "\n", - "", - "", - "UNION ALL", - "", - "", - "\n", - "", - "\nSELECT\n brand\nFROM\n ", - "table2", - "\n", - "", - "", - "\n", - "", - "\n", - ], - expected_raw_sliced__source_list=[ - "{% set products = [\n 'table1',\n 'table2',\n ] %}", - "\n\n", - "{% for product in products %}", - "\nSELECT\n brand\nFROM\n ", - "{{ product }}", - "\n", - "{% if not loop.last -%}", - " ", - "UNION ALL", - " ", - "{%- endif %}", - "\n", - "{% endfor %}", - "\n", - ], - ), - RawTemplatedTestCase( - "set_multiple_variables_and_define_macro", - """{% macro echo(text) %} -{{text}} -{% endmacro %} - -{% set a, b = 1, 2 %} - -SELECT - {{ echo(a) }}, - {{ echo(b) }}""", - "\n\n\n\nSELECT\n \n1\n,\n \n2\n", - [ - "{% macro echo(text) %}", - "\n", - "{{text}}", - "\n", - "{% endmacro %}", - "\n\n", - "{% set a, b = 1, 2 %}", - "\n\nSELECT\n ", - "{{ echo(a) }}", - ",\n ", - "{{ echo(b) }}", - ], - [ - "", - "", - "", - "", - "", - "\n\n", - "", - "\n\nSELECT\n ", - "\n1\n", - ",\n ", - "\n2\n", - ], - [ - "{% macro echo(text) %}", - "\n", - "{{text}}", - "\n", - "{% endmacro %}", - "\n\n", - "{% set a, b = 1, 2 %}", - "\n\nSELECT\n ", - "{{ echo(a) }}", - ",\n ", - "{{ echo(b) }}", - ], - ), - ], - ids=lambda case: case.name, -) -def test__templater_jinja_slices(case: RawTemplatedTestCase): - """Test that Jinja templater slices raw and templated file correctly.""" - t = JinjaTemplater() - templated_file, _ = t.process( - in_str=case.instr, - fname="test", - config=FluffConfig(overrides={"dialect": "ansi"}), - ) - assert templated_file is not None - assert templated_file.source_str == case.instr - assert templated_file.templated_str == case.templated_str - # Build and check the list of source strings referenced by "sliced_file". - actual_ts_source_list = [ - case.instr[ts.source_slice] for ts in templated_file.sliced_file - ] - assert actual_ts_source_list == case.expected_templated_sliced__source_list - - # Build and check the list of templated strings referenced by "sliced_file". - actual_ts_templated_list = [ - templated_file.templated_str[ts.templated_slice] - for ts in templated_file.sliced_file - ] - assert actual_ts_templated_list == case.expected_templated_sliced__templated_list - - # Build and check the list of source strings referenced by "raw_sliced". - previous_rs = None - actual_rs_source_list: List[RawFileSlice] = [] - for rs in templated_file.raw_sliced + [None]: # type: ignore - if previous_rs: - if rs: - actual_source = case.instr[previous_rs.source_idx : rs.source_idx] - else: - actual_source = case.instr[previous_rs.source_idx :] - actual_rs_source_list.append(actual_source) - previous_rs = rs - assert actual_rs_source_list == case.expected_raw_sliced__source_list - - -def test_templater_set_block_handling(): - """Test handling of literals in {% set %} blocks. - - Specifically, verify they are not modified in the alternate template. - """ - - def run_query(sql): - # Prior to the bug fix, this assertion failed. This was bad because, - # inside JinjaTracer, dbt templates similar to the one in this test - # would call the database with funky SQL (including weird strings it - # uses internally like: 00000000000000000000000000000002. - assert sql == "\n\nselect 1 from foobarfoobarfoobarfoobar_dev\n\n" - return sql - - t = JinjaTemplater(override_context=dict(run_query=run_query)) - instr = """{% set my_query1 %} -select 1 from foobarfoobarfoobarfoobar_{{ "dev" }} -{% endset %} -{% set my_query2 %} -{{ my_query1 }} -{% endset %} - -{{ run_query(my_query2) }} -""" - outstr, vs = t.process( - in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) - ) - assert str(outstr) == "\n\n\n\n\nselect 1 from foobarfoobarfoobarfoobar_dev\n\n\n" - assert len(vs) == 0 - - -def test__templater_jinja_error_variable(): - """Test missing variable error handling in the jinja templater.""" - t = JinjaTemplater(override_context=dict(blah="foo")) - instr = JINJA_STRING - outstr, vs = t.process( - in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) + outstr, _ = process_from_rust( + string=instr, fname="test", live_context=dict(blah="foo", condition="a < 10") ) - assert str(outstr) == "SELECT * FROM f, o, o WHERE \n\n" - # Check we have violations. - assert len(vs) > 0 - # Check one of them is a templating error on line 1 - assert any(v.rule_code() == "TMP" and v.line_no == 1 for v in vs) - - -def test__templater_jinja_dynamic_variable_no_violations(): - """Test no templater violation for variable defined within template.""" - t = JinjaTemplater(override_context=dict(blah="foo")) - instr = """{% if True %} - {% set some_var %}1{% endset %} - SELECT {{some_var}} -{% endif %} -""" - outstr, vs = t.process( - in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) - ) - assert str(outstr) == "\n \n SELECT 1\n\n" - # Check we have no violations. - assert len(vs) == 0 - - -def test__templater_jinja_error_syntax(): - """Test syntax problems in the jinja templater.""" - t = JinjaTemplater() - instr = "SELECT {{foo} FROM jinja_error\n" - with pytest.raises(SQLTemplaterError) as excinfo: - t.process( - in_str=instr, - fname="test", - config=FluffConfig(overrides={"dialect": "ansi"}), - ) - templater_exception = excinfo.value - assert templater_exception.rule_code() == "TMP" - assert templater_exception.line_no == 1 - assert "Failed to parse Jinja syntax" in str(templater_exception) - - -def test__templater_jinja_error_catastrophic(): - """Test error handling in the jinja templater.""" - t = JinjaTemplater(override_context=dict(blah=7)) - instr = JINJA_STRING - with pytest.raises(SQLTemplaterError) as excinfo: - t.process( - in_str=instr, - fname="test", - config=FluffConfig(overrides={"dialect": "ansi"}), - ) - templater_exception = excinfo.value - assert templater_exception.rule_code() == "TMP" - assert templater_exception.line_no == 1 - assert "Unrecoverable failure in Jinja templating" in str(templater_exception) - - -def test__templater_jinja_error_macro_path_does_not_exist(): - """Tests that an error is raised if macro path doesn't exist.""" - with pytest.raises(ValueError) as e: - JinjaTemplater().construct_render_func( - config=FluffConfig.from_path( - "test/fixtures/templater/jinja_macro_path_does_not_exist" - ) - ) - assert str(e.value).startswith("Path does not exist") - - -def test__templater_jinja_error_macro_invalid(): - """Tests that an error is raised if a macro is invalid.""" - invalid_macro_config_string = ( - "[sqlfluff]\n" - "templater = jinja\n" - "dialect = ansi\n" - "[sqlfluff:templater:jinja:macros]\n" - "a_macro_def = {% macro pkg.my_macro() %}pass{% endmacro %}\n" - ) - config = FluffConfig.from_string(invalid_macro_config_string) - with pytest.raises(SQLFluffUserError) as e: - JinjaTemplater().construct_render_func(config=config) - error_string = str(e.value) - assert error_string.startswith("Error loading user provided macro") - assert "{% macro pkg.my_macro() %}pass{% endmacro %}" in error_string - - -def test__templater_jinja_lint_empty(): - """Check that parsing a file which renders to an empty string. - - No exception should be raised, and we should get a single templated element. - """ - lntr = Linter(dialect="ansi") - parsed = lntr.parse_string(in_str='{{ "" }}') - parsed_variant = parsed.parsed_variants[0] - assert parsed_variant.templated_file.source_str == '{{ "" }}' - assert parsed_variant.templated_file.templated_str == "" - # Get the types of the segments - print(f"Segments: {parsed_variant.tree.raw_segments}") - seg_types = [seg.get_type() for seg in parsed_variant.tree.raw_segments] - assert seg_types == ["placeholder", "end_of_file"] - - -def assert_structure(yaml_loader, path, code_only=True, include_meta=False): - """Check that a parsed sql file matches the yaml file with the same name.""" - parsed = get_parsed(path + ".sql") - # Whitespace is important here to test how that's treated - tpl = parsed.to_tuple(code_only=code_only, show_raw=True, include_meta=include_meta) - # Check nothing unparsable - if "unparsable" in parsed.type_set(): - print(parsed.stringify()) - raise ValueError("Input file is unparsable.") - _, expected = yaml_loader(path + ".yml") - assert tpl == expected - - -@pytest.mark.parametrize( - "subpath,code_only,include_meta", - [ - # Config Scalar - ("jinja_a/jinja", True, False), - # Macros - ("jinja_b/jinja", False, False), - # dbt builtins - ("jinja_c_dbt/dbt_builtins_cross_ref", True, False), - ("jinja_c_dbt/dbt_builtins_config", True, False), - ("jinja_c_dbt/dbt_builtins_is_incremental", True, False), - ("jinja_c_dbt/dbt_builtins_ref", True, False), - ("jinja_c_dbt/dbt_builtins_source", True, False), - ("jinja_c_dbt/dbt_builtins_this", True, False), - ("jinja_c_dbt/dbt_builtins_this_callable", True, False), - ("jinja_c_dbt/dbt_builtins_var_default", True, False), - ("jinja_c_dbt/dbt_builtins_test", True, False), - # do directive - ("jinja_e/jinja", True, False), - # case sensitivity and python literals - ("jinja_f/jinja", True, False), - # Macro loading from a folder - ("jinja_g_macros/jinja", True, False), - # Excluding macros - ("jinja_exclude_macro_path/jinja", True, False), - # Excluding macros with running from subdirectory - ("jinja_exclude_macro_path/model_directory/jinja_sub_directory", True, False), - # jinja raw tag - ("jinja_h_macros/jinja", True, False), - ("jinja_i_raw/raw_tag", True, False), - ("jinja_i_raw/raw_tag_2", True, False), - # Library Loading from a folder - ("jinja_j_libraries/jinja", True, False), - # Priority of macros - ("jinja_k_config_override_path_macros/jinja", True, False), - # Placeholders and metas - ("jinja_l_metas/001", False, True), - ("jinja_l_metas/002", False, True), - ("jinja_l_metas/003", False, True), - ("jinja_l_metas/004", False, True), - ("jinja_l_metas/005", False, True), - ("jinja_l_metas/006", False, True), - ("jinja_l_metas/007", False, True), - ("jinja_l_metas/008", False, True), - ("jinja_l_metas/009", False, True), - ("jinja_l_metas/010", False, True), - ("jinja_l_metas/011", False, True), - # Library Loading from a folder when library is module - ("jinja_m_libraries_module/jinja", True, False), - ("jinja_n_nested_macros/jinja", True, False), - # Test more dbt configurations - ("jinja_o_config_override_dbt_builtins/override_dbt_builtins", True, False), - ("jinja_p_disable_dbt_builtins/disable_dbt_builtins", True, False), - # Load all the macros - ("jinja_q_multiple_path_macros/jinja", True, False), - ("jinja_s_filters_in_library/jinja", True, False), - # Jinja loader search path, without also loading macros into global namespace - ("jinja_t_loader_search_path/jinja", True, False), - ], -) -def test__templater_full(subpath, code_only, include_meta, yaml_loader, caplog): - """Check structure can be parsed from jinja templated files.""" - # Log the templater and lexer throughout this test - caplog.set_level(logging.DEBUG, logger="sqlfluff.templater") - caplog.set_level(logging.DEBUG, logger="sqlfluff.lexer") - - assert_structure( - yaml_loader, - "test/fixtures/templater/" + subpath, - code_only=code_only, - include_meta=include_meta, - ) - - -def test__templater_jinja_block_matching(caplog): - """Test the block UUID matching works with a complicated case.""" - caplog.set_level(logging.DEBUG, logger="sqlfluff.lexer") - path = "test/fixtures/templater/jinja_l_metas/002.sql" - # Parse the file. - parsed = get_parsed(path) - # We only care about the template elements - template_segments = [ - seg - for seg in parsed.raw_segments - if seg.is_type("template_loop") - or ( - seg.is_type("placeholder") - and seg.block_type in ("block_start", "block_end", "block_mid") - ) - ] - - # Group them together by block UUID - assert all(seg.block_uuid for seg in template_segments), ( - "All templated segments should have a block uuid!" - ) - grouped = defaultdict(list) - for seg in template_segments: - grouped[seg.block_uuid].append(seg.pos_marker.working_loc) - - print(grouped) - - # Now the matching block IDs should be found at the following positions. - # NOTE: These are working locations in the rendered file. - groups = { - "for actions clause 1": [(6, 5), (9, 5), (12, 5), (15, 5)], - "for actions clause 2": [(17, 5), (21, 5), (29, 5), (37, 5)], - # NOTE: all the if loop clauses are grouped together. - "if loop.first": [ - (18, 9), - (20, 9), - (20, 9), - (22, 9), - (22, 9), - (28, 9), - (30, 9), - (30, 9), - (36, 9), - ], - } - - # Check all are accounted for: - for clause in groups.keys(): - for block_uuid, locations in grouped.items(): - if groups[clause] == locations: - print(f"Found {clause}, locations with UUID: {block_uuid}") - break - else: - raise ValueError(f"Couldn't find appropriate grouping of blocks: {clause}") - - -class DerivedJinjaAnalyzer(JinjaAnalyzer): - """An analyzer that includes some custom Jinja tags. - - This is used for tests that show the analyzer can be extended for custom plugin - templaters that support custom tags. - """ - - @classmethod - def _get_tag_configuration(cls, tag: str) -> JinjaTagConfiguration: - tag_map = { - "up": JinjaTagConfiguration( - block_type="block_start", - block_tracking=True, - ), - "down": JinjaTagConfiguration( - block_type="block_mid", - block_tracking=True, - ), - "end": JinjaTagConfiguration( - block_type="block_end", - block_tracking=True, - ), - } - return tag_map.get(tag, super()._get_tag_configuration(tag)) - - -@pytest.mark.parametrize( - "test,result,analyzer_class", - [ - ("", [], JinjaAnalyzer), - ("foo", [("foo", "literal", 0)], JinjaAnalyzer), - ( - "foo {{bar}} z ", - [ - ("foo ", "literal", 0), - ("{{bar}}", "templated", 4), - (" z ", "literal", 11), - ], - JinjaAnalyzer, - ), - ( - ( - "SELECT {# A comment #} {{field}} {% for i in [1, 3]%}, " - "fld_{{i}}{% endfor %} FROM my_schema.{{my_table}} " - ), - [ - ("SELECT ", "literal", 0), - ("{# A comment #}", "comment", 7), - (" ", "literal", 22), - ("{{field}}", "templated", 23), - (" ", "literal", 32), - ("{% for i in [1, 3]%}", "block_start", 33, 1, "for"), - (", fld_", "literal", 53, 1), - ("{{i}}", "templated", 59, 1), - ("{% endfor %}", "block_end", 64, 1, "endfor"), - (" FROM my_schema.", "literal", 76, 2), - ("{{my_table}}", "templated", 92, 2), - (" ", "literal", 104, 2), - ], - JinjaAnalyzer, - ), - ( - "{% set thing %}FOO{% endset %} BAR", - [ - ("{% set thing %}", "block_start", 0, 1, "set"), - ("FOO", "literal", 15, 1), - ("{% endset %}", "block_end", 18, 1, "endset"), - (" BAR", "literal", 30, 2), - ], - JinjaAnalyzer, - ), - ( - # Tests Jinja "block assignment" syntax. Also tests the use of - # template substitution within the block: {{ "dev" }}. - """{% set my_query %} -select 1 from foobarfoobarfoobarfoobar_{{ "dev" }} -{% endset %} -{{ my_query }} -""", - [ - ("{% set my_query %}", "block_start", 0, 1, "set"), - ("\nselect 1 from foobarfoobarfoobarfoobar_", "literal", 18, 1), - ('{{ "dev" }}', "templated", 58, 1), - ("\n", "literal", 69, 1), - ("{% endset %}", "block_end", 70, 1, "endset"), - ("\n", "literal", 82, 2), - ("{{ my_query }}", "templated", 83, 2), - ("\n", "literal", 97, 2), - ], - JinjaAnalyzer, - ), - # Tests for jinja blocks that consume whitespace. - ( - """SELECT 1 FROM {%+if true-%} {{ref('foo')}} {%-endif%}""", - [ - ("SELECT 1 FROM ", "literal", 0), - ("{%+if true-%}", "block_start", 14, 1, "if"), - (" ", "literal", 27, 1), - ("{{ref('foo')}}", "templated", 28, 1), - (" ", "literal", 42, 1), - ("{%-endif%}", "block_end", 43, 1, "endif"), - ], - JinjaAnalyzer, - ), - ( - """{% for item in some_list -%} - SELECT * - FROM some_table -{{ "UNION ALL\n" if not loop.last }} -{%- endfor %}""", - [ - ("{% for item in some_list -%}", "block_start", 0, 1, "for"), - # This gets consumed in the templated file, but it's still here. - ("\n ", "literal", 28, 1), - ("SELECT *\n FROM some_table\n", "literal", 33, 1), - ('{{ "UNION ALL\n" if not loop.last }}', "templated", 62, 1), - ("\n", "literal", 97, 1), - ("{%- endfor %}", "block_end", 98, 1, "endfor"), - ], - JinjaAnalyzer, - ), - ( - JINJA_MACRO_CALL_SQL, - [ - ("{% macro render_name(title) %}", "block_start", 0, 1, "macro"), - ("\n '", "literal", 30, 1), - ("{{ title }}", "templated", 34, 1), - (". foo' as ", "literal", 45, 1), - ("{{ caller() }}", "templated", 55, 1), - ("\n", "literal", 69, 1), - ("{% endmacro %}", "block_end", 70, 1, "endmacro"), - ("\nSELECT\n ", "literal", 84, 2), - ("{% call render_name('Sir') %}", "block_start", 96, 3, "call"), - ("\n bar\n ", "literal", 125, 3), - ("{% endcall %}", "block_end", 142, 3, "endcall"), - ("\nFROM baz\n", "literal", 155, 4), - ], - JinjaAnalyzer, - ), - ( - # Test of tag heuristics in the default _get_tag_configuration - """{% randomtagstart %} - SELECT 1; -{% elphony %} - SELECT 2; -{% endsomethingweird %}""", - [ - ("{% randomtagstart %}", "block_start", 0, 1, "randomtagstart"), - ("\n SELECT 1;\n", "literal", 20, 1), - ("{% elphony %}", "block_mid", 35, 1, "elphony"), - ("\n SELECT 2;\n", "literal", 48, 1), - ("{% endsomethingweird %}", "block_end", 63, 1, "endsomethingweird"), - ], - JinjaAnalyzer, - ), - ( - # Basic test with a derived JinjaAnalyzer that supports some custom tags - """{% up 'create table xyz' %} - CREATE TABLE xyz (id int); -{% down %} - DROP TABLE xyz; -{% end %}""", - [ - ("{% up 'create table xyz' %}", "block_start", 0, 1, "up"), - ("\n CREATE TABLE xyz (id int);\n", "literal", 27, 1), - ("{% down %}", "block_mid", 59, 1, "down"), - ("\n DROP TABLE xyz;\n", "literal", 69, 1), - ("{% end %}", "block_end", 90, 1, "end"), - ], - DerivedJinjaAnalyzer, - ), - ], -) -def test__templater_jinja_slice_template(test, result, analyzer_class): - """Test _slice_template.""" - templater = JinjaTemplater() - env, _, render_func = templater.construct_render_func() - - analyzer = analyzer_class(test, env) - analyzer.analyze(render_func=render_func) - resp = analyzer.raw_sliced - # check contiguous (unless there's a comment in it) - if "{#" not in test: - assert "".join(elem.raw for elem in resp) == test - # check indices - idx = 0 - for raw_slice in resp: - assert raw_slice.source_idx == idx - idx += len(raw_slice.raw) - # Check total result - assert resp == [RawFileSlice(*args) for args in result] - - -class DBMigrationExtension(Extension): - """Example of a hypothetical custom Jinja extension. - - This extension might ostensibly be used to represent up/down database migrations. - """ - - tags = {"up"} - - def parse(self, parser: Parser) -> Union[Node, List[Node]]: - """Parse the up/down blocks.""" - # {% up 'migration name' %} - next(parser.stream) # skip the "up" token - parser.parse_expression() # skip the name of this migration - up_body = parser.parse_statements(("name:down",)) - # {% down %} - next(parser.stream) # skip the "down" token - down_body = parser.parse_statements(("name:end",)) - # {% end %} - next(parser.stream) - - # This is just a test, so output the blocks verbatim one after the other: - return [nodes.Scope(up_body), nodes.Scope(down_body)] - - -class DerivedJinjaTemplater(JinjaTemplater): - """A templater that includes some custom Jinja tags. - - This is used for tests that show the templater can be extended for custom plugin - templaters that support custom tags. - """ - - name = "derivedtemplater" - - def _get_jinja_env(self, config=None): - env = super()._get_jinja_env(config) - env.add_extension(DBMigrationExtension) - return env - def _get_jinja_analyzer(self, raw_str: str, env: Environment) -> JinjaAnalyzer: - return DerivedJinjaAnalyzer(raw_str, env) - - -def _statement(*args, **kwargs): - # NOTE: The standard dbt statement() call returns nothing. - return "" - - -def _load_result(*args, **kwargs): - return "_load_result" - - -@pytest.mark.parametrize( - "raw_file,override_context,result,templater_class", - [ - ("", None, [], JinjaTemplater), - ( - "foo", - None, - [("literal", slice(0, 3, None), slice(0, 3, None))], - JinjaTemplater, - ), - # Example with no loops - ( - "SELECT {{blah}}, boo {# comment #} from something", - dict(blah="foobar"), - [ - ("literal", slice(0, 7, None), slice(0, 7, None)), - ("templated", slice(7, 15, None), slice(7, 13, None)), - ("literal", slice(15, 21, None), slice(13, 19, None)), - ("comment", slice(21, 34, None), slice(19, 19, None)), - ("literal", slice(34, 49, None), slice(19, 34, None)), - ], - JinjaTemplater, - ), - # Example with loops - ( - ( - "SELECT {# A comment #} {{field}} {% for i in [1, 3, 7]%}, " - "fld_{{i}}_x{% endfor %} FROM my_schema.{{my_table}} " - ), - dict(field="foobar", my_table="barfoo"), - [ - ("literal", slice(0, 7, None), slice(0, 7, None)), - ("comment", slice(7, 22, None), slice(7, 7, None)), - ("literal", slice(22, 23, None), slice(7, 8, None)), - ("templated", slice(23, 32, None), slice(8, 14, None)), - ("literal", slice(32, 33, None), slice(14, 15, None)), - ("block_start", slice(33, 56, None), slice(15, 15, None)), - ("literal", slice(56, 62, None), slice(15, 21, None)), - ("templated", slice(62, 67, None), slice(21, 22, None)), - ("literal", slice(67, 69, None), slice(22, 24, None)), - ("block_end", slice(69, 81, None), slice(24, 24, None)), - ("literal", slice(56, 62, None), slice(24, 30, None)), - ("templated", slice(62, 67, None), slice(30, 31, None)), - ("literal", slice(67, 69, None), slice(31, 33, None)), - ("block_end", slice(69, 81, None), slice(33, 33, None)), - ("literal", slice(56, 62, None), slice(33, 39, None)), - ("templated", slice(62, 67, None), slice(39, 40, None)), - ("literal", slice(67, 69, None), slice(40, 42, None)), - ("block_end", slice(69, 81, None), slice(42, 42, None)), - ("literal", slice(81, 97, None), slice(42, 58, None)), - ("templated", slice(97, 109, None), slice(58, 64, None)), - ("literal", slice(109, 110, None), slice(64, 65, None)), - ], - JinjaTemplater, - ), - # Example with loops (and utilising the end slice code) - ( - ( - "SELECT {# A comment #} {{field}} {% for i in [1, 3, 7]%}, " - "fld_{{i}}{% endfor %} FROM my_schema.{{my_table}} " - ), - dict(field="foobar", my_table="barfoo"), - [ - ("literal", slice(0, 7, None), slice(0, 7, None)), - ("comment", slice(7, 22, None), slice(7, 7, None)), - ("literal", slice(22, 23, None), slice(7, 8, None)), - ("templated", slice(23, 32, None), slice(8, 14, None)), - ("literal", slice(32, 33, None), slice(14, 15, None)), - ("block_start", slice(33, 56, None), slice(15, 15, None)), - ("literal", slice(56, 62, None), slice(15, 21, None)), - ("templated", slice(62, 67, None), slice(21, 22, None)), - ("block_end", slice(67, 79, None), slice(22, 22, None)), - ("literal", slice(56, 62, None), slice(22, 28, None)), - ("templated", slice(62, 67, None), slice(28, 29, None)), - ("block_end", slice(67, 79, None), slice(29, 29, None)), - ("literal", slice(56, 62, None), slice(29, 35, None)), - ("templated", slice(62, 67, None), slice(35, 36, None)), - ("block_end", slice(67, 79, None), slice(36, 36, None)), - ("literal", slice(79, 95, None), slice(36, 52, None)), - ("templated", slice(95, 107, None), slice(52, 58, None)), - ("literal", slice(107, 108, None), slice(58, 59, None)), - ], - JinjaTemplater, - ), - # Test a trailing split, and some variables which don't refer anything. - ( - "{{ config(materialized='view') }}\n\nSELECT 1 FROM {{ source('finance', " - "'reconciled_cash_facts') }}\n\n", - dict( - config=lambda *args, **kwargs: "", - source=lambda *args, **kwargs: "finance_reconciled_cash_facts", - ), - [ - ("templated", slice(0, 33, None), slice(0, 0, None)), - ("literal", slice(33, 49, None), slice(0, 16, None)), - ("templated", slice(49, 97, None), slice(16, 45, None)), - ("literal", slice(97, 99, None), slice(45, 47, None)), - ], - JinjaTemplater, - ), - # Test splitting with a loop. - ( - "SELECT\n " - "{% for i in [1, 2, 3] %}\n , " - "c_{{i}}+42 AS the_meaning_of_li{{ 'f' * i }}\n " - "{% endfor %}\n" - "FROM my_table", - None, - [ - ("literal", slice(0, 11, None), slice(0, 11, None)), - ("block_start", slice(11, 35, None), slice(11, 11, None)), - ("literal", slice(35, 48, None), slice(11, 24, None)), - ("templated", slice(48, 53, None), slice(24, 25, None)), - ("literal", slice(53, 77, None), slice(25, 49, None)), - ("templated", slice(77, 90, None), slice(49, 50, None)), - ("literal", slice(90, 95, None), slice(50, 55, None)), - ("block_end", slice(95, 107, None), slice(55, 55, None)), - ("literal", slice(35, 48, None), slice(55, 68, None)), - ("templated", slice(48, 53, None), slice(68, 69, None)), - ("literal", slice(53, 77, None), slice(69, 93, None)), - ("templated", slice(77, 90, None), slice(93, 95, None)), - ("literal", slice(90, 95, None), slice(95, 100, None)), - ("block_end", slice(95, 107, None), slice(100, 100, None)), - ("literal", slice(35, 48, None), slice(100, 113, None)), - ("templated", slice(48, 53, None), slice(113, 114, None)), - ("literal", slice(53, 77, None), slice(114, 138, None)), - ("templated", slice(77, 90, None), slice(138, 141, None)), - ("literal", slice(90, 95, None), slice(141, 146, None)), - ("block_end", slice(95, 107, None), slice(146, 146, None)), - ("literal", slice(107, 121, None), slice(146, 160, None)), - ], - JinjaTemplater, - ), - # Test an example where a block is removed entirely. - ( - "{% set thing %}FOO{% endset %} SELECT 1", - None, - [ - ("block_start", slice(0, 15, None), slice(0, 0, None)), - ("literal", slice(15, 18, None), slice(0, 0, None)), - ("block_end", slice(18, 30, None), slice(0, 0, None)), - ("literal", slice(30, 39, None), slice(0, 9, None)), - ], - JinjaTemplater, - ), - ( - # Tests Jinja "include" directive. - """{% include 'subdir/include_comment.sql' %} - -SELECT 1 -""", - None, - [ - ("templated", slice(0, 42, None), slice(0, 18, None)), - ("literal", slice(42, 53, None), slice(18, 29, None)), - ], - JinjaTemplater, - ), - ( - # Tests Jinja "import" directive. - """{% import 'echo.sql' as echo %} - -SELECT 1 -""", - None, - [ - ("templated", slice(0, 31, None), slice(0, 0, None)), - ("literal", slice(31, 42, None), slice(0, 11, None)), - ], - JinjaTemplater, - ), - ( - # Tests Jinja "from import" directive.. - """{% from 'echo.sql' import echo %} -{% from 'echoecho.sql' import echoecho %} - -SELECT - {{ echo("foo") }}, - {{ echoecho("bar") }} -""", - None, - [ - ("templated", slice(0, 33, None), slice(0, 0, None)), - ("literal", slice(33, 34, None), slice(0, 1, None)), - ("templated", slice(34, 75, None), slice(1, 1, None)), - ("literal", slice(75, 88, None), slice(1, 14, None)), - ("templated", slice(88, 105, None), slice(14, 19, None)), - ("literal", slice(105, 111, None), slice(19, 25, None)), - ("templated", slice(111, 132, None), slice(25, 34, None)), - ("literal", slice(132, 133, None), slice(34, 35, None)), - ], - JinjaTemplater, - ), - ( - # Tests Jinja "do" directive. Should be treated as a - # templated instead of block - issue 4603. - """{% do true %} - -{% if true %} - select 1 -{% endif %}""", - None, - [ - ("templated", slice(0, 13, None), slice(0, 0, None)), - ("literal", slice(13, 15, None), slice(0, 2, None)), - ("block_start", slice(15, 28, None), slice(2, 2, None)), - ("literal", slice(28, 42, None), slice(2, 16, None)), - ("block_end", slice(42, 53, None), slice(16, 16, None)), - ], - JinjaTemplater, - ), - ( - # Tests issue 2541, a bug where the {%- endfor %} was causing - # IndexError: list index out of range. - """{% for x in ['A', 'B'] %} - {% if x != 'A' %} - SELECT 'E' - {% endif %} -{%- endfor %} -""", - None, - [ - ("block_start", slice(0, 25, None), slice(0, 0, None)), - ("literal", slice(25, 30, None), slice(0, 5, None)), - ("block_start", slice(30, 47, None), slice(5, 5, None)), - ("block_end", slice(67, 78, None), slice(5, 5, None)), - ("literal", slice(78, 79, None), slice(5, 5, None)), - ("block_end", slice(79, 92, None), slice(5, 5, None)), - ("literal", slice(25, 30, None), slice(5, 10, None)), - ("block_start", slice(30, 47, None), slice(10, 10, None)), - ("literal", slice(47, 67, None), slice(10, 30, None)), - ("block_end", slice(67, 78, None), slice(30, 30, None)), - ("literal", slice(78, 79, None), slice(30, 30, None)), - ("block_end", slice(79, 92, None), slice(30, 30, None)), - ("literal", slice(92, 93, None), slice(30, 31, None)), - ], - JinjaTemplater, - ), - ( - # Similar to the test above for issue 2541, but it's even trickier: - # whitespace control everywhere and NO NEWLINES or other characters - # between Jinja segments. In order to get a thorough-enough trace, - # JinjaTracer has to build the alternate template with whitespace - # control removed, as this increases the amount of trace output. - "{%- for x in ['A', 'B'] -%}" - "{%- if x == 'B' -%}" - "SELECT 'B';" - "{%- endif -%}" - "{%- if x == 'A' -%}" - "SELECT 'A';" - "{%- endif -%}" - "{%- endfor -%}", - None, - [ - ("block_start", slice(0, 27, None), slice(0, 0, None)), - ("block_start", slice(27, 46, None), slice(0, 0, None)), - ("block_end", slice(57, 70, None), slice(0, 0, None)), - ("block_start", slice(70, 89, None), slice(0, 0, None)), - ("literal", slice(89, 100, None), slice(0, 11, None)), - ("block_end", slice(100, 113, None), slice(11, 11, None)), - ("block_end", slice(113, 127, None), slice(11, 11, None)), - ("block_start", slice(27, 46, None), slice(11, 11, None)), - ("literal", slice(46, 57, None), slice(11, 22, None)), - ("block_end", slice(57, 70, None), slice(22, 22, None)), - ("block_start", slice(70, 89, None), slice(22, 22, None)), - ("block_end", slice(100, 113, None), slice(22, 22, None)), - ("block_end", slice(113, 127, None), slice(22, 22, None)), - ], - JinjaTemplater, - ), - ( - # Test for issue 2786. Also lots of whitespace control. In this - # case, removing whitespace control alone wasn't enough. In order - # to get a good trace, JinjaTracer had to be updated so the - # alternate template included output for the discarded whitespace. - """select - id, - {%- for features in ["value4", "value5"] %} - {%- if features in ["value7"] %} - {{features}} - {%- if not loop.last -%},{% endif %} - {%- else -%} - {{features}} - {%- if not loop.last -%},{% endif %} - {%- endif -%} - {%- endfor %} -from my_table -""", - None, - [ - ("literal", slice(0, 14, None), slice(0, 14, None)), - ("literal", slice(14, 19, None), slice(14, 14, None)), - ("block_start", slice(19, 62, None), slice(14, 14, None)), - ("literal", slice(62, 71, None), slice(14, 14, None)), - ("block_start", slice(71, 103, None), slice(14, 14, None)), - ("block_mid", slice(186, 198, None), slice(14, 14, None)), - ("literal", slice(198, 211, None), slice(14, 14, None)), - ("templated", slice(211, 223, None), slice(14, 20, None)), - ("literal", slice(223, 236, None), slice(20, 20, None)), - ("block_start", slice(236, 260, None), slice(20, 20, None)), - ("literal", slice(260, 261, None), slice(20, 21, None)), - ("block_end", slice(261, 272, None), slice(21, 21, None)), - ("literal", slice(272, 281, None), slice(21, 21, None)), - ("block_end", slice(281, 294, None), slice(21, 21, None)), - ("literal", slice(294, 299, None), slice(21, 21, None)), - ("block_end", slice(299, 312, None), slice(21, 21, None)), - ("literal", slice(62, 71, None), slice(21, 21, None)), - ("block_start", slice(71, 103, None), slice(21, 21, None)), - ("block_mid", slice(186, 198, None), slice(21, 21, None)), - ("literal", slice(198, 211, None), slice(21, 21, None)), - ("templated", slice(211, 223, None), slice(21, 27, None)), - ("literal", slice(223, 236, None), slice(27, 27, None)), - ("block_start", slice(236, 260, None), slice(27, 27, None)), - ("block_end", slice(261, 272, None), slice(27, 27, None)), - ("literal", slice(272, 281, None), slice(27, 27, None)), - ("block_end", slice(281, 294, None), slice(27, 27, None)), - ("literal", slice(294, 299, None), slice(27, 27, None)), - ("block_end", slice(299, 312, None), slice(27, 27, None)), - ("literal", slice(312, 327, None), slice(27, 42, None)), - ], - JinjaTemplater, - ), - ( - # Test for issue 2835. There's no space between "col" and "=". - # Also tests for issue 3750 that self contained set statements - # are parsed as "templated" and not "block_start". - """{% set col= "col1" %} -SELECT {{ col }} -""", - None, - [ - ("templated", slice(0, 21, None), slice(0, 0, None)), - ("literal", slice(21, 29, None), slice(0, 8, None)), - ("templated", slice(29, 38, None), slice(8, 12, None)), - ("literal", slice(38, 39, None), slice(12, 13, None)), - ], - JinjaTemplater, - ), - ( - # Another test for issue 2835. The {% for %} loop inside the - # {% set %} caused JinjaTracer to think the {% set %} ended - # at the {% endfor %} - """{% set some_part_of_the_query %} - {% for col in ["col1"] %} - {{col}} - {% endfor %} -{% endset %} - -SELECT {{some_part_of_the_query}} -FROM SOME_TABLE -""", - None, - [ - ("block_start", slice(0, 32, None), slice(0, 0, None)), - ("literal", slice(32, 37, None), slice(0, 0, None)), - ("block_start", slice(37, 62, None), slice(0, 0, None)), - ("literal", slice(62, 67, None), slice(0, 0, None)), - ("templated", slice(67, 74, None), slice(0, 0, None)), - ("literal", slice(74, 79, None), slice(0, 0, None)), - ("block_end", slice(79, 91, None), slice(0, 0, None)), - ("literal", slice(91, 92, None), slice(0, 0, None)), - ("block_end", slice(92, 104, None), slice(0, 0, None)), - ("literal", slice(104, 113, None), slice(0, 9, None)), - ("templated", slice(113, 139, None), slice(9, 29, None)), - ("literal", slice(139, 156, None), slice(29, 46, None)), - ], - JinjaTemplater, - ), - ( - # Third test for issue 2835. This was the original SQL provided in - # the issue report. - # Also tests for issue 3750 that self contained set statements - # are parsed as "templated" and not "block_start". - """{% set whitelisted= [ - {'name': 'COL_1'}, - {'name': 'COL_2'}, - {'name': 'COL_3'} -] %} - -{% set some_part_of_the_query %} - {% for col in whitelisted %} - {{col.name}}{{ ", " if not loop.last }} - {% endfor %} -{% endset %} - -SELECT {{some_part_of_the_query}} -FROM SOME_TABLE -""", - None, - [ - ("templated", slice(0, 94, None), slice(0, 0, None)), - ("literal", slice(94, 96, None), slice(0, 2, None)), - ("block_start", slice(96, 128, None), slice(2, 2, None)), - ("literal", slice(128, 133, None), slice(2, 2, None)), - ("block_start", slice(133, 161, None), slice(2, 2, None)), - ("literal", slice(161, 166, None), slice(2, 2, None)), - ("templated", slice(166, 178, None), slice(2, 2, None)), - ("templated", slice(178, 205, None), slice(2, 2, None)), - ("literal", slice(205, 210, None), slice(2, 2, None)), - ("block_end", slice(210, 222, None), slice(2, 2, None)), - ("literal", slice(222, 223, None), slice(2, 2, None)), - ("block_end", slice(223, 235, None), slice(2, 2, None)), - ("literal", slice(235, 244, None), slice(2, 11, None)), - ("templated", slice(244, 270, None), slice(11, 66, None)), - ("literal", slice(270, 287, None), slice(66, 83, None)), - ], - JinjaTemplater, - ), - ( - # Test for issue 2822: Handle slicing when there's no newline after - # the Jinja block end. - "{% if true %}\nSELECT 1 + 1\n{%- endif %}", - None, - [ - ("block_start", slice(0, 13, None), slice(0, 0, None)), - ("literal", slice(13, 26, None), slice(0, 13, None)), - ("literal", slice(26, 27, None), slice(13, 13, None)), - ("block_end", slice(27, 39, None), slice(13, 13, None)), - ], - JinjaTemplater, - ), - ( - # Test for issue 3434: Handle {% block %}. - "SELECT {% block table_name %}block_contents{% endblock %} " - "FROM {{ self.table_name() }}\n", - None, - [ - ("literal", slice(0, 7, None), slice(0, 7, None)), - ("literal", slice(29, 43, None), slice(7, 21, None)), - ("block_start", slice(7, 29, None), slice(21, 21, None)), - ("literal", slice(29, 43, None), slice(21, 21, None)), - ("block_end", slice(43, 57, None), slice(21, 21, None)), - ("literal", slice(57, 63, None), slice(21, 27, None)), - ("templated", slice(63, 86, None), slice(27, 27, None)), - ("literal", slice(29, 43, None), slice(27, 41, None)), - ("literal", slice(86, 87, None), slice(41, 42, None)), - ], - JinjaTemplater, - ), - ( - # Another test for issue 3434: Similar to the first, but uses - # the block inside a loop. - """{% block table_name %}block_contents{% endblock %} -SELECT -{% for j in [4, 5, 6] %} -FROM {{ j }}{{ self.table_name() }} -{% endfor %} -""", - None, - [ - ("literal", slice(22, 36, None), slice(0, 14, None)), - ("block_start", slice(0, 22, None), slice(14, 14, None)), - ("literal", slice(22, 36, None), slice(14, 14, None)), - ("block_end", slice(36, 50, None), slice(14, 14, None)), - ("literal", slice(50, 58, None), slice(14, 22, None)), - ("block_start", slice(58, 82, None), slice(22, 22, None)), - ("literal", slice(82, 88, None), slice(22, 28, None)), - ("templated", slice(88, 95, None), slice(28, 29, None)), - ("templated", slice(95, 118, None), slice(29, 29, None)), - ("literal", slice(22, 36, None), slice(29, 43, None)), - ("literal", slice(118, 119, None), slice(43, 44, None)), - ("block_end", slice(119, 131, None), slice(44, 44, None)), - ("literal", slice(82, 88, None), slice(44, 50, None)), - ("templated", slice(88, 95, None), slice(50, 51, None)), - ("templated", slice(95, 118, None), slice(51, 51, None)), - ("literal", slice(22, 36, None), slice(51, 65, None)), - ("literal", slice(118, 119, None), slice(65, 66, None)), - ("block_end", slice(119, 131, None), slice(66, 66, None)), - ("literal", slice(82, 88, None), slice(66, 72, None)), - ("templated", slice(88, 95, None), slice(72, 73, None)), - ("templated", slice(95, 118, None), slice(73, 73, None)), - ("literal", slice(22, 36, None), slice(73, 87, None)), - ("literal", slice(118, 119, None), slice(87, 88, None)), - ("block_end", slice(119, 131, None), slice(88, 88, None)), - ("literal", slice(131, 132, None), slice(88, 89, None)), - ], - JinjaTemplater, - ), - ( - "{{ statement('variables', fetch_result=true) }}\n", - dict( - statement=_statement, - load_result=_load_result, - ), - [ - ("templated", slice(0, 47, None), slice(0, 0, None)), - ("literal", slice(47, 48, None), slice(0, 1, None)), - ], - JinjaTemplater, - ), - ( - "{% call statement('variables', fetch_result=true) %}\n" - "select 1 as test\n" - "{% endcall %}\n" - "select 2 as foo\n", - dict( - statement=_statement, - load_result=_load_result, - ), - [ - ("block_start", slice(0, 52, None), slice(0, 0, None)), - ("literal", slice(52, 70, None), slice(0, 0, None)), - ("block_end", slice(70, 83, None), slice(0, 0, None)), - ("literal", slice(83, 100, None), slice(0, 17, None)), - ], - JinjaTemplater, - ), - ( - JINJA_MACRO_CALL_SQL, - None, - [ - # First all of this is the call block. - ("block_start", slice(0, 30, None), slice(0, 0, None)), - ("literal", slice(30, 34, None), slice(0, 0, None)), - ("templated", slice(34, 45, None), slice(0, 0, None)), - ("literal", slice(45, 55, None), slice(0, 0, None)), - ("templated", slice(55, 69, None), slice(0, 0, None)), - ("literal", slice(69, 70, None), slice(0, 0, None)), - ("block_end", slice(70, 84, None), slice(0, 0, None)), - # Then the actual query. - ("literal", slice(84, 96, None), slice(0, 12, None)), - # The block_start (call) contains the actual content. - ("block_start", slice(96, 125, None), slice(12, 47, None)), - # The middle and end of the call, have zero length in the template - ("literal", slice(125, 142, None), slice(47, 47, None)), - ("block_end", slice(142, 155, None), slice(47, 47, None)), - ("literal", slice(155, 165, None), slice(47, 57, None)), - ], - JinjaTemplater, - ), - ( - # Simple test of a derived templater with custom tags - """{% up 'create table xyz' %} - CREATE TABLE xyz (id int); -{% down %} - DROP TABLE xyz; -{% end %}""", - None, - [ - ("block_start", slice(0, 27, None), slice(0, 0, None)), - ("literal", slice(27, 59, None), slice(0, 32, None)), - ("block_mid", slice(59, 69, None), slice(32, 32, None)), - ("literal", slice(69, 90, None), slice(32, 53, None)), - ("block_end", slice(90, 99, None), slice(53, 53, None)), - ], - DerivedJinjaTemplater, - ), - ( - # test for issue 6121: The first rendered element - # inside the loop is far from the start position of the loop. - """ -{% for i in range(2) %}{% set a = 0 %}{% set b = 0 %}{% set c = 0 %} -SELECT 1; -{% endfor %} -""", - None, - [ - ("literal", slice(0, 1, None), slice(0, 1, None)), - ("block_start", slice(1, 24, None), slice(1, 1, None)), - ("templated", slice(24, 39, None), slice(1, 1, None)), - ("templated", slice(39, 54, None), slice(1, 1, None)), - ("templated", slice(54, 69, None), slice(1, 1, None)), - ("literal", slice(69, 80, None), slice(1, 12, None)), - ("block_end", slice(80, 92, None), slice(12, 12, None)), - ("templated", slice(24, 39, None), slice(12, 12, None)), - ("templated", slice(39, 54, None), slice(12, 12, None)), - ("templated", slice(54, 69, None), slice(12, 12, None)), - ("literal", slice(69, 80, None), slice(12, 23, None)), - ("block_end", slice(80, 92, None), slice(23, 23, None)), - ("literal", slice(92, 93, None), slice(23, 24, None)), - ], - JinjaTemplater, - ), - ], -) -def test__templater_jinja_slice_file( - raw_file, override_context, result, templater_class, caplog -): - """Test slice_file.""" - templater = templater_class(override_context=override_context) - _, _, render_func = templater.construct_render_func( - config=FluffConfig.from_path( - "test/fixtures/templater/jinja_slice_template_macros" - ) - ) - - with caplog.at_level(logging.DEBUG, logger="sqlfluff.templater"): - raw_sliced, sliced_file, templated_str = templater.slice_file( - raw_file, render_func=render_func - ) - # Create a TemplatedFile from the results. This runs some useful sanity - # checks. - _ = TemplatedFile(raw_file, "<>", templated_str, sliced_file, raw_sliced) - # Check contiguous on the TEMPLATED VERSION - print(sliced_file) - prev_slice = None - for elem in sliced_file: - print(elem) - if prev_slice: - assert elem[2].start == prev_slice.stop - prev_slice = elem[2] - # Check that all literal segments have a raw slice - for elem in sliced_file: - if elem[0] == "literal": - assert elem[1] is not None - # check result - actual = [ - ( - templated_file_slice.slice_type, - templated_file_slice.source_slice, - templated_file_slice.templated_slice, - ) - for templated_file_slice in sliced_file - ] - assert actual == result - - -def test__templater_jinja_large_file_check(): - """Test large file skipping. - - The check is separately called on each .process() method - so it makes sense to test a few templaters. - """ - # First check we can process the file normally without specific config. - # i.e. check the defaults work and the default is high. - JinjaTemplater().process( - in_str="SELECT 1", - fname="", - config=FluffConfig(overrides={"dialect": "ansi"}), - ) - # Second check setting the value low disables the check - JinjaTemplater().process( - in_str="SELECT 1", - fname="", - config=FluffConfig( - overrides={"dialect": "ansi", "large_file_skip_char_limit": 0} - ), - ) - # Finally check we raise a skip exception when config is set low. - with pytest.raises(SQLFluffSkipFile) as excinfo: - JinjaTemplater().process( - in_str="SELECT 1", - fname="", - config=FluffConfig( - overrides={"dialect": "ansi", "large_file_skip_char_limit": 2}, - ), - ) - - assert "Length of file" in str(excinfo.value) - - -@pytest.mark.parametrize( - "in_str, ignore, expected_violation", - [ - ( - """WITH a AS ({{ b(c=d, e=f) }}) SELECT * FROM final""", - "", - SQLTemplaterError("Undefined jinja template variable: 'b'"), - ), - ("""WITH a AS ({{ b(c=d, e=f) }}) SELECT * FROM final""", "templating", None), - ( - # https://github.com/sqlfluff/sqlfluff/issues/6360 - """{% for tbl in tbl_list %}SELECT a FROM {{ tbl }};{% endfor %}""", - "", - SQLTemplaterError("Undefined jinja template variable: 'tbl_list'"), - ), - ( - """SELECT a FROM {{ tbl['name'] }};""", - "", - SQLTemplaterError("Undefined jinja template variable: 'tbl'"), - ), - ], -) -def test_jinja_undefined_callable(in_str, ignore, expected_violation): - """Test undefined callable returns TemplatedFile and sensible error.""" - templater = JinjaTemplater() - templated_file, violations = templater.process( - in_str=in_str, - fname="test.sql", - config=FluffConfig(overrides={"dialect": "ansi", "ignore": ignore}), - ) - # This was previously failing to process, due to UndefinedRecorder not - # supporting __call__(), also Jinja thinking it was not *safe* to call. - assert templated_file is not None - if expected_violation: - assert len(violations) == 1 - isinstance(violations[0], type(expected_violation)) - assert str(violations[0]) == str(expected_violation) - else: - assert len(violations) == 0 - - -def test_dummy_undefined_fail_with_undefined_error(): - """Tests that a recursion error bug no longer occurs.""" - ud = DummyUndefined("name") - with pytest.raises(UndefinedError): - # This was previously causing a recursion error. - ud._fail_with_undefined_error() - - -def test_undefined_magic_methods(): - """Test all the magic methods defined on DummyUndefined.""" - ud = DummyUndefined("name") - - # _self_impl - assert ud + ud is ud - assert ud - ud is ud - assert ud / ud is ud - assert ud // ud is ud - assert ud % ud is ud - assert ud**ud is ud - assert +ud is ud - assert -ud is ud - assert ud << ud is ud - assert ud[ud] is ud - assert ~ud is ud - assert ud(ud) is ud - - # _bool_impl - assert ud and ud - assert ud or ud - assert ud ^ ud - assert bool(ud) - assert ud < ud - assert ud <= ud - assert ud == ud - assert ud != ud - assert ud >= ud - assert ud > ud - - assert ud + ud is ud + assert str(outstr) == expected_outstr -@pytest.mark.parametrize( - "sql_path, expected_renderings", - [ - pytest.param( - "simple_if_true.sql", - [ - "\nSELECT 1\n\n", - "\nSELECT 2\n\n", - ], - id="simple_if_true", - ), - pytest.param( - "simple_if_false.sql", - [ - "\nSELECT 2\n\n", - "\nSELECT 1\n\n", - ], - id="simple_if_false", - ), - pytest.param( - "if_elif_else.sql", - [ - "\nSELECT 1\n\n", - "\nSELECT 2\n\n", - "\nSELECT 3\n\n", - ], - id="if_elif_else", - ), - pytest.param( - "if_else_if_nested.sql", - [ - "\nSELECT 1\n\n", - "\n\nSELECT 2\n\n\n", - "\n\nSELECT 3\n\n\n", - ], - id="if_else_if_nested", - ), - # This test case exercises the scoring function. Generates up to 10 - # variants, but only the top 5 are returned. - pytest.param( - "if_elif_else_chain_scoring.sql", - [ - "\nSELECT 1\n\n", - "\nSELECT 100000000\n\n", - "\nSELECT 10000000\n\n", - "\nSELECT 1000000\n\n", - "\nSELECT 100000\n\n", - "\nSELECT 10000\n\n", - ], - id="if_elif_else_chain_scoring", - ), - # This test case results in a TypeError executing the variant. This - # should be ignored, and only the primary should be returned. - pytest.param( - "if_true_elif_type_error_else.sql", - [ - "\nSELECT 1\n\n", - "\nSELECT 2\n\n", - ], - id="if_true_elif_type_error_else", - ), - # https://github.com/sqlfluff/sqlfluff/issues/5803 - pytest.param( - "inline_select.sql", - [ - "select 2\n", - "select 1\n", - ], - id="inline_select", - ), - ], -) -def test__templater_lint_unreached_code(sql_path: str, expected_renderings): - """Test that Jinja templater slices raw and templated file correctly.""" - test_dir = Path("test/fixtures/templater/jinja_lint_unreached_code") - t = JinjaTemplater() - renderings = [] - raw_slicings = [] - final_source_slices = [] - for templated_file, _ in t.process_with_variants( - in_str=(test_dir / sql_path).read_text(), - fname=str(sql_path), - config=FluffConfig.from_path(str(test_dir)), - ): - renderings.append(templated_file.templated_str) - raw_slicings.append(templated_file.raw_sliced) - # Capture the final slice for all of them. - final_source_slices.append(templated_file.sliced_file[-1].source_slice) - assert renderings == expected_renderings - # Compare all of the additional raw slicings to make sure they're the - # same as the root. - root_slicing = raw_slicings[0] - for additional_slicing in raw_slicings[1:]: - assert additional_slicing == root_slicing - # Check that the final source slices also line up in the templated files. - # NOTE: Clearly the `templated_slice` values _won't_ be the same. - # We're doing the _final_ slice, because it's very likely to be the same - # _type_ and if it's in the right place, we can assume that all of the - # others probably are. - root_final_slice = final_source_slices[0] - for additional_final_slice in final_source_slices[1:]: - assert additional_final_slice == root_final_slice +# class RawTemplatedTestCase(NamedTuple): +# """Instances of this object are test cases for test__templater_jinja_slices.""" +# +# name: str +# instr: str +# templated_str: str +# +# # These fields are used to check TemplatedFile.sliced_file. +# expected_templated_sliced__source_list: List[str] +# expected_templated_sliced__templated_list: List[str] +# +# # This field is used to check TemplatedFile.raw_sliced. +# expected_raw_sliced__source_list: List[str] +# +# +# @pytest.mark.parametrize( +# "case", +# [ +# RawTemplatedTestCase( +# name="basic_block", +# instr="\n\n{% set x = 42 %}\nSELECT 1, 2\n", +# templated_str="\n\n\nSELECT 1, 2\n", +# expected_templated_sliced__source_list=[ +# "\n\n", +# "{% set x = 42 %}", +# "\nSELECT 1, 2\n", +# ], +# expected_templated_sliced__templated_list=[ +# "\n\n", +# "", +# "\nSELECT 1, 2\n", +# ], +# expected_raw_sliced__source_list=[ +# "\n\n", +# "{% set x = 42 %}", +# "\nSELECT 1, 2\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_left_block", +# instr="\n\n{%- set x = 42 %}\nSELECT 1, 2\n", +# templated_str="\nSELECT 1, 2\n", +# expected_templated_sliced__source_list=[ +# "\n\n", +# "{%- set x = 42 %}", +# "\nSELECT 1, 2\n", +# ], +# expected_templated_sliced__templated_list=[ +# "", +# "", +# "\nSELECT 1, 2\n", +# ], +# expected_raw_sliced__source_list=[ +# "\n\n", +# "{%- set x = 42 %}", +# "\nSELECT 1, 2\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_both_block", +# instr="\n\n{%- set x = 42 -%}\nSELECT 1, 2\n", +# templated_str="SELECT 1, 2\n", +# expected_templated_sliced__source_list=[ +# "\n\n", +# "{%- set x = 42 -%}", +# "\n", +# "SELECT 1, 2\n", +# ], +# expected_templated_sliced__templated_list=[ +# "", +# "", +# "", +# "SELECT 1, 2\n", +# ], +# expected_raw_sliced__source_list=[ +# "\n\n", +# "{%- set x = 42 -%}", +# "\n", +# "SELECT 1, 2\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_and_templated_whitespace", +# instr="SELECT {{- ' ' -}} 1{{ ' , 2' -}}\n", +# templated_str="SELECT 1 , 2", +# expected_templated_sliced__source_list=[ +# "SELECT", +# " ", +# "{{- ' ' -}}", +# " ", +# "1", +# "{{ ' , 2' -}}", +# "\n", +# ], +# expected_templated_sliced__templated_list=[ +# "SELECT", +# "", # Placeholder for consumed whitespace +# " ", # Placeholder for templated whitespace +# "", # Placeholder for consumed whitespace +# "1", +# " , 2", +# "", # Placeholder for consumed newline +# ], +# expected_raw_sliced__source_list=[ +# "SELECT", +# " ", +# "{{- ' ' -}}", +# " ", +# "1", +# "{{ ' , 2' -}}", +# "\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_both_block_hard", +# instr="SELECT {%- set x = 42 %} 1 {%- if true -%} , 2{% endif -%}\n", +# templated_str="SELECT 1, 2", +# expected_templated_sliced__source_list=[ +# "SELECT", +# # NB: Even though the jinja tag consumes whitespace, we still +# # get it here as a placeholder. +# " ", +# "{%- set x = 42 %}", +# " 1", +# # This whitespace is a separate from the 1 because it's consumed. +# " ", +# "{%- if true -%}", +# " ", +# ", 2", +# "{% endif -%}", +# "\n", +# ], +# expected_templated_sliced__templated_list=[ +# "SELECT", +# "", # Consumed whitespace placeholder +# "", # Jinja block placeholder +# " 1", +# "", # Consumed whitespace +# "", # Jinja block placeholder +# "", # More consumed whitespace +# ", 2", +# "", # Jinja block +# "", # Consumed final newline. +# ], +# expected_raw_sliced__source_list=[ +# "SELECT", +# " ", +# "{%- set x = 42 %}", +# " 1", +# " ", +# "{%- if true -%}", +# " ", +# ", 2", +# "{% endif -%}", +# "\n", +# ], +# ), +# RawTemplatedTestCase( +# name="basic_data", +# instr="""select +# c1, +# {{ 'c' }}2 as user_id +# """, +# templated_str="""select +# c1, +# c2 as user_id +# """, +# expected_templated_sliced__source_list=[ +# "select\n c1,\n ", +# "{{ 'c' }}", +# "2 as user_id\n", +# ], +# expected_templated_sliced__templated_list=[ +# "select\n c1,\n ", +# "c", +# "2 as user_id\n", +# ], +# expected_raw_sliced__source_list=[ +# "select\n c1,\n ", +# "{{ 'c' }}", +# "2 as user_id\n", +# ], +# ), +# # Note this is basically identical to the "basic_data" case above. +# # "Right strip" is not actually a thing in Jinja. +# RawTemplatedTestCase( +# name="strip_right_data", +# instr="""SELECT +# {{ 'col1,' -}} +# col2 +# """, +# templated_str="""SELECT +# col1,col2 +# """, +# expected_templated_sliced__source_list=[ +# "SELECT\n ", +# "{{ 'col1,' -}}", +# "\n ", +# "col2\n", +# ], +# expected_templated_sliced__templated_list=[ +# "SELECT\n ", +# "col1,", +# "", +# "col2\n", +# ], +# expected_raw_sliced__source_list=[ +# "SELECT\n ", +# "{{ 'col1,' -}}", +# "\n ", +# "col2\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_both_data", +# instr="""select +# c1, +# {{- 'c' -}} +# 2 as user_id +# """, +# templated_str="""select +# c1,c2 as user_id +# """, +# expected_templated_sliced__source_list=[ +# "select\n c1,", +# "\n ", +# "{{- 'c' -}}", +# "\n", +# "2 as user_id\n", +# ], +# expected_templated_sliced__templated_list=[ +# "select\n c1,", +# "", +# "c", +# "", +# "2 as user_id\n", +# ], +# expected_raw_sliced__source_list=[ +# "select\n c1,", +# "\n ", +# "{{- 'c' -}}", +# "\n", +# "2 as user_id\n", +# ], +# ), +# RawTemplatedTestCase( +# name="strip_both_comment", +# instr="""select +# c1, +# {#- Column 2 -#} c2 as user_id +# """, +# templated_str="""select +# c1,c2 as user_id +# """, +# expected_templated_sliced__source_list=[ +# "select\n c1,", +# "\n ", +# "{#- Column 2 -#}", +# " ", +# "c2 as user_id\n", +# ], +# expected_templated_sliced__templated_list=[ +# "select\n c1,", +# "", +# "", +# "", +# "c2 as user_id\n", +# ], +# expected_raw_sliced__source_list=[ +# "select\n c1,", +# "\n ", +# "{#- Column 2 -#}", +# " ", +# "c2 as user_id\n", +# ], +# ), +# RawTemplatedTestCase( +# name="union_all_loop1", +# instr="""{% set products = [ +# 'table1', +# 'table2', +# ] %} +# +# {% for product in products %} +# SELECT +# brand +# FROM +# {{ product }} +# {% if not loop.last -%} UNION ALL {%- endif %} +# {% endfor %} +# """, +# templated_str=( +# "\n\n\nSELECT\n brand\nFROM\n table1\nUNION ALL\n\nSELECT\n " +# "brand\nFROM\n table2\n\n\n" +# ), +# expected_templated_sliced__source_list=[ +# "{% set products = [\n 'table1',\n 'table2',\n ] %}", +# "\n\n", +# "{% for product in products %}", +# "\nSELECT\n brand\nFROM\n ", +# "{{ product }}", +# "\n", +# "{% if not loop.last -%}", +# " ", +# "UNION ALL", +# " ", +# "{%- endif %}", +# "\n", +# "{% endfor %}", +# "\nSELECT\n brand\nFROM\n ", +# "{{ product }}", +# "\n", +# "{% if not loop.last -%}", +# "{%- endif %}", +# "\n", +# "{% endfor %}", +# "\n", +# ], +# expected_templated_sliced__templated_list=[ +# "", +# "\n\n", +# "", +# "\nSELECT\n brand\nFROM\n ", +# "table1", +# "\n", +# "", +# "", +# "UNION ALL", +# "", +# "", +# "\n", +# "", +# "\nSELECT\n brand\nFROM\n ", +# "table2", +# "\n", +# "", +# "", +# "\n", +# "", +# "\n", +# ], +# expected_raw_sliced__source_list=[ +# "{% set products = [\n 'table1',\n 'table2',\n ] %}", +# "\n\n", +# "{% for product in products %}", +# "\nSELECT\n brand\nFROM\n ", +# "{{ product }}", +# "\n", +# "{% if not loop.last -%}", +# " ", +# "UNION ALL", +# " ", +# "{%- endif %}", +# "\n", +# "{% endfor %}", +# "\n", +# ], +# ), +# RawTemplatedTestCase( +# "set_multiple_variables_and_define_macro", +# """{% macro echo(text) %} +# {{text}} +# {% endmacro %} +# +# {% set a, b = 1, 2 %} +# +# SELECT +# {{ echo(a) }}, +# {{ echo(b) }}""", +# "\n\n\n\nSELECT\n \n1\n,\n \n2\n", +# [ +# "{% macro echo(text) %}", +# "\n", +# "{{text}}", +# "\n", +# "{% endmacro %}", +# "\n\n", +# "{% set a, b = 1, 2 %}", +# "\n\nSELECT\n ", +# "{{ echo(a) }}", +# ",\n ", +# "{{ echo(b) }}", +# ], +# [ +# "", +# "", +# "", +# "", +# "", +# "\n\n", +# "", +# "\n\nSELECT\n ", +# "\n1\n", +# ",\n ", +# "\n2\n", +# ], +# [ +# "{% macro echo(text) %}", +# "\n", +# "{{text}}", +# "\n", +# "{% endmacro %}", +# "\n\n", +# "{% set a, b = 1, 2 %}", +# "\n\nSELECT\n ", +# "{{ echo(a) }}", +# ",\n ", +# "{{ echo(b) }}", +# ], +# ), +# ], +# ids=lambda case: case.name, +# ) +# def test__templater_jinja_slices(case: RawTemplatedTestCase): +# """Test that Jinja templater slices raw and templated file correctly.""" +# t = JinjaTemplater() +# templated_file, _ = t.process( +# in_str=case.instr, +# fname="test", +# config=FluffConfig(overrides={"dialect": "ansi"}), +# ) +# assert templated_file is not None +# assert templated_file.source_str == case.instr +# assert templated_file.templated_str == case.templated_str +# # Build and check the list of source strings referenced by "sliced_file". +# actual_ts_source_list = [ +# case.instr[ts.source_slice] for ts in templated_file.sliced_file +# ] +# assert actual_ts_source_list == case.expected_templated_sliced__source_list +# +# # Build and check the list of templated strings referenced by "sliced_file". +# actual_ts_templated_list = [ +# templated_file.templated_str[ts.templated_slice] +# for ts in templated_file.sliced_file +# ] +# assert actual_ts_templated_list == case.expected_templated_sliced__templated_list +# +# # Build and check the list of source strings referenced by "raw_sliced". +# previous_rs = None +# actual_rs_source_list: List[RawFileSlice] = [] +# for rs in templated_file.raw_sliced + [None]: # type: ignore +# if previous_rs: +# if rs: +# actual_source = case.instr[previous_rs.source_idx : rs.source_idx] +# else: +# actual_source = case.instr[previous_rs.source_idx :] +# actual_rs_source_list.append(actual_source) +# previous_rs = rs +# assert actual_rs_source_list == case.expected_raw_sliced__source_list +# +# +# def test_templater_set_block_handling(): +# """Test handling of literals in {% set %} blocks. +# +# Specifically, verify they are not modified in the alternate template. +# """ +# +# def run_query(sql): +# # Prior to the bug fix, this assertion failed. This was bad because, +# # inside JinjaTracer, dbt templates similar to the one in this test +# # would call the database with funky SQL (including weird strings it +# # uses internally like: 00000000000000000000000000000002. +# assert sql == "\n\nselect 1 from foobarfoobarfoobarfoobar_dev\n\n" +# return sql +# +# t = JinjaTemplater(override_context=dict(run_query=run_query)) +# instr = """{% set my_query1 %} +# select 1 from foobarfoobarfoobarfoobar_{{ "dev" }} +# {% endset %} +# {% set my_query2 %} +# {{ my_query1 }} +# {% endset %} +# +# {{ run_query(my_query2) }} +# """ +# outstr, vs = t.process( +# in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) +# ) +# assert str(outstr) == "\n\n\n\n\nselect 1 from foobarfoobarfoobarfoobar_dev\n\n\n" +# assert len(vs) == 0 +# +# +# def test__templater_jinja_error_variable(): +# """Test missing variable error handling in the jinja templater.""" +# t = JinjaTemplater(override_context=dict(blah="foo")) +# instr = JINJA_STRING +# outstr, vs = t.process( +# in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) +# ) +# assert str(outstr) == "SELECT * FROM f, o, o WHERE \n\n" +# # Check we have violations. +# assert len(vs) > 0 +# # Check one of them is a templating error on line 1 +# assert any(v.rule_code() == "TMP" and v.line_no == 1 for v in vs) +# +# +# def test__templater_jinja_dynamic_variable_no_violations(): +# """Test no templater violation for variable defined within template.""" +# t = JinjaTemplater(override_context=dict(blah="foo")) +# instr = """{% if True %} +# {% set some_var %}1{% endset %} +# SELECT {{some_var}} +# {% endif %} +# """ +# outstr, vs = t.process( +# in_str=instr, fname="test", config=FluffConfig(overrides={"dialect": "ansi"}) +# ) +# assert str(outstr) == "\n \n SELECT 1\n\n" +# # Check we have no violations. +# assert len(vs) == 0 +# +# +# def test__templater_jinja_error_syntax(): +# """Test syntax problems in the jinja templater.""" +# t = JinjaTemplater() +# instr = "SELECT {{foo} FROM jinja_error\n" +# with pytest.raises(SQLTemplaterError) as excinfo: +# t.process( +# in_str=instr, +# fname="test", +# config=FluffConfig(overrides={"dialect": "ansi"}), +# ) +# templater_exception = excinfo.value +# assert templater_exception.rule_code() == "TMP" +# assert templater_exception.line_no == 1 +# assert "Failed to parse Jinja syntax" in str(templater_exception) +# +# +# def test__templater_jinja_error_catastrophic(): +# """Test error handling in the jinja templater.""" +# t = JinjaTemplater(override_context=dict(blah=7)) +# instr = JINJA_STRING +# with pytest.raises(SQLTemplaterError) as excinfo: +# t.process( +# in_str=instr, +# fname="test", +# config=FluffConfig(overrides={"dialect": "ansi"}), +# ) +# templater_exception = excinfo.value +# assert templater_exception.rule_code() == "TMP" +# assert templater_exception.line_no == 1 +# assert "Unrecoverable failure in Jinja templating" in str(templater_exception) +# +# +# def test__templater_jinja_error_macro_path_does_not_exist(): +# """Tests that an error is raised if macro path doesn't exist.""" +# with pytest.raises(ValueError) as e: +# JinjaTemplater().construct_render_func( +# config=FluffConfig.from_path( +# "test/fixtures/templater/jinja_macro_path_does_not_exist" +# ) +# ) +# assert str(e.value).startswith("Path does not exist") +# +# +# def test__templater_jinja_error_macro_invalid(): +# """Tests that an error is raised if a macro is invalid.""" +# invalid_macro_config_string = ( +# "[sqlfluff]\n" +# "templater = jinja\n" +# "dialect = ansi\n" +# "[sqlfluff:templater:jinja:macros]\n" +# "a_macro_def = {% macro pkg.my_macro() %}pass{% endmacro %}\n" +# ) +# config = FluffConfig.from_string(invalid_macro_config_string) +# with pytest.raises(SQLFluffUserError) as e: +# JinjaTemplater().construct_render_func(config=config) +# error_string = str(e.value) +# assert error_string.startswith("Error loading user provided macro") +# assert "{% macro pkg.my_macro() %}pass{% endmacro %}" in error_string +# +# +# def test__templater_jinja_lint_empty(): +# """Check that parsing a file which renders to an empty string. +# +# No exception should be raised, and we should get a single templated element. +# """ +# lntr = Linter(dialect="ansi") +# parsed = lntr.parse_string(in_str='{{ "" }}') +# parsed_variant = parsed.parsed_variants[0] +# assert parsed_variant.templated_file.source_str == '{{ "" }}' +# assert parsed_variant.templated_file.templated_str == "" +# # Get the types of the segments +# print(f"Segments: {parsed_variant.tree.raw_segments}") +# seg_types = [seg.get_type() for seg in parsed_variant.tree.raw_segments] +# assert seg_types == ["placeholder", "end_of_file"] +# +# +# def assert_structure(yaml_loader, path, code_only=True, include_meta=False): +# """Check that a parsed sql file matches the yaml file with the same name.""" +# parsed = get_parsed(path + ".sql") +# # Whitespace is important here to test how that's treated +# tpl = parsed.to_tuple(code_only=code_only, show_raw=True, include_meta=include_meta) +# # Check nothing unparsable +# if "unparsable" in parsed.type_set(): +# print(parsed.stringify()) +# raise ValueError("Input file is unparsable.") +# _, expected = yaml_loader(path + ".yml") +# assert tpl == expected +# +# +# @pytest.mark.parametrize( +# "subpath,code_only,include_meta", +# [ +# # Config Scalar +# ("jinja_a/jinja", True, False), +# # Macros +# ("jinja_b/jinja", False, False), +# # dbt builtins +# ("jinja_c_dbt/dbt_builtins_cross_ref", True, False), +# ("jinja_c_dbt/dbt_builtins_config", True, False), +# ("jinja_c_dbt/dbt_builtins_is_incremental", True, False), +# ("jinja_c_dbt/dbt_builtins_ref", True, False), +# ("jinja_c_dbt/dbt_builtins_source", True, False), +# ("jinja_c_dbt/dbt_builtins_this", True, False), +# ("jinja_c_dbt/dbt_builtins_this_callable", True, False), +# ("jinja_c_dbt/dbt_builtins_var_default", True, False), +# ("jinja_c_dbt/dbt_builtins_test", True, False), +# # do directive +# ("jinja_e/jinja", True, False), +# # case sensitivity and python literals +# ("jinja_f/jinja", True, False), +# # Macro loading from a folder +# ("jinja_g_macros/jinja", True, False), +# # Excluding macros +# ("jinja_exclude_macro_path/jinja", True, False), +# # Excluding macros with running from subdirectory +# ("jinja_exclude_macro_path/model_directory/jinja_sub_directory", True, False), +# # jinja raw tag +# ("jinja_h_macros/jinja", True, False), +# ("jinja_i_raw/raw_tag", True, False), +# ("jinja_i_raw/raw_tag_2", True, False), +# # Library Loading from a folder +# ("jinja_j_libraries/jinja", True, False), +# # Priority of macros +# ("jinja_k_config_override_path_macros/jinja", True, False), +# # Placeholders and metas +# ("jinja_l_metas/001", False, True), +# ("jinja_l_metas/002", False, True), +# ("jinja_l_metas/003", False, True), +# ("jinja_l_metas/004", False, True), +# ("jinja_l_metas/005", False, True), +# ("jinja_l_metas/006", False, True), +# ("jinja_l_metas/007", False, True), +# ("jinja_l_metas/008", False, True), +# ("jinja_l_metas/009", False, True), +# ("jinja_l_metas/010", False, True), +# ("jinja_l_metas/011", False, True), +# # Library Loading from a folder when library is module +# ("jinja_m_libraries_module/jinja", True, False), +# ("jinja_n_nested_macros/jinja", True, False), +# # Test more dbt configurations +# ("jinja_o_config_override_dbt_builtins/override_dbt_builtins", True, False), +# ("jinja_p_disable_dbt_builtins/disable_dbt_builtins", True, False), +# # Load all the macros +# ("jinja_q_multiple_path_macros/jinja", True, False), +# ("jinja_s_filters_in_library/jinja", True, False), +# # Jinja loader search path, without also loading macros into global namespace +# ("jinja_t_loader_search_path/jinja", True, False), +# ], +# ) +# def test__templater_full(subpath, code_only, include_meta, yaml_loader, caplog): +# """Check structure can be parsed from jinja templated files.""" +# # Log the templater and lexer throughout this test +# caplog.set_level(logging.DEBUG, logger="sqlfluff.templater") +# caplog.set_level(logging.DEBUG, logger="sqlfluff.lexer") +# +# assert_structure( +# yaml_loader, +# "test/fixtures/templater/" + subpath, +# code_only=code_only, +# include_meta=include_meta, +# ) +# +# +# def test__templater_jinja_block_matching(caplog): +# """Test the block UUID matching works with a complicated case.""" +# caplog.set_level(logging.DEBUG, logger="sqlfluff.lexer") +# path = "test/fixtures/templater/jinja_l_metas/002.sql" +# # Parse the file. +# parsed = get_parsed(path) +# # We only care about the template elements +# template_segments = [ +# seg +# for seg in parsed.raw_segments +# if seg.is_type("template_loop") +# or ( +# seg.is_type("placeholder") +# and seg.block_type in ("block_start", "block_end", "block_mid") +# ) +# ] +# +# # Group them together by block UUID +# assert all(seg.block_uuid for seg in template_segments), ( +# "All templated segments should have a block uuid!" +# ) +# grouped = defaultdict(list) +# for seg in template_segments: +# grouped[seg.block_uuid].append(seg.pos_marker.working_loc) +# +# print(grouped) +# +# # Now the matching block IDs should be found at the following positions. +# # NOTE: These are working locations in the rendered file. +# groups = { +# "for actions clause 1": [(6, 5), (9, 5), (12, 5), (15, 5)], +# "for actions clause 2": [(17, 5), (21, 5), (29, 5), (37, 5)], +# # NOTE: all the if loop clauses are grouped together. +# "if loop.first": [ +# (18, 9), +# (20, 9), +# (20, 9), +# (22, 9), +# (22, 9), +# (28, 9), +# (30, 9), +# (30, 9), +# (36, 9), +# ], +# } +# +# # Check all are accounted for: +# for clause in groups.keys(): +# for block_uuid, locations in grouped.items(): +# if groups[clause] == locations: +# print(f"Found {clause}, locations with UUID: {block_uuid}") +# break +# else: +# raise ValueError(f"Couldn't find appropriate grouping of blocks: {clause}") +# +# +# class DerivedJinjaAnalyzer(JinjaAnalyzer): +# """An analyzer that includes some custom Jinja tags. +# +# This is used for tests that show the analyzer can be extended for custom plugin +# templaters that support custom tags. +# """ +# +# @classmethod +# def _get_tag_configuration(cls, tag: str) -> JinjaTagConfiguration: +# tag_map = { +# "up": JinjaTagConfiguration( +# block_type="block_start", +# block_tracking=True, +# ), +# "down": JinjaTagConfiguration( +# block_type="block_mid", +# block_tracking=True, +# ), +# "end": JinjaTagConfiguration( +# block_type="block_end", +# block_tracking=True, +# ), +# } +# return tag_map.get(tag, super()._get_tag_configuration(tag)) +# +# +# @pytest.mark.parametrize( +# "test,result,analyzer_class", +# [ +# ("", [], JinjaAnalyzer), +# ("foo", [("foo", "literal", 0)], JinjaAnalyzer), +# ( +# "foo {{bar}} z ", +# [ +# ("foo ", "literal", 0), +# ("{{bar}}", "templated", 4), +# (" z ", "literal", 11), +# ], +# JinjaAnalyzer, +# ), +# ( +# ( +# "SELECT {# A comment #} {{field}} {% for i in [1, 3]%}, " +# "fld_{{i}}{% endfor %} FROM my_schema.{{my_table}} " +# ), +# [ +# ("SELECT ", "literal", 0), +# ("{# A comment #}", "comment", 7), +# (" ", "literal", 22), +# ("{{field}}", "templated", 23), +# (" ", "literal", 32), +# ("{% for i in [1, 3]%}", "block_start", 33, 1, "for"), +# (", fld_", "literal", 53, 1), +# ("{{i}}", "templated", 59, 1), +# ("{% endfor %}", "block_end", 64, 1, "endfor"), +# (" FROM my_schema.", "literal", 76, 2), +# ("{{my_table}}", "templated", 92, 2), +# (" ", "literal", 104, 2), +# ], +# JinjaAnalyzer, +# ), +# ( +# "{% set thing %}FOO{% endset %} BAR", +# [ +# ("{% set thing %}", "block_start", 0, 1, "set"), +# ("FOO", "literal", 15, 1), +# ("{% endset %}", "block_end", 18, 1, "endset"), +# (" BAR", "literal", 30, 2), +# ], +# JinjaAnalyzer, +# ), +# ( +# # Tests Jinja "block assignment" syntax. Also tests the use of +# # template substitution within the block: {{ "dev" }}. +# """{% set my_query %} +# select 1 from foobarfoobarfoobarfoobar_{{ "dev" }} +# {% endset %} +# {{ my_query }} +# """, +# [ +# ("{% set my_query %}", "block_start", 0, 1, "set"), +# ("\nselect 1 from foobarfoobarfoobarfoobar_", "literal", 18, 1), +# ('{{ "dev" }}', "templated", 58, 1), +# ("\n", "literal", 69, 1), +# ("{% endset %}", "block_end", 70, 1, "endset"), +# ("\n", "literal", 82, 2), +# ("{{ my_query }}", "templated", 83, 2), +# ("\n", "literal", 97, 2), +# ], +# JinjaAnalyzer, +# ), +# # Tests for jinja blocks that consume whitespace. +# ( +# """SELECT 1 FROM {%+if true-%} {{ref('foo')}} {%-endif%}""", +# [ +# ("SELECT 1 FROM ", "literal", 0), +# ("{%+if true-%}", "block_start", 14, 1, "if"), +# (" ", "literal", 27, 1), +# ("{{ref('foo')}}", "templated", 28, 1), +# (" ", "literal", 42, 1), +# ("{%-endif%}", "block_end", 43, 1, "endif"), +# ], +# JinjaAnalyzer, +# ), +# ( +# """{% for item in some_list -%} +# SELECT * +# FROM some_table +# {{ "UNION ALL\n" if not loop.last }} +# {%- endfor %}""", +# [ +# ("{% for item in some_list -%}", "block_start", 0, 1, "for"), +# # This gets consumed in the templated file, but it's still here. +# ("\n ", "literal", 28, 1), +# ("SELECT *\n FROM some_table\n", "literal", 33, 1), +# ('{{ "UNION ALL\n" if not loop.last }}', "templated", 62, 1), +# ("\n", "literal", 97, 1), +# ("{%- endfor %}", "block_end", 98, 1, "endfor"), +# ], +# JinjaAnalyzer, +# ), +# ( +# JINJA_MACRO_CALL_SQL, +# [ +# ("{% macro render_name(title) %}", "block_start", 0, 1, "macro"), +# ("\n '", "literal", 30, 1), +# ("{{ title }}", "templated", 34, 1), +# (". foo' as ", "literal", 45, 1), +# ("{{ caller() }}", "templated", 55, 1), +# ("\n", "literal", 69, 1), +# ("{% endmacro %}", "block_end", 70, 1, "endmacro"), +# ("\nSELECT\n ", "literal", 84, 2), +# ("{% call render_name('Sir') %}", "block_start", 96, 3, "call"), +# ("\n bar\n ", "literal", 125, 3), +# ("{% endcall %}", "block_end", 142, 3, "endcall"), +# ("\nFROM baz\n", "literal", 155, 4), +# ], +# JinjaAnalyzer, +# ), +# ( +# # Test of tag heuristics in the default _get_tag_configuration +# """{% randomtagstart %} +# SELECT 1; +# {% elphony %} +# SELECT 2; +# {% endsomethingweird %}""", +# [ +# ("{% randomtagstart %}", "block_start", 0, 1, "randomtagstart"), +# ("\n SELECT 1;\n", "literal", 20, 1), +# ("{% elphony %}", "block_mid", 35, 1, "elphony"), +# ("\n SELECT 2;\n", "literal", 48, 1), +# ("{% endsomethingweird %}", "block_end", 63, 1, "endsomethingweird"), +# ], +# JinjaAnalyzer, +# ), +# ( +# # Basic test with a derived JinjaAnalyzer that supports some custom tags +# """{% up 'create table xyz' %} +# CREATE TABLE xyz (id int); +# {% down %} +# DROP TABLE xyz; +# {% end %}""", +# [ +# ("{% up 'create table xyz' %}", "block_start", 0, 1, "up"), +# ("\n CREATE TABLE xyz (id int);\n", "literal", 27, 1), +# ("{% down %}", "block_mid", 59, 1, "down"), +# ("\n DROP TABLE xyz;\n", "literal", 69, 1), +# ("{% end %}", "block_end", 90, 1, "end"), +# ], +# DerivedJinjaAnalyzer, +# ), +# ], +# ) +# def test__templater_jinja_slice_template(test, result, analyzer_class): +# """Test _slice_template.""" +# templater = JinjaTemplater() +# env, _, render_func = templater.construct_render_func() +# +# analyzer = analyzer_class(test, env) +# analyzer.analyze(render_func=render_func) +# resp = analyzer.raw_sliced +# # check contiguous (unless there's a comment in it) +# if "{#" not in test: +# assert "".join(elem.raw for elem in resp) == test +# # check indices +# idx = 0 +# for raw_slice in resp: +# assert raw_slice.source_idx == idx +# idx += len(raw_slice.raw) +# # Check total result +# assert resp == [RawFileSlice(*args) for args in result] +# +# +# class DBMigrationExtension(Extension): +# """Example of a hypothetical custom Jinja extension. +# +# This extension might ostensibly be used to represent up/down database migrations. +# """ +# +# tags = {"up"} +# +# def parse(self, parser: Parser) -> Union[Node, List[Node]]: +# """Parse the up/down blocks.""" +# # {% up 'migration name' %} +# next(parser.stream) # skip the "up" token +# parser.parse_expression() # skip the name of this migration +# up_body = parser.parse_statements(("name:down",)) +# # {% down %} +# next(parser.stream) # skip the "down" token +# down_body = parser.parse_statements(("name:end",)) +# # {% end %} +# next(parser.stream) +# +# # This is just a test, so output the blocks verbatim one after the other: +# return [nodes.Scope(up_body), nodes.Scope(down_body)] +# +# +# class DerivedJinjaTemplater(JinjaTemplater): +# """A templater that includes some custom Jinja tags. +# +# This is used for tests that show the templater can be extended for custom plugin +# templaters that support custom tags. +# """ +# +# name = "derivedtemplater" +# +# def _get_jinja_env(self, config=None): +# env = super()._get_jinja_env(config) +# env.add_extension(DBMigrationExtension) +# return env +# +# def _get_jinja_analyzer(self, raw_str: str, env: Environment) -> JinjaAnalyzer: +# return DerivedJinjaAnalyzer(raw_str, env) +# +# +# def _statement(*args, **kwargs): +# # NOTE: The standard dbt statement() call returns nothing. +# return "" +# +# +# def _load_result(*args, **kwargs): +# return "_load_result" +# +# +# @pytest.mark.parametrize( +# "raw_file,override_context,result,templater_class", +# [ +# ("", None, [], JinjaTemplater), +# ( +# "foo", +# None, +# [("literal", slice(0, 3, None), slice(0, 3, None))], +# JinjaTemplater, +# ), +# # Example with no loops +# ( +# "SELECT {{blah}}, boo {# comment #} from something", +# dict(blah="foobar"), +# [ +# ("literal", slice(0, 7, None), slice(0, 7, None)), +# ("templated", slice(7, 15, None), slice(7, 13, None)), +# ("literal", slice(15, 21, None), slice(13, 19, None)), +# ("comment", slice(21, 34, None), slice(19, 19, None)), +# ("literal", slice(34, 49, None), slice(19, 34, None)), +# ], +# JinjaTemplater, +# ), +# # Example with loops +# ( +# ( +# "SELECT {# A comment #} {{field}} {% for i in [1, 3, 7]%}, " +# "fld_{{i}}_x{% endfor %} FROM my_schema.{{my_table}} " +# ), +# dict(field="foobar", my_table="barfoo"), +# [ +# ("literal", slice(0, 7, None), slice(0, 7, None)), +# ("comment", slice(7, 22, None), slice(7, 7, None)), +# ("literal", slice(22, 23, None), slice(7, 8, None)), +# ("templated", slice(23, 32, None), slice(8, 14, None)), +# ("literal", slice(32, 33, None), slice(14, 15, None)), +# ("block_start", slice(33, 56, None), slice(15, 15, None)), +# ("literal", slice(56, 62, None), slice(15, 21, None)), +# ("templated", slice(62, 67, None), slice(21, 22, None)), +# ("literal", slice(67, 69, None), slice(22, 24, None)), +# ("block_end", slice(69, 81, None), slice(24, 24, None)), +# ("literal", slice(56, 62, None), slice(24, 30, None)), +# ("templated", slice(62, 67, None), slice(30, 31, None)), +# ("literal", slice(67, 69, None), slice(31, 33, None)), +# ("block_end", slice(69, 81, None), slice(33, 33, None)), +# ("literal", slice(56, 62, None), slice(33, 39, None)), +# ("templated", slice(62, 67, None), slice(39, 40, None)), +# ("literal", slice(67, 69, None), slice(40, 42, None)), +# ("block_end", slice(69, 81, None), slice(42, 42, None)), +# ("literal", slice(81, 97, None), slice(42, 58, None)), +# ("templated", slice(97, 109, None), slice(58, 64, None)), +# ("literal", slice(109, 110, None), slice(64, 65, None)), +# ], +# JinjaTemplater, +# ), +# # Example with loops (and utilising the end slice code) +# ( +# ( +# "SELECT {# A comment #} {{field}} {% for i in [1, 3, 7]%}, " +# "fld_{{i}}{% endfor %} FROM my_schema.{{my_table}} " +# ), +# dict(field="foobar", my_table="barfoo"), +# [ +# ("literal", slice(0, 7, None), slice(0, 7, None)), +# ("comment", slice(7, 22, None), slice(7, 7, None)), +# ("literal", slice(22, 23, None), slice(7, 8, None)), +# ("templated", slice(23, 32, None), slice(8, 14, None)), +# ("literal", slice(32, 33, None), slice(14, 15, None)), +# ("block_start", slice(33, 56, None), slice(15, 15, None)), +# ("literal", slice(56, 62, None), slice(15, 21, None)), +# ("templated", slice(62, 67, None), slice(21, 22, None)), +# ("block_end", slice(67, 79, None), slice(22, 22, None)), +# ("literal", slice(56, 62, None), slice(22, 28, None)), +# ("templated", slice(62, 67, None), slice(28, 29, None)), +# ("block_end", slice(67, 79, None), slice(29, 29, None)), +# ("literal", slice(56, 62, None), slice(29, 35, None)), +# ("templated", slice(62, 67, None), slice(35, 36, None)), +# ("block_end", slice(67, 79, None), slice(36, 36, None)), +# ("literal", slice(79, 95, None), slice(36, 52, None)), +# ("templated", slice(95, 107, None), slice(52, 58, None)), +# ("literal", slice(107, 108, None), slice(58, 59, None)), +# ], +# JinjaTemplater, +# ), +# # Test a trailing split, and some variables which don't refer anything. +# ( +# "{{ config(materialized='view') }}\n\nSELECT 1 FROM {{ source('finance', " +# "'reconciled_cash_facts') }}\n\n", +# dict( +# config=lambda *args, **kwargs: "", +# source=lambda *args, **kwargs: "finance_reconciled_cash_facts", +# ), +# [ +# ("templated", slice(0, 33, None), slice(0, 0, None)), +# ("literal", slice(33, 49, None), slice(0, 16, None)), +# ("templated", slice(49, 97, None), slice(16, 45, None)), +# ("literal", slice(97, 99, None), slice(45, 47, None)), +# ], +# JinjaTemplater, +# ), +# # Test splitting with a loop. +# ( +# "SELECT\n " +# "{% for i in [1, 2, 3] %}\n , " +# "c_{{i}}+42 AS the_meaning_of_li{{ 'f' * i }}\n " +# "{% endfor %}\n" +# "FROM my_table", +# None, +# [ +# ("literal", slice(0, 11, None), slice(0, 11, None)), +# ("block_start", slice(11, 35, None), slice(11, 11, None)), +# ("literal", slice(35, 48, None), slice(11, 24, None)), +# ("templated", slice(48, 53, None), slice(24, 25, None)), +# ("literal", slice(53, 77, None), slice(25, 49, None)), +# ("templated", slice(77, 90, None), slice(49, 50, None)), +# ("literal", slice(90, 95, None), slice(50, 55, None)), +# ("block_end", slice(95, 107, None), slice(55, 55, None)), +# ("literal", slice(35, 48, None), slice(55, 68, None)), +# ("templated", slice(48, 53, None), slice(68, 69, None)), +# ("literal", slice(53, 77, None), slice(69, 93, None)), +# ("templated", slice(77, 90, None), slice(93, 95, None)), +# ("literal", slice(90, 95, None), slice(95, 100, None)), +# ("block_end", slice(95, 107, None), slice(100, 100, None)), +# ("literal", slice(35, 48, None), slice(100, 113, None)), +# ("templated", slice(48, 53, None), slice(113, 114, None)), +# ("literal", slice(53, 77, None), slice(114, 138, None)), +# ("templated", slice(77, 90, None), slice(138, 141, None)), +# ("literal", slice(90, 95, None), slice(141, 146, None)), +# ("block_end", slice(95, 107, None), slice(146, 146, None)), +# ("literal", slice(107, 121, None), slice(146, 160, None)), +# ], +# JinjaTemplater, +# ), +# # Test an example where a block is removed entirely. +# ( +# "{% set thing %}FOO{% endset %} SELECT 1", +# None, +# [ +# ("block_start", slice(0, 15, None), slice(0, 0, None)), +# ("literal", slice(15, 18, None), slice(0, 0, None)), +# ("block_end", slice(18, 30, None), slice(0, 0, None)), +# ("literal", slice(30, 39, None), slice(0, 9, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Tests Jinja "include" directive. +# """{% include 'subdir/include_comment.sql' %} +# +# SELECT 1 +# """, +# None, +# [ +# ("templated", slice(0, 42, None), slice(0, 18, None)), +# ("literal", slice(42, 53, None), slice(18, 29, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Tests Jinja "import" directive. +# """{% import 'echo.sql' as echo %} +# +# SELECT 1 +# """, +# None, +# [ +# ("templated", slice(0, 31, None), slice(0, 0, None)), +# ("literal", slice(31, 42, None), slice(0, 11, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Tests Jinja "from import" directive.. +# """{% from 'echo.sql' import echo %} +# {% from 'echoecho.sql' import echoecho %} +# +# SELECT +# {{ echo("foo") }}, +# {{ echoecho("bar") }} +# """, +# None, +# [ +# ("templated", slice(0, 33, None), slice(0, 0, None)), +# ("literal", slice(33, 34, None), slice(0, 1, None)), +# ("templated", slice(34, 75, None), slice(1, 1, None)), +# ("literal", slice(75, 88, None), slice(1, 14, None)), +# ("templated", slice(88, 105, None), slice(14, 19, None)), +# ("literal", slice(105, 111, None), slice(19, 25, None)), +# ("templated", slice(111, 132, None), slice(25, 34, None)), +# ("literal", slice(132, 133, None), slice(34, 35, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Tests Jinja "do" directive. Should be treated as a +# # templated instead of block - issue 4603. +# """{% do true %} +# +# {% if true %} +# select 1 +# {% endif %}""", +# None, +# [ +# ("templated", slice(0, 13, None), slice(0, 0, None)), +# ("literal", slice(13, 15, None), slice(0, 2, None)), +# ("block_start", slice(15, 28, None), slice(2, 2, None)), +# ("literal", slice(28, 42, None), slice(2, 16, None)), +# ("block_end", slice(42, 53, None), slice(16, 16, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Tests issue 2541, a bug where the {%- endfor %} was causing +# # IndexError: list index out of range. +# """{% for x in ['A', 'B'] %} +# {% if x != 'A' %} +# SELECT 'E' +# {% endif %} +# {%- endfor %} +# """, +# None, +# [ +# ("block_start", slice(0, 25, None), slice(0, 0, None)), +# ("literal", slice(25, 30, None), slice(0, 5, None)), +# ("block_start", slice(30, 47, None), slice(5, 5, None)), +# ("block_end", slice(67, 78, None), slice(5, 5, None)), +# ("literal", slice(78, 79, None), slice(5, 5, None)), +# ("block_end", slice(79, 92, None), slice(5, 5, None)), +# ("literal", slice(25, 30, None), slice(5, 10, None)), +# ("block_start", slice(30, 47, None), slice(10, 10, None)), +# ("literal", slice(47, 67, None), slice(10, 30, None)), +# ("block_end", slice(67, 78, None), slice(30, 30, None)), +# ("literal", slice(78, 79, None), slice(30, 30, None)), +# ("block_end", slice(79, 92, None), slice(30, 30, None)), +# ("literal", slice(92, 93, None), slice(30, 31, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Similar to the test above for issue 2541, but it's even trickier: +# # whitespace control everywhere and NO NEWLINES or other characters +# # between Jinja segments. In order to get a thorough-enough trace, +# # JinjaTracer has to build the alternate template with whitespace +# # control removed, as this increases the amount of trace output. +# "{%- for x in ['A', 'B'] -%}" +# "{%- if x == 'B' -%}" +# "SELECT 'B';" +# "{%- endif -%}" +# "{%- if x == 'A' -%}" +# "SELECT 'A';" +# "{%- endif -%}" +# "{%- endfor -%}", +# None, +# [ +# ("block_start", slice(0, 27, None), slice(0, 0, None)), +# ("block_start", slice(27, 46, None), slice(0, 0, None)), +# ("block_end", slice(57, 70, None), slice(0, 0, None)), +# ("block_start", slice(70, 89, None), slice(0, 0, None)), +# ("literal", slice(89, 100, None), slice(0, 11, None)), +# ("block_end", slice(100, 113, None), slice(11, 11, None)), +# ("block_end", slice(113, 127, None), slice(11, 11, None)), +# ("block_start", slice(27, 46, None), slice(11, 11, None)), +# ("literal", slice(46, 57, None), slice(11, 22, None)), +# ("block_end", slice(57, 70, None), slice(22, 22, None)), +# ("block_start", slice(70, 89, None), slice(22, 22, None)), +# ("block_end", slice(100, 113, None), slice(22, 22, None)), +# ("block_end", slice(113, 127, None), slice(22, 22, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Test for issue 2786. Also lots of whitespace control. In this +# # case, removing whitespace control alone wasn't enough. In order +# # to get a good trace, JinjaTracer had to be updated so the +# # alternate template included output for the discarded whitespace. +# """select +# id, +# {%- for features in ["value4", "value5"] %} +# {%- if features in ["value7"] %} +# {{features}} +# {%- if not loop.last -%},{% endif %} +# {%- else -%} +# {{features}} +# {%- if not loop.last -%},{% endif %} +# {%- endif -%} +# {%- endfor %} +# from my_table +# """, +# None, +# [ +# ("literal", slice(0, 14, None), slice(0, 14, None)), +# ("literal", slice(14, 19, None), slice(14, 14, None)), +# ("block_start", slice(19, 62, None), slice(14, 14, None)), +# ("literal", slice(62, 71, None), slice(14, 14, None)), +# ("block_start", slice(71, 103, None), slice(14, 14, None)), +# ("block_mid", slice(186, 198, None), slice(14, 14, None)), +# ("literal", slice(198, 211, None), slice(14, 14, None)), +# ("templated", slice(211, 223, None), slice(14, 20, None)), +# ("literal", slice(223, 236, None), slice(20, 20, None)), +# ("block_start", slice(236, 260, None), slice(20, 20, None)), +# ("literal", slice(260, 261, None), slice(20, 21, None)), +# ("block_end", slice(261, 272, None), slice(21, 21, None)), +# ("literal", slice(272, 281, None), slice(21, 21, None)), +# ("block_end", slice(281, 294, None), slice(21, 21, None)), +# ("literal", slice(294, 299, None), slice(21, 21, None)), +# ("block_end", slice(299, 312, None), slice(21, 21, None)), +# ("literal", slice(62, 71, None), slice(21, 21, None)), +# ("block_start", slice(71, 103, None), slice(21, 21, None)), +# ("block_mid", slice(186, 198, None), slice(21, 21, None)), +# ("literal", slice(198, 211, None), slice(21, 21, None)), +# ("templated", slice(211, 223, None), slice(21, 27, None)), +# ("literal", slice(223, 236, None), slice(27, 27, None)), +# ("block_start", slice(236, 260, None), slice(27, 27, None)), +# ("block_end", slice(261, 272, None), slice(27, 27, None)), +# ("literal", slice(272, 281, None), slice(27, 27, None)), +# ("block_end", slice(281, 294, None), slice(27, 27, None)), +# ("literal", slice(294, 299, None), slice(27, 27, None)), +# ("block_end", slice(299, 312, None), slice(27, 27, None)), +# ("literal", slice(312, 327, None), slice(27, 42, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Test for issue 2835. There's no space between "col" and "=". +# # Also tests for issue 3750 that self contained set statements +# # are parsed as "templated" and not "block_start". +# """{% set col= "col1" %} +# SELECT {{ col }} +# """, +# None, +# [ +# ("templated", slice(0, 21, None), slice(0, 0, None)), +# ("literal", slice(21, 29, None), slice(0, 8, None)), +# ("templated", slice(29, 38, None), slice(8, 12, None)), +# ("literal", slice(38, 39, None), slice(12, 13, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Another test for issue 2835. The {% for %} loop inside the +# # {% set %} caused JinjaTracer to think the {% set %} ended +# # at the {% endfor %} +# """{% set some_part_of_the_query %} +# {% for col in ["col1"] %} +# {{col}} +# {% endfor %} +# {% endset %} +# +# SELECT {{some_part_of_the_query}} +# FROM SOME_TABLE +# """, +# None, +# [ +# ("block_start", slice(0, 32, None), slice(0, 0, None)), +# ("literal", slice(32, 37, None), slice(0, 0, None)), +# ("block_start", slice(37, 62, None), slice(0, 0, None)), +# ("literal", slice(62, 67, None), slice(0, 0, None)), +# ("templated", slice(67, 74, None), slice(0, 0, None)), +# ("literal", slice(74, 79, None), slice(0, 0, None)), +# ("block_end", slice(79, 91, None), slice(0, 0, None)), +# ("literal", slice(91, 92, None), slice(0, 0, None)), +# ("block_end", slice(92, 104, None), slice(0, 0, None)), +# ("literal", slice(104, 113, None), slice(0, 9, None)), +# ("templated", slice(113, 139, None), slice(9, 29, None)), +# ("literal", slice(139, 156, None), slice(29, 46, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Third test for issue 2835. This was the original SQL provided in +# # the issue report. +# # Also tests for issue 3750 that self contained set statements +# # are parsed as "templated" and not "block_start". +# """{% set whitelisted= [ +# {'name': 'COL_1'}, +# {'name': 'COL_2'}, +# {'name': 'COL_3'} +# ] %} +# +# {% set some_part_of_the_query %} +# {% for col in whitelisted %} +# {{col.name}}{{ ", " if not loop.last }} +# {% endfor %} +# {% endset %} +# +# SELECT {{some_part_of_the_query}} +# FROM SOME_TABLE +# """, +# None, +# [ +# ("templated", slice(0, 94, None), slice(0, 0, None)), +# ("literal", slice(94, 96, None), slice(0, 2, None)), +# ("block_start", slice(96, 128, None), slice(2, 2, None)), +# ("literal", slice(128, 133, None), slice(2, 2, None)), +# ("block_start", slice(133, 161, None), slice(2, 2, None)), +# ("literal", slice(161, 166, None), slice(2, 2, None)), +# ("templated", slice(166, 178, None), slice(2, 2, None)), +# ("templated", slice(178, 205, None), slice(2, 2, None)), +# ("literal", slice(205, 210, None), slice(2, 2, None)), +# ("block_end", slice(210, 222, None), slice(2, 2, None)), +# ("literal", slice(222, 223, None), slice(2, 2, None)), +# ("block_end", slice(223, 235, None), slice(2, 2, None)), +# ("literal", slice(235, 244, None), slice(2, 11, None)), +# ("templated", slice(244, 270, None), slice(11, 66, None)), +# ("literal", slice(270, 287, None), slice(66, 83, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Test for issue 2822: Handle slicing when there's no newline after +# # the Jinja block end. +# "{% if true %}\nSELECT 1 + 1\n{%- endif %}", +# None, +# [ +# ("block_start", slice(0, 13, None), slice(0, 0, None)), +# ("literal", slice(13, 26, None), slice(0, 13, None)), +# ("literal", slice(26, 27, None), slice(13, 13, None)), +# ("block_end", slice(27, 39, None), slice(13, 13, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Test for issue 3434: Handle {% block %}. +# "SELECT {% block table_name %}block_contents{% endblock %} " +# "FROM {{ self.table_name() }}\n", +# None, +# [ +# ("literal", slice(0, 7, None), slice(0, 7, None)), +# ("literal", slice(29, 43, None), slice(7, 21, None)), +# ("block_start", slice(7, 29, None), slice(21, 21, None)), +# ("literal", slice(29, 43, None), slice(21, 21, None)), +# ("block_end", slice(43, 57, None), slice(21, 21, None)), +# ("literal", slice(57, 63, None), slice(21, 27, None)), +# ("templated", slice(63, 86, None), slice(27, 27, None)), +# ("literal", slice(29, 43, None), slice(27, 41, None)), +# ("literal", slice(86, 87, None), slice(41, 42, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Another test for issue 3434: Similar to the first, but uses +# # the block inside a loop. +# """{% block table_name %}block_contents{% endblock %} +# SELECT +# {% for j in [4, 5, 6] %} +# FROM {{ j }}{{ self.table_name() }} +# {% endfor %} +# """, +# None, +# [ +# ("literal", slice(22, 36, None), slice(0, 14, None)), +# ("block_start", slice(0, 22, None), slice(14, 14, None)), +# ("literal", slice(22, 36, None), slice(14, 14, None)), +# ("block_end", slice(36, 50, None), slice(14, 14, None)), +# ("literal", slice(50, 58, None), slice(14, 22, None)), +# ("block_start", slice(58, 82, None), slice(22, 22, None)), +# ("literal", slice(82, 88, None), slice(22, 28, None)), +# ("templated", slice(88, 95, None), slice(28, 29, None)), +# ("templated", slice(95, 118, None), slice(29, 29, None)), +# ("literal", slice(22, 36, None), slice(29, 43, None)), +# ("literal", slice(118, 119, None), slice(43, 44, None)), +# ("block_end", slice(119, 131, None), slice(44, 44, None)), +# ("literal", slice(82, 88, None), slice(44, 50, None)), +# ("templated", slice(88, 95, None), slice(50, 51, None)), +# ("templated", slice(95, 118, None), slice(51, 51, None)), +# ("literal", slice(22, 36, None), slice(51, 65, None)), +# ("literal", slice(118, 119, None), slice(65, 66, None)), +# ("block_end", slice(119, 131, None), slice(66, 66, None)), +# ("literal", slice(82, 88, None), slice(66, 72, None)), +# ("templated", slice(88, 95, None), slice(72, 73, None)), +# ("templated", slice(95, 118, None), slice(73, 73, None)), +# ("literal", slice(22, 36, None), slice(73, 87, None)), +# ("literal", slice(118, 119, None), slice(87, 88, None)), +# ("block_end", slice(119, 131, None), slice(88, 88, None)), +# ("literal", slice(131, 132, None), slice(88, 89, None)), +# ], +# JinjaTemplater, +# ), +# ( +# "{{ statement('variables', fetch_result=true) }}\n", +# dict( +# statement=_statement, +# load_result=_load_result, +# ), +# [ +# ("templated", slice(0, 47, None), slice(0, 0, None)), +# ("literal", slice(47, 48, None), slice(0, 1, None)), +# ], +# JinjaTemplater, +# ), +# ( +# "{% call statement('variables', fetch_result=true) %}\n" +# "select 1 as test\n" +# "{% endcall %}\n" +# "select 2 as foo\n", +# dict( +# statement=_statement, +# load_result=_load_result, +# ), +# [ +# ("block_start", slice(0, 52, None), slice(0, 0, None)), +# ("literal", slice(52, 70, None), slice(0, 0, None)), +# ("block_end", slice(70, 83, None), slice(0, 0, None)), +# ("literal", slice(83, 100, None), slice(0, 17, None)), +# ], +# JinjaTemplater, +# ), +# ( +# JINJA_MACRO_CALL_SQL, +# None, +# [ +# # First all of this is the call block. +# ("block_start", slice(0, 30, None), slice(0, 0, None)), +# ("literal", slice(30, 34, None), slice(0, 0, None)), +# ("templated", slice(34, 45, None), slice(0, 0, None)), +# ("literal", slice(45, 55, None), slice(0, 0, None)), +# ("templated", slice(55, 69, None), slice(0, 0, None)), +# ("literal", slice(69, 70, None), slice(0, 0, None)), +# ("block_end", slice(70, 84, None), slice(0, 0, None)), +# # Then the actual query. +# ("literal", slice(84, 96, None), slice(0, 12, None)), +# # The block_start (call) contains the actual content. +# ("block_start", slice(96, 125, None), slice(12, 47, None)), +# # The middle and end of the call, have zero length in the template +# ("literal", slice(125, 142, None), slice(47, 47, None)), +# ("block_end", slice(142, 155, None), slice(47, 47, None)), +# ("literal", slice(155, 165, None), slice(47, 57, None)), +# ], +# JinjaTemplater, +# ), +# ( +# # Simple test of a derived templater with custom tags +# """{% up 'create table xyz' %} +# CREATE TABLE xyz (id int); +# {% down %} +# DROP TABLE xyz; +# {% end %}""", +# None, +# [ +# ("block_start", slice(0, 27, None), slice(0, 0, None)), +# ("literal", slice(27, 59, None), slice(0, 32, None)), +# ("block_mid", slice(59, 69, None), slice(32, 32, None)), +# ("literal", slice(69, 90, None), slice(32, 53, None)), +# ("block_end", slice(90, 99, None), slice(53, 53, None)), +# ], +# DerivedJinjaTemplater, +# ), +# ( +# # test for issue 6121: The first rendered element +# # inside the loop is far from the start position of the loop. +# """ +# {% for i in range(2) %}{% set a = 0 %}{% set b = 0 %}{% set c = 0 %} +# SELECT 1; +# {% endfor %} +# """, +# None, +# [ +# ("literal", slice(0, 1, None), slice(0, 1, None)), +# ("block_start", slice(1, 24, None), slice(1, 1, None)), +# ("templated", slice(24, 39, None), slice(1, 1, None)), +# ("templated", slice(39, 54, None), slice(1, 1, None)), +# ("templated", slice(54, 69, None), slice(1, 1, None)), +# ("literal", slice(69, 80, None), slice(1, 12, None)), +# ("block_end", slice(80, 92, None), slice(12, 12, None)), +# ("templated", slice(24, 39, None), slice(12, 12, None)), +# ("templated", slice(39, 54, None), slice(12, 12, None)), +# ("templated", slice(54, 69, None), slice(12, 12, None)), +# ("literal", slice(69, 80, None), slice(12, 23, None)), +# ("block_end", slice(80, 92, None), slice(23, 23, None)), +# ("literal", slice(92, 93, None), slice(23, 24, None)), +# ], +# JinjaTemplater, +# ), +# ], +# ) +# def test__templater_jinja_slice_file( +# raw_file, override_context, result, templater_class, caplog +# ): +# """Test slice_file.""" +# templater = templater_class(override_context=override_context) +# _, _, render_func = templater.construct_render_func( +# config=FluffConfig.from_path( +# "test/fixtures/templater/jinja_slice_template_macros" +# ) +# ) +# +# with caplog.at_level(logging.DEBUG, logger="sqlfluff.templater"): +# raw_sliced, sliced_file, templated_str = templater.slice_file( +# raw_file, render_func=render_func +# ) +# # Create a TemplatedFile from the results. This runs some useful sanity +# # checks. +# _ = TemplatedFile(raw_file, "<>", templated_str, sliced_file, raw_sliced) +# # Check contiguous on the TEMPLATED VERSION +# print(sliced_file) +# prev_slice = None +# for elem in sliced_file: +# print(elem) +# if prev_slice: +# assert elem[2].start == prev_slice.stop +# prev_slice = elem[2] +# # Check that all literal segments have a raw slice +# for elem in sliced_file: +# if elem[0] == "literal": +# assert elem[1] is not None +# # check result +# actual = [ +# ( +# templated_file_slice.slice_type, +# templated_file_slice.source_slice, +# templated_file_slice.templated_slice, +# ) +# for templated_file_slice in sliced_file +# ] +# assert actual == result +# +# +# def test__templater_jinja_large_file_check(): +# """Test large file skipping. +# +# The check is separately called on each .process() method +# so it makes sense to test a few templaters. +# """ +# # First check we can process the file normally without specific config. +# # i.e. check the defaults work and the default is high. +# JinjaTemplater().process( +# in_str="SELECT 1", +# fname="", +# config=FluffConfig(overrides={"dialect": "ansi"}), +# ) +# # Second check setting the value low disables the check +# JinjaTemplater().process( +# in_str="SELECT 1", +# fname="", +# config=FluffConfig( +# overrides={"dialect": "ansi", "large_file_skip_char_limit": 0} +# ), +# ) +# # Finally check we raise a skip exception when config is set low. +# with pytest.raises(SQLFluffSkipFile) as excinfo: +# JinjaTemplater().process( +# in_str="SELECT 1", +# fname="", +# config=FluffConfig( +# overrides={"dialect": "ansi", "large_file_skip_char_limit": 2}, +# ), +# ) +# +# assert "Length of file" in str(excinfo.value) +# +# +# @pytest.mark.parametrize( +# "in_str, ignore, expected_violation", +# [ +# ( +# """WITH a AS ({{ b(c=d, e=f) }}) SELECT * FROM final""", +# "", +# SQLTemplaterError("Undefined jinja template variable: 'b'"), +# ), +# ("""WITH a AS ({{ b(c=d, e=f) }}) SELECT * FROM final""", "templating", None), +# ( +# # https://github.com/sqlfluff/sqlfluff/issues/6360 +# """{% for tbl in tbl_list %}SELECT a FROM {{ tbl }};{% endfor %}""", +# "", +# SQLTemplaterError("Undefined jinja template variable: 'tbl_list'"), +# ), +# ( +# """SELECT a FROM {{ tbl['name'] }};""", +# "", +# SQLTemplaterError("Undefined jinja template variable: 'tbl'"), +# ), +# ], +# ) +# def test_jinja_undefined_callable(in_str, ignore, expected_violation): +# """Test undefined callable returns TemplatedFile and sensible error.""" +# templater = JinjaTemplater() +# templated_file, violations = templater.process( +# in_str=in_str, +# fname="test.sql", +# config=FluffConfig(overrides={"dialect": "ansi", "ignore": ignore}), +# ) +# # This was previously failing to process, due to UndefinedRecorder not +# # supporting __call__(), also Jinja thinking it was not *safe* to call. +# assert templated_file is not None +# if expected_violation: +# assert len(violations) == 1 +# isinstance(violations[0], type(expected_violation)) +# assert str(violations[0]) == str(expected_violation) +# else: +# assert len(violations) == 0 +# +# +# def test_dummy_undefined_fail_with_undefined_error(): +# """Tests that a recursion error bug no longer occurs.""" +# ud = DummyUndefined("name") +# with pytest.raises(UndefinedError): +# # This was previously causing a recursion error. +# ud._fail_with_undefined_error() +# +# +# def test_undefined_magic_methods(): +# """Test all the magic methods defined on DummyUndefined.""" +# ud = DummyUndefined("name") +# +# # _self_impl +# assert ud + ud is ud +# assert ud - ud is ud +# assert ud / ud is ud +# assert ud // ud is ud +# assert ud % ud is ud +# assert ud**ud is ud +# assert +ud is ud +# assert -ud is ud +# assert ud << ud is ud +# assert ud[ud] is ud +# assert ~ud is ud +# assert ud(ud) is ud +# +# # _bool_impl +# assert ud and ud +# assert ud or ud +# assert ud ^ ud +# assert bool(ud) +# assert ud < ud +# assert ud <= ud +# assert ud == ud +# assert ud != ud +# assert ud >= ud +# assert ud > ud +# +# assert ud + ud is ud +# +# +# @pytest.mark.parametrize( +# "sql_path, expected_renderings", +# [ +# pytest.param( +# "simple_if_true.sql", +# [ +# "\nSELECT 1\n\n", +# "\nSELECT 2\n\n", +# ], +# id="simple_if_true", +# ), +# pytest.param( +# "simple_if_false.sql", +# [ +# "\nSELECT 2\n\n", +# "\nSELECT 1\n\n", +# ], +# id="simple_if_false", +# ), +# pytest.param( +# "if_elif_else.sql", +# [ +# "\nSELECT 1\n\n", +# "\nSELECT 2\n\n", +# "\nSELECT 3\n\n", +# ], +# id="if_elif_else", +# ), +# pytest.param( +# "if_else_if_nested.sql", +# [ +# "\nSELECT 1\n\n", +# "\n\nSELECT 2\n\n\n", +# "\n\nSELECT 3\n\n\n", +# ], +# id="if_else_if_nested", +# ), +# # This test case exercises the scoring function. Generates up to 10 +# # variants, but only the top 5 are returned. +# pytest.param( +# "if_elif_else_chain_scoring.sql", +# [ +# "\nSELECT 1\n\n", +# "\nSELECT 100000000\n\n", +# "\nSELECT 10000000\n\n", +# "\nSELECT 1000000\n\n", +# "\nSELECT 100000\n\n", +# "\nSELECT 10000\n\n", +# ], +# id="if_elif_else_chain_scoring", +# ), +# # This test case results in a TypeError executing the variant. This +# # should be ignored, and only the primary should be returned. +# pytest.param( +# "if_true_elif_type_error_else.sql", +# [ +# "\nSELECT 1\n\n", +# "\nSELECT 2\n\n", +# ], +# id="if_true_elif_type_error_else", +# ), +# # https://github.com/sqlfluff/sqlfluff/issues/5803 +# pytest.param( +# "inline_select.sql", +# [ +# "select 2\n", +# "select 1\n", +# ], +# id="inline_select", +# ), +# ], +# ) +# def test__templater_lint_unreached_code(sql_path: str, expected_renderings): +# """Test that Jinja templater slices raw and templated file correctly.""" +# test_dir = Path("test/fixtures/templater/jinja_lint_unreached_code") +# t = JinjaTemplater() +# renderings = [] +# raw_slicings = [] +# final_source_slices = [] +# for templated_file, _ in t.process_with_variants( +# in_str=(test_dir / sql_path).read_text(), +# fname=str(sql_path), +# config=FluffConfig.from_path(str(test_dir)), +# ): +# renderings.append(templated_file.templated_str) +# raw_slicings.append(templated_file.raw_sliced) +# # Capture the final slice for all of them. +# final_source_slices.append(templated_file.sliced_file[-1].source_slice) +# assert renderings == expected_renderings +# # Compare all of the additional raw slicings to make sure they're the +# # same as the root. +# root_slicing = raw_slicings[0] +# for additional_slicing in raw_slicings[1:]: +# assert additional_slicing == root_slicing +# # Check that the final source slices also line up in the templated files. +# # NOTE: Clearly the `templated_slice` values _won't_ be the same. +# # We're doing the _final_ slice, because it's very likely to be the same +# # _type_ and if it's in the right place, we can assume that all of the +# # others probably are. +# root_final_slice = final_source_slices[0] +# for additional_final_slice in final_source_slices[1:]: +# assert additional_final_slice == root_final_slice diff --git a/crates/lib/src/templaters/jinja_templater_tracers.py b/crates/lib/src/templaters/jinja_templater_tracers.py index 1babb1393..b18a5d336 100644 --- a/crates/lib/src/templaters/jinja_templater_tracers.py +++ b/crates/lib/src/templaters/jinja_templater_tracers.py @@ -24,7 +24,7 @@ from jinja2 import Environment from jinja2.exceptions import TemplateSyntaxError -from sqlfluff.core.templaters.base import RawFileSlice, TemplatedFileSlice +from templaters.python_templater import RawFileSlice, TemplatedFileSlice # Instantiate the templater logger templater_logger = logging.getLogger("sqlfluff.templater") diff --git a/pyproject.toml b/pyproject.toml index 43b3ebbc0..735ce158c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,8 @@ version = "0.0.1" readme = "README.md" requires-python = ">=3.9" dependencies = [ - "Jinja2>=3.0.0", + "Jinja2>=3.0.0", + "regex" ] [project.optional-dependencies]