Major changes to interpolations and resolvers (omry#445)

* Add a grammar to parse interpolations
* Add support for nested interpolations
* Deprecate register_resolver() and introduce new_register_resolver() (that allows resolvers to use non-string arguments, and to decide whether or not to use the cache)
* The `env` resolver now parses environment variables in a way that is consistent with the new interpolation grammar

Fixes omry#100 omry#230 omry#266 omry#318

Author: odelalleau

.circleci/config.yml

@@ -1,47 +1,41 @@
 version: 2.1
 
-jobs:
-  py36_linux:
-    docker:
-      - image: circleci/python:3.6
-    steps:
-      - checkout
-      - run: echo 'export NOX_PYTHON_VERSIONS=3.6' >> $BASH_ENV
-      - run: sudo pip install nox
-      - run: nox --add-timestamp
-
-  py37_linux:
-    docker:
-      - image: circleci/python:3.7
+commands:
+  linux:
+    description: "Commands run on Linux"
+    parameters:
+      py_version:
+        type: string
     steps:
       - checkout
-      - run: echo 'export NOX_PYTHON_VERSIONS=3.7' >> $BASH_ENV
-      - run: sudo pip install nox
-      - run: nox --add-timestamp
+      - run:
+          name: "Preparing environment"
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y openjdk-11-jre
+            sudo pip install nox
+      - run:
+          name: "Testing OmegaConf"
+          command: |
+            export NOX_PYTHON_VERSIONS=<< parameters.py_version >>
+            nox --add-timestamp
 
-  py38_linux:
-    docker:
-      - image: circleci/python:3.8
-    steps:
-      - checkout
-      - run: echo 'export NOX_PYTHON_VERSIONS=3.8' >> $BASH_ENV
-      - run: sudo pip install nox
-      - run: nox --add-timestamp
-  
-  py39_linux:
+jobs:
+  test_linux:
+    parameters:
+      py_version:
+        type: string
     docker:
-      - image: circleci/python:3.9
+      - image: circleci/python:<< parameters.py_version >>
     steps:
-      - checkout
-      - run: echo 'export NOX_PYTHON_VERSIONS=3.9' >> $BASH_ENV
-      - run: sudo pip install nox
-      - run: nox --add-timestamp
+      - linux:
+          py_version: << parameters.py_version >>
 
 workflows:
   version: 2
   build:
     jobs:
-      - py36_linux
-      - py37_linux
-      - py38_linux
-      - py39_linux
+      - test_linux:
+          matrix:
+            parameters:
+              py_version: ["3.6", "3.7", "3.8", "3.9"]

.coveragerc

@@ -3,6 +3,7 @@ omit =
     .nox/*
     *tests*
     docs/*
+    omegaconf/grammar/gen/*
     omegaconf/version.py
     .stubs
 
@@ -15,7 +16,7 @@ exclude_lines =
     assert False
     @abstractmethod
     \.\.\.
-
+    if TYPE_CHECKING:
 
 [html]
 directory = docs/build/coverage

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-exclude = .git,.nox,.tox
+exclude = .git,.nox,.tox,omegaconf/grammar/gen
 max-line-length = 119
 select = E,F,W,C
 ignore=W503,E203

.gitattributes

@@ -0,0 +1 @@
+*.jar binary

.gitignore

@@ -16,7 +16,9 @@ TODO
 .coverage
 .eggs
 .mypy_cache
+/omegaconf/grammar/gen
 /pip-wheel-metadata
 /.pyre
 .dmypy.json
 .python-version
+.vscode

.isort.cfg

@@ -7,3 +7,4 @@ line_length=88
 ensure_newline_before_comments=True
 known_third_party=attr,pytest
 known_first_party=omegaconf
+skip=.eggs,.nox,omegaconf/grammar/gen

benchmark/benchmark.py

@@ -4,6 +4,7 @@
 from pytest import fixture, mark, param
 
 from omegaconf import OmegaConf
+from omegaconf._utils import ValueKind, get_value_kind
 
 
 def build_dict(
@@ -119,3 +120,23 @@ def iterate(seq: Any) -> None:
             pass
 
     benchmark(iterate, lst)
+
+
+@mark.parametrize(
+    "strict_interpolation_validation",
+    [True, False],
+)
+@mark.parametrize(
+    ("value", "expected"),
+    [
+        ("simple", ValueKind.VALUE),
+        ("${a}", ValueKind.INTERPOLATION),
+        ("${a:b,c,d}", ValueKind.INTERPOLATION),
+        ("${${b}}", ValueKind.INTERPOLATION),
+        ("${a:${b}}", ValueKind.INTERPOLATION),
+    ],
+)
+def test_get_value_kind(
+    strict_interpolation_validation: bool, value: Any, expected: Any, benchmark: Any
+) -> None:
+    assert benchmark(get_value_kind, value, strict_interpolation_validation) == expected

build_helpers/__init__.py

@@ -0,0 +1,3 @@
+# Order of imports is important (see warning otherwise when running tests)
+import setuptools  # isort:skip # noqa
+import distutils  # isort:skip # noqa

build_helpers/bin/antlr-4.8-complete.jar

@@ -0,0 +1,195 @@
+import codecs
+import distutils.log
+import errno
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from setuptools import Command
+from setuptools.command import build_py, develop, sdist  # type: ignore
+
+
+class ANTLRCommand(Command):  # type: ignore  # pragma: no cover
+    """Generate parsers using ANTLR."""
+
+    description = "Run ANTLR"
+    user_options: List[str] = []
+
+    def run(self) -> None:
+        """Run command."""
+        build_dir = Path(__file__).parent.absolute()
+        project_root = build_dir.parent
+        for grammar in [
+            "OmegaConfGrammarLexer.g4",
+            "OmegaConfGrammarParser.g4",
+        ]:
+            command = [
+                "java",
+                "-jar",
+                str(build_dir / "bin" / "antlr-4.8-complete.jar"),
+                "-Dlanguage=Python3",
+                "-o",
+                str(project_root / "omegaconf" / "grammar" / "gen"),
+                "-Xexact-output-dir",
+                "-visitor",
+                str(project_root / "omegaconf" / "grammar" / grammar),
+            ]
+
+            self.announce(
+                f"Generating parser for Python3: {command}",
+                level=distutils.log.INFO,
+            )
+
+            subprocess.check_call(command)
+
+    def initialize_options(self) -> None:
+        pass
+
+    def finalize_options(self) -> None:
+        pass
+
+
+class BuildPyCommand(build_py.build_py):  # type: ignore  # pragma: no cover
+    def run(self) -> None:
+        if not self.dry_run:
+            self.run_command("clean")
+            run_antlr(self)
+        build_py.build_py.run(self)
+
+
+class CleanCommand(Command):  # type: ignore  # pragma: no cover
+    """
+    Our custom command to clean out junk files.
+    """
+
+    description = "Cleans out generated and junk files we don't want in the repo"
+    dry_run: bool
+    user_options: List[str] = []
+
+    def run(self) -> None:
+        root = Path(__file__).parent.parent.absolute()
+        files = find(
+            root=root,
+            include_files=["^omegaconf/grammar/gen/.*"],
+            include_dirs=[
+                "^omegaconf\\.egg-info$",
+                "\\.eggs$",
+                "^\\.mypy_cache$",
+                "^\\.nox$",
+                "^\\.pytest_cache$",
+                ".*/__pycache__$",
+                "^__pycache__$",
+                "^build$",
+                "^dist$",
+            ],
+            scan_exclude=["^.git$", "^.nox/.*$"],
+            excludes=[".*\\.gitignore$", ".*/__init__.py"],
+        )
+
+        if self.dry_run:
+            print("Dry run! Would clean up the following files and dirs:")
+            print("\n".join(sorted(map(str, files))))
+        else:
+            for f in files:
+                if f.exists():
+                    if f.is_dir():
+                        shutil.rmtree(f, ignore_errors=True)
+                    else:
+                        f.unlink()
+
+    def initialize_options(self) -> None:
+        pass
+
+    def finalize_options(self) -> None:
+        pass
+
+
+class DevelopCommand(develop.develop):  # type: ignore  # pragma: no cover
+    def run(self) -> None:
+        if not self.dry_run:
+            run_antlr(self)
+        develop.develop.run(self)
+
+
+class SDistCommand(sdist.sdist):  # type: ignore  # pragma: no cover
+    def run(self) -> None:
+        if not self.dry_run:
+            self.run_command("clean")
+            run_antlr(self)
+        sdist.sdist.run(self)
+
+
+def find(
+    root: Path,
+    include_files: List[str],
+    include_dirs: List[str],
+    excludes: List[str],
+    rbase: Optional[Path] = None,
+    scan_exclude: Optional[List[str]] = None,
+) -> List[Path]:
+    if rbase is None:
+        rbase = Path()
+    if scan_exclude is None:
+        scan_exclude = []
+    files = []
+    scan_root = root / rbase
+    for entry in scan_root.iterdir():
+        path = rbase / entry.name
+        if matches(scan_exclude, path):
+            continue
+
+        if entry.is_dir():
+            if matches(include_dirs, path):
+                if not matches(excludes, path):
+                    files.append(path)
+            else:
+                ret = find(
+                    root=root,
+                    include_files=include_files,
+                    include_dirs=include_dirs,
+                    excludes=excludes,
+                    rbase=path,
+                    scan_exclude=scan_exclude,
+                )
+                files.extend(ret)
+        else:
+            if matches(include_files, path) and not matches(excludes, path):
+                files.append(path)
+
+    return files
+
+
+def find_version(*file_paths: str) -> str:
+    root = Path(__file__).parent.parent.absolute()
+    with codecs.open(root / Path(*file_paths), "r") as fp:  # type: ignore
+        version_file = fp.read()
+    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
+    if version_match:
+        return version_match.group(1)
+    raise RuntimeError("Unable to find version string.")  # pragma: no cover
+
+
+def matches(patterns: List[str], path: Path) -> bool:
+    string = str(path).replace(os.sep, "/")  # for Windows
+    for pattern in patterns:
+        if re.match(pattern, string):
+            return True
+    return False
+
+
+def run_antlr(cmd: Command) -> None:  # pragma: no cover
+    try:
+        cmd.announce("Generating parsers with antlr4", level=distutils.log.INFO)
+        cmd.run_command("antlr")
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            msg = f"| Unable to generate parsers: {e} |"
+            msg = "=" * len(msg) + "\n" + msg + "\n" + "=" * len(msg)
+            cmd.announce(f"{msg}", level=distutils.log.FATAL)
+            sys.exit(1)
+        else:
+            raise

build_helpers/build_helpers.py

@@ -0,0 +1 @@
+Intentionally left empty for git to keep the directory

build_helpers/test_files/a/b/bad_dir/.gitkeep

@@ -0,0 +1 @@
+Intentionally left empty for git to keep the directory