Update transpile to support --overrides-path and --target-technology arguments

labs.yml

@@ -26,6 +26,12 @@ commands:
       - name: source-dialect
         description: Dialect name as selected during `install-transpile` or refer to documentation
         default: null
+      - name: overrides-file
+        description: Path to a file containing transpiler overrides, if supported by the transpiler in use.
+        default: null
+      - name: target-technology
+        description: Target technology to use for code generation, if supported by the transpiler in use.
+        default: null
       - name: input-source
         description: Input Script Folder or File
         default: null
@@ -37,7 +43,7 @@ commands:
         default: null
       - name: skip-validation
         description: Validate Transpiled Code, default True validation skipped, False validate
-        default: true
+        default: "true"
       - name: catalog-name
         description: Catalog Name Applicable only when Validation Mode is DATABRICKS
         default: null

src/databricks/labs/lakebridge/cli.py

@@ -16,7 +16,7 @@
 
 from databricks.labs.blueprint.cli import App
 from databricks.labs.blueprint.entrypoint import get_logger, is_in_debug
-from databricks.labs.blueprint.installation import RootJsonValue
+from databricks.labs.blueprint.installation import RootJsonValue, JsonObject, JsonValue
 from databricks.labs.blueprint.tui import Prompts
 
 
@@ -25,7 +25,7 @@
     PROFILER_SOURCE_SYSTEM,
 )
 
-from databricks.labs.lakebridge.config import TranspileConfig
+from databricks.labs.lakebridge.config import TranspileConfig, LSPConfigOptionV1
 from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
@@ -86,26 +86,33 @@ def _remove_warehouse(ws: WorkspaceClient, warehouse_id: str):
 
 
 @lakebridge.command
-def transpile(
+def transpile(  # pylint: disable=too-many-arguments
     *,
     w: WorkspaceClient,
     transpiler_config_path: str | None = None,
     source_dialect: str | None = None,
+    overrides_file: str | None = None,
+    target_technology: str | None = None,
     input_source: str | None = None,
     output_folder: str | None = None,
     error_file_path: str | None = None,
     skip_validation: str | None = None,
     catalog_name: str | None = None,
     schema_name: str | None = None,
+    ctx: ApplicationContext | None = None,
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ):
     """Transpiles source dialect to databricks dialect"""
-    ctx = ApplicationContext(w)
+    if ctx is None:
+        ctx = ApplicationContext(w)
+    del w
     logger.debug(f"Preconfigured transpiler config: {ctx.transpile_config!r}")
     ctx.add_user_agent_extra("cmd", "execute-transpile")
     checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts, transpiler_repository)
     checker.use_transpiler_config_path(transpiler_config_path)
     checker.use_source_dialect(source_dialect)
+    checker.use_overrides_file(overrides_file)
+    checker.use_target_technology(target_technology)
     checker.use_input_source(input_source)
     checker.use_output_folder(output_folder)
     checker.use_error_file_path(error_file_path)
@@ -217,6 +224,36 @@ def use_source_dialect(self, source_dialect: str | None) -> None:
             logger.debug(f"Pending source_dialect override: {source_dialect!r}")
             self._source_dialect_override = source_dialect
 
+    @staticmethod
+    def _validate_overrides_file(overrides_file: str, msg: str) -> None:
+        """Validate the overrides file: it must be a valid path that exists."""
+        # Note: in addition to this check, later we verify the transpiler supports it.
+        if not Path(overrides_file).exists():
+            raise_validation_exception(msg)
+
+    def use_overrides_file(self, overrides_file: str | None) -> None:
+        if overrides_file is not None:
+            logger.debug(f"Setting overrides_file to: {overrides_file!r}")
+            msg = f"Invalid path for '--overrides-file', does not exist: {overrides_file}"
+            self._validate_overrides_file(overrides_file, msg)
+            try:
+                self._set_config_transpiler_option("overrides-file", overrides_file)
+            except ValueError:
+                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
+                msg = "Cannot use --overrides-file; workspace config.yml has incompatible transpiler_options."
+                raise_validation_exception(msg)
+
+    def use_target_technology(self, target_technology: str | None) -> None:
+        if target_technology is not None:
+            logger.debug(f"Setting target_technology to: {target_technology!r}")
+            # Cannot validate this here: depends on the transpiler engine, and will be checked later.
+            try:
+                self._set_config_transpiler_option("target-tech", target_technology)
+            except ValueError:
+                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
+                msg = "Cannot use --target-technology; workspace config.yml has incompatible transpiler_options."
+                raise_validation_exception(msg)
+
     @staticmethod
     def _validate_input_source(input_source: str, msg: str) -> None:
         """Validate the input source: it must be a path that exists."""
@@ -321,6 +358,19 @@ def use_schema_name(self, schema_name: str | None) -> None:
             logger.debug(f"Setting schema_name to: {schema_name!r}")
             self._config = dataclasses.replace(self._config, schema_name=schema_name)
 
+    def _set_config_transpiler_option(self, flag: str, value: str) -> None:
+        transpiler_options: JsonObject
+        match self._config.transpiler_options:
+            case None:
+                transpiler_options = {flag: value}
+            case Mapping() as found_options:
+                transpiler_options = {**found_options, flag: value}
+            case found_options:
+                # TODO: Update `config.yml' to constrain `transpiler_options` to be a dict[str, str].
+                msg = f"Incompatible transpiler options configured, must be a mapping: {found_options!r}"
+                raise ValueError(msg)
+        self._config = dataclasses.replace(self._config, transpiler_options=transpiler_options)
+
     def _configure_transpiler_config_path(self, source_dialect: str) -> TranspileEngine | None:
         """Configure the transpiler config path based on the requested source dialect."""
         # Names of compatible transpiler engines for the given dialect.
@@ -448,18 +498,52 @@ def _check_transpiler_options(self, engine: TranspileEngine) -> None:
         assert self._config.source_dialect is not None, "Source dialect must be set before checking transpiler options."
         options_for_dialect = engine.options_for_dialect(self._config.source_dialect)
         transpiler_options = self._config.transpiler_options
-        if not isinstance(transpiler_options, Mapping):
-            return
+        if transpiler_options is None:
+            transpiler_options = {}
+        elif not isinstance(transpiler_options, Mapping):
+            logger.warning(f"Ignoring transpiler_options in config.yml, must be a mapping: {transpiler_options!r}")
+            transpiler_options = {}
+        # Only checks if the option is present, does not validate the value.
+        # TODO: Validate the value for CHOICE/CONFIRM options.
+        # TODO: Handle FORCE options: these are fixed by the transpiler, and cannot be overridden.
         checked_options = {
             option.flag: (
                 transpiler_options[option.flag]
                 if option.flag in transpiler_options
-                else option.prompt_for_value(self._prompts)
+                else self._handle_missing_transpiler_option(option)
             )
             for option in options_for_dialect
         }
         self._config = dataclasses.replace(self._config, transpiler_options=checked_options)
 
+    def _handle_missing_transpiler_option(self, option: LSPConfigOptionV1) -> JsonValue:
+        # Semantics during configuration:
+        #  - Entries are present in the config file for all options the LSP server needs for a dialect.
+        #  - If a value is `None`, it means the user wants the value to be left unset.
+        #  - There is no 'provide it later' option: either it's set, or it's unset.
+        # As a corner case, if there is no entry present it means the user wasn't prompted. Here we have
+        # some complexity. We have two ways of obtaining a value:
+        #  - The user could provide it on the command-line, using --target-technology or --overrides-file.
+        #    Problem: via command-line options there's no way to indicate 'no value'.
+        #  - We could prompt for it, assuming the user is running interactively.
+        # In terms of what is required by the option:
+        #  - If the option has a default of <none>, it means that no value is required.
+        #  - Everything else requires a value.
+        #
+        # This leads to the following business rules:
+        #  - If the option has a default of <none> that means that no value is required, no further action is required.
+        #  - Otherwise, a value is required: prompt for it.
+        #
+        # TODO: When adding non-interactive support, the otherwise branch need to be modified:
+        #     1. If it can be provided by the command-line, fail and ask the user to provide it.
+        #     2. If it cannot be provided by the command-line, prompt for it if we are running interactively.
+        #     3. If we cannot prompt because we are not running interactively, use the default if there is one.
+        #     4. Fail: the only way to provide a value is via the config.yml, which can be set via 'install-transpile'.
+
+        if option.default == "<none>":
+            return None
+        return option.prompt_for_value(self._prompts)
+
     def check(self) -> tuple[TranspileConfig, TranspileEngine]:
         """Checks that all configuration parameters are present and valid."""
         logger.debug(f"Checking config: {self._config!r}")

src/databricks/labs/lakebridge/config.py

@@ -130,9 +130,15 @@ def prompt_for_value(self, prompts: Prompts) -> JsonValue:
         if self.method == LSPPromptMethod.CONFIRM:
             return prompts.confirm(self.prompt)
         if self.method == LSPPromptMethod.QUESTION:
-            default = self.default if self.default else "None"
+            # Hack to:
+            #  - trick prompts.question() into indicating that no answer is required;
+            #  - allow no answer to be given.
+            # Normally prompts.confirm() requires an answer, or returns the default, and the default can't be None.
+            # Note: LSP servers use '<none>' as a default to indicate that no answer is required.
+            no_answer = "<none>"
+            default = self.default if self.default else no_answer
             result = prompts.question(self.prompt, default=default)
-            if result == "<none>":
+            if result == no_answer:
                 return None
             return result
         if self.method == LSPPromptMethod.CHOICE:

src/databricks/labs/lakebridge/install.py

@@ -262,6 +262,11 @@ def _prompt_for_transpiler_options(self, transpiler_name: str, source_dialect: s
         config_options = self._transpiler_repository.transpiler_config_options(transpiler_name, source_dialect)
         if len(config_options) == 0:
             return None
+        # Semantics here are different from the other properties of a TranspileConfig. Specifically:
+        #  - Entries are present for all options.
+        #  - If the value is None, it means the user chose to not provide a value. (This differs to other
+        #    attributes, where None means the user chose to provide a value later.)
+        #  - There is no way to express 'provide a value later'.
         return {option.flag: option.prompt_for_value(self._prompts) for option in config_options}
 
     def _configure_catalog(self) -> str:

tests/conftest.py

@@ -284,7 +284,7 @@ def mock_data_source():
     return MockDataSource({}, {})
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def bladebridge_artifact() -> Path:
     artifact = (
         Path(__file__).parent
@@ -298,7 +298,7 @@ def bladebridge_artifact() -> Path:
     return artifact
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def morpheus_artifact() -> Path:
     artifact = (
         Path(__file__).parent

tests/integration/transpile/common_utils.py

@@ -4,14 +4,10 @@
 
 
 def assert_sql_outputs(output_folder: Path, expected_sql: str, expected_failure_sql: str) -> None:
-    assert (output_folder / "create_ddl.sql").exists()
-    with open(output_folder / "create_ddl.sql", "r", encoding="utf-8") as f:
-        actual_sql = f.read()
-    assert actual_sql.strip() == expected_sql.strip()
+    actual_sql = (output_folder / "create_ddl.sql").read_text(encoding="utf-8")
+    actual_failure_sql = (output_folder / "dummy_function.sql").read_text(encoding="utf-8")
 
-    assert (output_folder / "dummy_function.sql").exists()
-    with open(output_folder / "dummy_function.sql", "r", encoding="utf-8") as f:
-        actual_failure_sql = f.read()
+    assert actual_sql.strip() == expected_sql.strip()
     assert actual_failure_sql.strip() == expected_failure_sql.strip()