Update transpile to support --overrides-path and --target-technology arguments

docs/lakebridge/docs/installation.mdx

@@ -156,7 +156,7 @@ Specify the config file to override the default[Bladebridge] config - press <ent
 :::
 
 ### Verify Installation
-Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example screenshot provided:
+Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example output:
 
 Command:
 ```bash
@@ -176,9 +176,11 @@ Flags:
   -h, --help                          help for transpile
       --input-source path             (Optional) Local path of the sources to be convert
       --output-folder path            (Optional) Local path where converted code will be written
+      --overrides-file path           (Optional) Local path of a file containing transpiler overrides, if supported by the transpiler in use
       --schema-name name              (Optional) Schema name, only used when validating converted code
       --skip-validation string        (Optional) Whether to skip validating the output ('true') after conversion or not ('false')
       --source-dialect string         (Optional) The source dialect to use when performing conversion
+      --target-technology string      (Optional) Target technology to use for code generation, if supported by the transpiler in use
       --transpiler-config-path path   (Optional) Local path to the configuration file of the transpiler to use for conversion
 
 Global Flags:

docs/lakebridge/docs/transpile/index.mdx

@@ -48,7 +48,9 @@ The `transpile` command will trigger the conversion of the specified code. These
  - `input-source`: The local filesystem path to the sources that should be transpiled. This must be provided if not set during `install-transpile`.
  - `output-folder`: The local filesystem path where converted code will be written. This must be provided if not set during `install-transpile`.
  - `source-dialect`: Dialect name (ex: snowflake, oracle, datastage, etc). This must be provided if not set during `install-transpile`.
- - `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written.
+ - `overrides-file`: An optional local path to a JSON file containing custom overrides for the transpilation process, if the underlying transpiler supports this. (Refer to [this documentation](pluggable_transpilers/bladebridge_configuration) for more details on custom overrides.)
+ - `target-technology`: The target technology to use for conversion output. This must be provided if not set during `install-transpile` and the underlying transpiler requires it for the source dialect in use.
+- `error-file-path`: The path to the file where a log of conversion errors will be stored. If not provided here or during `install-transpile` no error log will be written.
  - `skip-validation`: Whether the transpiler will skip the validation of transpiled SQL sources. If not provided here or during `install-transpile` validation will be attempted by default.
  - `catalog-name`: The name of the catalog in Databricks to use when validating transpiled SQL sources. If not specified, `remorph` will be used as the default catalog.
  - `schema-name`: The name of the schema in Databricks to use when validating transpiled SQL sources. If not specified, `transpiler` will be used as the default schema.

labs.yml

@@ -25,6 +25,10 @@ commands:
         description: (Optional) Local `path` to the configuration file of the transpiler to use for conversion
       - name: source-dialect
         description: (Optional) The source dialect to use when performing conversion
+      - name: overrides-file
+        description: (Optional) Local `path` of a file containing transpiler overrides, if supported by the transpiler in use
+      - name: target-technology
+        description: (Optional) Target technology to use for code generation, if supported by the transpiler in use
       - name: input-source
         description: (Optional) Local `path` of the sources to be convert
       - name: output-folder

src/databricks/labs/lakebridge/cli.py

@@ -16,14 +16,14 @@
 
 from databricks.labs.blueprint.cli import App
 from databricks.labs.blueprint.entrypoint import get_logger, is_in_debug
-from databricks.labs.blueprint.installation import RootJsonValue
+from databricks.labs.blueprint.installation import RootJsonValue, JsonObject, JsonValue
 from databricks.labs.blueprint.tui import Prompts
 
 
 from databricks.labs.lakebridge.assessments.configure_assessment import create_assessment_configurator
 from databricks.labs.lakebridge.assessments import PROFILER_SOURCE_SYSTEM
 
-from databricks.labs.lakebridge.config import TranspileConfig
+from databricks.labs.lakebridge.config import TranspileConfig, LSPConfigOptionV1
 from databricks.labs.lakebridge.contexts.application import ApplicationContext
 from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
 from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
@@ -84,26 +84,33 @@ def _remove_warehouse(ws: WorkspaceClient, warehouse_id: str):
 
 
 @lakebridge.command
-def transpile(
+def transpile(  # pylint: disable=too-many-arguments
     *,
     w: WorkspaceClient,
     transpiler_config_path: str | None = None,
     source_dialect: str | None = None,
+    overrides_file: str | None = None,
+    target_technology: str | None = None,
     input_source: str | None = None,
     output_folder: str | None = None,
     error_file_path: str | None = None,
     skip_validation: str | None = None,
     catalog_name: str | None = None,
     schema_name: str | None = None,
+    ctx: ApplicationContext | None = None,
     transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
 ):
     """Transpiles source dialect to databricks dialect"""
-    ctx = ApplicationContext(w)
+    if ctx is None:
+        ctx = ApplicationContext(w)
+    del w
     logger.debug(f"Preconfigured transpiler config: {ctx.transpile_config!r}")
     ctx.add_user_agent_extra("cmd", "execute-transpile")
     checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts, transpiler_repository)
     checker.use_transpiler_config_path(transpiler_config_path)
     checker.use_source_dialect(source_dialect)
+    checker.use_overrides_file(overrides_file)
+    checker.use_target_technology(target_technology)
     checker.use_input_source(input_source)
     checker.use_output_folder(output_folder)
     checker.use_error_file_path(error_file_path)
@@ -213,6 +220,36 @@ def use_source_dialect(self, source_dialect: str | None) -> None:
             logger.debug(f"Pending source_dialect override: {source_dialect!r}")
             self._source_dialect_override = source_dialect
 
+    @staticmethod
+    def _validate_overrides_file(overrides_file: str, msg: str) -> None:
+        """Validate the overrides file: it must be a valid path that exists."""
+        # Note: in addition to this check, later we verify the transpiler supports it.
+        if not Path(overrides_file).exists():
+            raise_validation_exception(msg)
+
+    def use_overrides_file(self, overrides_file: str | None) -> None:
+        if overrides_file is not None:
+            logger.debug(f"Setting overrides_file to: {overrides_file!r}")
+            msg = f"Invalid path for '--overrides-file', does not exist: {overrides_file}"
+            self._validate_overrides_file(overrides_file, msg)
+            try:
+                self._set_config_transpiler_option("overrides-file", overrides_file)
+            except ValueError:
+                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
+                msg = "Cannot use --overrides-file; workspace config.yml has incompatible transpiler_options."
+                raise_validation_exception(msg)
+
+    def use_target_technology(self, target_technology: str | None) -> None:
+        if target_technology is not None:
+            logger.debug(f"Setting target_technology to: {target_technology!r}")
+            # Cannot validate this here: depends on the transpiler engine, and will be checked later.
+            try:
+                self._set_config_transpiler_option("target-tech", target_technology)
+            except ValueError:
+                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
+                msg = "Cannot use --target-technology; workspace config.yml has incompatible transpiler_options."
+                raise_validation_exception(msg)
+
     @staticmethod
     def _validate_input_source(input_source: str, msg: str) -> None:
         """Validate the input source: it must be a path that exists."""
@@ -317,6 +354,19 @@ def use_schema_name(self, schema_name: str | None) -> None:
             logger.debug(f"Setting schema_name to: {schema_name!r}")
             self._config = dataclasses.replace(self._config, schema_name=schema_name)
 
+    def _set_config_transpiler_option(self, flag: str, value: str) -> None:
+        transpiler_options: JsonObject
+        match self._config.transpiler_options:
+            case None:
+                transpiler_options = {flag: value}
+            case Mapping() as found_options:
+                transpiler_options = {**found_options, flag: value}
+            case found_options:
+                # TODO: Update `config.yml' to constrain `transpiler_options` to be a dict[str, str].
+                msg = f"Incompatible transpiler options configured, must be a mapping: {found_options!r}"
+                raise ValueError(msg)
+        self._config = dataclasses.replace(self._config, transpiler_options=transpiler_options)
+
     def _configure_transpiler_config_path(self, source_dialect: str) -> TranspileEngine | None:
         """Configure the transpiler config path based on the requested source dialect."""
         # Names of compatible transpiler engines for the given dialect.
@@ -444,18 +494,52 @@ def _check_transpiler_options(self, engine: TranspileEngine) -> None:
         assert self._config.source_dialect is not None, "Source dialect must be set before checking transpiler options."
         options_for_dialect = engine.options_for_dialect(self._config.source_dialect)
         transpiler_options = self._config.transpiler_options
-        if not isinstance(transpiler_options, Mapping):
-            return
+        if transpiler_options is None:
+            transpiler_options = {}
+        elif not isinstance(transpiler_options, Mapping):
+            logger.warning(f"Ignoring transpiler_options in config.yml, must be a mapping: {transpiler_options!r}")
+            transpiler_options = {}
+        # Only checks if the option is present, does not validate the value.
+        # TODO: Validate the value for CHOICE/CONFIRM options.
+        # TODO: Handle FORCE options: these are fixed by the transpiler, and cannot be overridden.
         checked_options = {
             option.flag: (
                 transpiler_options[option.flag]
                 if option.flag in transpiler_options
-                else option.prompt_for_value(self._prompts)
+                else self._handle_missing_transpiler_option(option)
             )
             for option in options_for_dialect
         }
         self._config = dataclasses.replace(self._config, transpiler_options=checked_options)
 
+    def _handle_missing_transpiler_option(self, option: LSPConfigOptionV1) -> JsonValue:
+        # Semantics during configuration:
+        #  - Entries are present in the config file for all options the LSP server needs for a dialect.
+        #  - If a value is `None`, it means the user wants the value to be left unset.
+        #  - There is no 'provide it later' option: either it's set, or it's unset.
+        # As a corner case, if there is no entry present it means the user wasn't prompted. Here we have
+        # some complexity. We have two ways of obtaining a value:
+        #  - The user could provide it on the command-line, using --target-technology or --overrides-file.
+        #    Problem: via command-line options there's no way to indicate 'no value'.
+        #  - We could prompt for it, assuming the user is running interactively.
+        # In terms of what is required by the option:
+        #  - If the option has a default of <none>, it means that no value is required.
+        #  - Everything else requires a value.
+        #
+        # This leads to the following business rules:
+        #  - If the option has a default of <none> that means that no value is required, no further action is required.
+        #  - Otherwise, a value is required: prompt for it.
+        #
+        # TODO: When adding non-interactive support, the otherwise branch need to be modified:
+        #     1. If it can be provided by the command-line, fail and ask the user to provide it.
+        #     2. If it cannot be provided by the command-line, prompt for it if we are running interactively.
+        #     3. If we cannot prompt because we are not running interactively, use the default if there is one.
+        #     4. Fail: the only way to provide a value is via the config.yml, which can be set via 'install-transpile'.
+
+        if option.is_optional():
+            return None
+        return option.prompt_for_value(self._prompts)
+
     def check(self) -> tuple[TranspileConfig, TranspileEngine]:
         """Checks that all configuration parameters are present and valid."""
         logger.debug(f"Checking config: {self._config!r}")

src/databricks/labs/lakebridge/config.py

@@ -123,16 +123,30 @@ def parse(cls, data: JsonValue) -> "LSPConfigOptionV1":
 
         return LSPConfigOptionV1(flag, method, prompt, **optional)
 
+    def is_optional(self) -> bool:
+        # Semantics are currently that a value for an option is always required, except in the specific case of:
+        #  - It being a QUESTION; AND
+        #  - The default is set to the special "<none>" value.
+        return self.method == LSPPromptMethod.QUESTION and self.default == self._question_optional_sentinel
+
+    # Magic value that indicates no answer is required for a QUESTION prompt.
+    _question_optional_sentinel = "<none>"
+
     def prompt_for_value(self, prompts: Prompts) -> JsonValue:
         if self.method == LSPPromptMethod.FORCE:
             return self.default
         assert self.prompt is not None
         if self.method == LSPPromptMethod.CONFIRM:
             return prompts.confirm(self.prompt)
         if self.method == LSPPromptMethod.QUESTION:
-            default = self.default if self.default else "None"
+            # Hack to:
+            #  - trick prompts.question() into indicating that no answer is required;
+            #  - allow no answer to be given.
+            # Normally prompts.confirm() requires an answer, or returns the default, and the default can't be None.
+            # Note: LSP servers use '<none>' as a default to indicate that no answer is required.
+            default = self.default if self.default else self._question_optional_sentinel
             result = prompts.question(self.prompt, default=default)
-            if result == "<none>":
+            if result == self._question_optional_sentinel:
                 return None
             return result
         if self.method == LSPPromptMethod.CHOICE:

src/databricks/labs/lakebridge/install.py

@@ -262,6 +262,11 @@ def _prompt_for_transpiler_options(self, transpiler_name: str, source_dialect: s
         config_options = self._transpiler_repository.transpiler_config_options(transpiler_name, source_dialect)
         if len(config_options) == 0:
             return None
+        # Semantics here are different from the other properties of a TranspileConfig. Specifically:
+        #  - Entries are present for all options.
+        #  - If the value is None, it means the user chose to not provide a value. (This differs to other
+        #    attributes, where None means the user chose to provide a value later.)
+        #  - There is no way to express 'provide a value later'.
         return {option.flag: option.prompt_for_value(self._prompts) for option in config_options}
 
     def _configure_catalog(self) -> str:

tests/conftest.py

@@ -284,7 +284,7 @@ def mock_data_source():
     return MockDataSource({}, {})
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def bladebridge_artifact() -> Path:
     artifact = (
         Path(__file__).parent
@@ -298,7 +298,7 @@ def bladebridge_artifact() -> Path:
     return artifact
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def morpheus_artifact() -> Path:
     artifact = (
         Path(__file__).parent

tests/integration/transpile/common_utils.py

@@ -4,14 +4,10 @@
 
 
 def assert_sql_outputs(output_folder: Path, expected_sql: str, expected_failure_sql: str) -> None:
-    assert (output_folder / "create_ddl.sql").exists()
-    with open(output_folder / "create_ddl.sql", "r", encoding="utf-8") as f:
-        actual_sql = f.read()
-    assert actual_sql.strip() == expected_sql.strip()
+    actual_sql = (output_folder / "create_ddl.sql").read_text(encoding="utf-8")
+    actual_failure_sql = (output_folder / "dummy_function.sql").read_text(encoding="utf-8")
 
-    assert (output_folder / "dummy_function.sql").exists()
-    with open(output_folder / "dummy_function.sql", "r", encoding="utf-8") as f:
-        actual_failure_sql = f.read()
+    assert actual_sql.strip() == expected_sql.strip()
     assert actual_failure_sql.strip() == expected_failure_sql.strip()