infer type from location, default to parquet if not set

duckdb · Oct 17, 2024 · 1cf821e · 1cf821e
1 parent 1a5155a
commit 1cf821e
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/dbt/include/duckdb/macros/materializations/external.sql b/dbt/include/duckdb/macros/materializations/external.sql
@@ -2,7 +2,17 @@
 
   {%- set location = render(config.get('location', default=external_location(this, config))) -%})
   {%- set rendered_options = render_write_options(config) -%}
-  {%- set format = config.get('format', 'parquet') -%}
+
+  {%- set format = config.get('format') -%}
+  {%- set allowed_formats = ['csv', 'parquet', 'json'] -%}
+
+  {%- if format -%}
+      {%- set format = format if format in allowed_formats else 'parquet' -%}
+  {%- else -%}
+      {%- set format = location.split('.')[-1] if '.' in location else 'parquet' -%}
+      {%- set format = format if format in allowed_formats else 'parquet' -%}
+  {%- endif -%}
+
   {%- set write_options = adapter.external_write_options(location, rendered_options) -%}
   {%- set read_location = adapter.external_read_location(location, rendered_options) -%}
   {%- set parquet_read_options = config.get('parquet_read_options', {'union_by_name': False}) -%}

diff --git a/tests/functional/adapter/test_external.py b/tests/functional/adapter/test_external.py
@@ -23,11 +23,11 @@
 """
 
 config_materialized_parquet_location = """
-  {{ config(materialized="external", location="{{ adapter.external_root() }}/test.parquet", format="parquet") }}
+  {{ config(materialized="external", location="{{ adapter.external_root() }}/test.parquet") }}
 """
 
 config_materialized_csv_location_delim = """
-  {{ config(materialized="external", location="{{ adapter.external_root() }}/test_delim.csv", delimiter="|", format="csv") }}
+  {{ config(materialized="external", location="{{ adapter.external_root() }}/test_delim.csv", delimiter="|") }}
 """
 
 config_json = """