Merge pull request #432 from duckdb/add_s3_snowflake_and_fix_glue_tests

Add s3 snowflake and fix glue tests

Author: Tmonster

.github/workflows/CloudTesting.yml

@@ -58,7 +58,7 @@ jobs:
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
           R2_TOKEN: ${{secrets.r2_token}}
         run: | 
-          python3 scripts/create_s3_insert_table.py --action=delete-and-create --catalogs=s3tables,r2
+          python3 scripts/create_s3_insert_table.py --action=delete-and-create --catalogs=s3tables,r2,glue
 
       - name: Test with rest catalog
         env:
@@ -67,6 +67,8 @@ jobs:
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
           SNOWFLAKE_KEY_ID_GCS: ${{secrets.SNOWFLAKE_KEY_ID_GCS}}
           SNOWFLAKE_SECRET_KEY_GCS: ${{secrets.SNOWFLAKE_SECRET_KEY_GCS}}
+          SNOWFLAKE_KEY_ID_S3: ${{secrets.SNOWFLAKE_KEY_ID_S3}}
+          SNOWFLAKE_SECRET_KEY_S3: ${{secrets.SNOWFLAKE_SECRET_KEY_S3}}
           SNOWFLAKE_CATALOG_URI_GCS: ${{secrets.SNOWFLAKE_CATALOG_URI_GCS}}
           R2_TOKEN: ${{secrets.r2_token}}
           ICEBERG_REMOTE_INSERT_READY: 1
@@ -76,7 +78,7 @@ jobs:
           make test_release
 
       - name: File issue if error
-        if: failure()
+        if: ${{ contains(github.ref_name, 'main') && failure() }}
         env:
           GH_TOKEN: ${{ github.token }}
         run: |

test/sql/cloud/glue/test_create_table_glue.test

@@ -18,10 +18,6 @@ require httpfs
 
 require aws
 
-# credentials in CI cannot set up the environment for this test
-# need to give the crentials glue:DropTable priviledges
-mode skip
-
 statement ok
 CREATE SECRET (
   TYPE S3,

test/sql/cloud/glue/test_direct_keys_glue.test

@@ -18,11 +18,6 @@ require httpfs
 
 require aws
 
-# TODO: re-enable these tests once we know what account has these
-#  credentials, and we can grant them access to the glue catalog
-# test using keys directory
-mode skip
-
 statement ok
 CREATE SECRET s1 (
       TYPE S3,
@@ -37,10 +32,6 @@ attach '840140254803:s3tablescatalog/pyiceberg-blog-bucket' as my_datalake (
     ENDPOINT_TYPE 'GLUE'
 );
 
-query T nosort tables_1
-show all tables;
-----
-
 statement ok
 SELECT count(*) FROM my_datalake.myblognamespace.lineitem;
 

test/sql/cloud/glue/test_direct_keys_glue_no_endpoint_type.test

@@ -18,10 +18,6 @@ require httpfs
 
 require aws
 
-# TODO: re-enable these tests once we know what account has these
-#  credentials, and we can grant them access to the glue catalog
-mode skip
-
 # test using keys directory
 statement ok
 CREATE SECRET s1 (
@@ -38,10 +34,6 @@ attach '840140254803:s3tablescatalog/pyiceberg-blog-bucket' as my_datalake (
 	ENDPOINT 'glue.us-east-1.amazonaws.com/iceberg'
 );
 
-query T nosort tables_1
-show all tables;
-----
-
 statement ok
 SELECT count(*) FROM my_datalake.myblognamespace.lineitem;
 
@@ -51,8 +43,6 @@ drop secret s1;
 statement ok
 detach my_datalake;
 
-mode unskip
-
 # test using assume role
 statement ok
 CREATE SECRET assume_role_secret (

test/sql/cloud/glue/test_insert_glue.test

@@ -20,10 +20,6 @@ require httpfs
 
 require aws
 
-# credentials in CI cannot set up the environment for this test
-# need to give the crentials glue:DropTable priviledges
-mode skip
-
 statement ok
 CREATE SECRET (
     TYPE S3,

test/sql/cloud/snowflake/test_snowflake.test

@@ -40,9 +40,14 @@ attach 'GCS_catalog' as my_datalake (
 	ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
 );
 
+statement ok
+create schema if not exists my_datalake.test_create_schema;
+
+statement ok
+create table my_datalake.test_create_schema.table1 as select range a from range(10);
 
 query I
-select * from my_datalake.default.duckdb_created_table;
+select * from my_datalake.test_create_schema.table1;
 ----
 0
 1
@@ -54,3 +59,9 @@ select * from my_datalake.default.duckdb_created_table;
 7
 8
 9
+
+statement ok
+drop table if exists my_datalake.test_create_schema.table1;
+
+statement ok
+drop schema if exists my_datalake.test_create_schema;

test/sql/cloud/snowflake/test_snowflake_s3.test

@@ -0,0 +1,64 @@
+# name: test/sql/cloud/snowflake/test_snowflake_s3.test
+# group: [snowflake]
+
+require-env SNOWFLAKE_KEY_ID_S3
+
+require-env SNOWFLAKE_SECRET_ID_S3
+
+require-env SNOWFLAKE_CATALOG_URI_GCS
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+require aws
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+statement ok
+create secret polaris_secret (
+        TYPE ICEBERG,
+        CLIENT_ID '${SNOWFLAKE_KEY_ID_S3}',
+        CLIENT_SECRET '${SNOWFLAKE_SECRET_KEY_S3}',
+        ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+    );
+
+
+statement ok
+attach 's3-catalog' as my_datalake (
+    type ICEBERG,
+    default_region 'eu-west-2',
+    ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+ );
+
+
+statement ok
+create schema if not exists my_datalake.test_create_schema;
+
+statement ok
+create table my_datalake.test_create_schema.table1 as select range a from range(10);
+
+query I
+select * from my_datalake.test_create_schema.table1;
+----
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+statement ok
+drop table if exists my_datalake.test_create_schema.table1;
+
+statement ok
+drop schema if exists my_datalake.test_create_schema;