From 749bd6c9de26d36cb1b41c0424d3ba30904c988d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 14 Jan 2025 14:45:31 +0100 Subject: [PATCH 01/11] use path from java.io.File instead of raw user provided string to enable triple slash file:/// --- .../net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index f003e763f..51040f820 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1852,7 +1852,7 @@ private void uploadFiles(Set fileList, int parallel) throws SnowflakeSQL threadExecutor.submit( getUploadFileCallable( stageInfo, - srcFile, + srcFileObj.getPath(), fileMetadata, (stageInfo.getStageType() == StageInfo.StageType.LOCAL_FS) ? null From b17cd506f5c9e62c3b37b329231436fa20c9e6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 31 Jan 2025 12:55:01 +0100 Subject: [PATCH 02/11] add test for triple slash, comment out the fix to test on windows --- .../jdbc/SnowflakeFileTransferAgent.java | 3 ++- .../client/jdbc/FileUploaderLatestIT.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 7db10665f..205e82563 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1865,7 +1865,8 @@ private void uploadFiles(Set fileList, int parallel) throws SnowflakeSQL threadExecutor.submit( getUploadFileCallable( stageInfo, - srcFileObj.getPath(), + // srcFileObj.getPath(), + srcFile, fileMetadata, (stageInfo.getStageType() == StageInfo.StageType.LOCAL_FS) ? null diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index a116a794b..904bf4451 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -871,4 +871,23 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { FileUtils.deleteDirectory(subDir.toFile()); } } + + @Test + public void testUploadWithTripleSlashFilePrefix() throws SQLException { + try (Connection connection = getConnection(); + Statement statement = connection.createStatement()) { + try { + statement.execute("CREATE OR REPLACE STAGE testStage"); + SFSession sfSession = connection.unwrap(SnowflakeConnectionV1.class).getSfSession(); + + String command = "PUT file:///" + getFullPathFileInResource(TEST_DATA_FILE) + " @testStage"; + SnowflakeFileTransferAgent sfAgent = + new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); + assertTrue(sfAgent.execute()); + } finally { + statement.execute("DROP STAGE if exists testStage"); + } + } + SnowflakeFileTransferAgent.setInjectedFileTransferException(null); + } } From 6433ed2487d2c8dc32ced100d6be7b52b88bb3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 31 Jan 2025 14:03:21 +0100 Subject: [PATCH 03/11] add get to put test to verify if uploaded file can be downloaded --- .../net/snowflake/client/jdbc/FileUploaderLatestIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 904bf4451..a05be722c 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -884,6 +884,12 @@ public void testUploadWithTripleSlashFilePrefix() throws SQLException { SnowflakeFileTransferAgent sfAgent = new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); + + String getCommand = "GET @testStage file:///tmp"; + SnowflakeFileTransferAgent sfAgent1 = + new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); + assertTrue(sfAgent1.execute()); + assertEquals(1, sfAgent1.statusRows.size()); } finally { statement.execute("DROP STAGE if exists testStage"); } From b9694702778641eaca3c6f67f7f755063ea726f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 31 Jan 2025 14:51:41 +0100 Subject: [PATCH 04/11] re-enable fix, format everything --- .../net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java | 3 +-- .../java/net/snowflake/client/jdbc/FileUploaderLatestIT.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 205e82563..7db10665f 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1865,8 +1865,7 @@ private void uploadFiles(Set fileList, int parallel) throws SnowflakeSQL threadExecutor.submit( getUploadFileCallable( stageInfo, - // srcFileObj.getPath(), - srcFile, + srcFileObj.getPath(), fileMetadata, (stageInfo.getStageType() == StageInfo.StageType.LOCAL_FS) ? null diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index a05be722c..c2dd7cca3 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -887,7 +887,7 @@ public void testUploadWithTripleSlashFilePrefix() throws SQLException { String getCommand = "GET @testStage file:///tmp"; SnowflakeFileTransferAgent sfAgent1 = - new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); + new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent1.execute()); assertEquals(1, sfAgent1.statusRows.size()); } finally { From 72617fbbe5e77370944c87987ca3b061163cd68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Fri, 31 Jan 2025 17:10:10 +0100 Subject: [PATCH 05/11] use tempdir and randomized stage name, remove obsolete exception injection --- .../client/jdbc/FileUploaderLatestIT.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index c2dd7cca3..0d51c538f 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -48,6 +48,7 @@ import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; /** Tests for SnowflakeFileTransferAgent that require an active connection */ @Tag(TestTags.OTHERS) @@ -873,27 +874,28 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { } @Test - public void testUploadWithTripleSlashFilePrefix() throws SQLException { + public void testUploadWithTripleSlashFilePrefix(@TempDir Path tempDir) throws SQLException { + String stageName = "testStage" + SnowflakeUtil.randomAlphaNumeric(10); try (Connection connection = getConnection(); Statement statement = connection.createStatement()) { try { - statement.execute("CREATE OR REPLACE STAGE testStage"); + statement.execute("CREATE OR REPLACE STAGE " + stageName); SFSession sfSession = connection.unwrap(SnowflakeConnectionV1.class).getSfSession(); - String command = "PUT file:///" + getFullPathFileInResource(TEST_DATA_FILE) + " @testStage"; + String command = + "PUT file:///" + getFullPathFileInResource(TEST_DATA_FILE) + " @" + stageName; SnowflakeFileTransferAgent sfAgent = new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String getCommand = "GET @testStage file:///tmp"; + String getCommand = "GET @" + stageName + " file://" + tempDir; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent1.execute()); assertEquals(1, sfAgent1.statusRows.size()); } finally { - statement.execute("DROP STAGE if exists testStage"); + statement.execute("DROP STAGE if exists " + stageName); } } - SnowflakeFileTransferAgent.setInjectedFileTransferException(null); } } From e5c67ec0b2320d46b989584dbfe9175c52363aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Mon, 3 Feb 2025 16:15:21 +0100 Subject: [PATCH 06/11] use File instead of Path in TempDir --- .../java/net/snowflake/client/jdbc/FileUploaderLatestIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 0648d166b..57e014987 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -888,7 +888,7 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { } @Test - public void testUploadWithTripleSlashFilePrefix(@TempDir Path tempDir) throws SQLException { + public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQLException { String stageName = "testStage" + SnowflakeUtil.randomAlphaNumeric(10); try (Connection connection = getConnection(); Statement statement = connection.createStatement()) { @@ -902,7 +902,7 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir Path tempDir) throws SQ new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String getCommand = "GET @" + stageName + " file://" + tempDir; + String getCommand = "GET @" + stageName + " file:///" + tempDir; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent1.execute()); From a2aa9c317e4dc58e37f8b55b1f4bc5b48b32ceb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 4 Feb 2025 10:09:54 +0100 Subject: [PATCH 07/11] sanitize path for windows --- .../java/net/snowflake/client/jdbc/FileUploaderLatestIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 57e014987..fa13a93ef 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -902,7 +902,8 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQ new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String getCommand = "GET @" + stageName + " file:///" + tempDir; + String tempDirPath = tempDir.getPath().replace("\\", "/"); + String getCommand = "GET @" + stageName + " file:///" + tempDirPath; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent1.execute()); From 3ca9d2671a9137e851e752156c37f23551c2d78f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 4 Feb 2025 11:10:05 +0100 Subject: [PATCH 08/11] sanitize path for windows pt2 --- .../net/snowflake/client/jdbc/FileUploaderLatestIT.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index fa13a93ef..0acddbc20 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -902,7 +902,11 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQ new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String tempDirPath = tempDir.getPath().replace("\\", "/"); + String tempDirPath = + tempDir + .getPath() + .replace("\\", "/") + .replaceFirst("^~", System.getProperty("user.home")); String getCommand = "GET @" + stageName + " file:///" + tempDirPath; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); From 19b94f62a1425512fd2b9844e0580a6a28f6481e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 4 Feb 2025 12:42:23 +0100 Subject: [PATCH 09/11] add tempdir factory --- .../client/jdbc/FileUploaderLatestIT.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 0acddbc20..8a7d09abb 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -49,7 +49,10 @@ import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.AnnotatedElementContext; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.io.TempDirFactory; /** Tests for SnowflakeFileTransferAgent that require an active connection */ @Tag(TestTags.OTHERS) @@ -888,7 +891,7 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { } @Test - public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQLException { + public void testUploadWithTripleSlashFilePrefix(@TempDir(factory = GetTempDirFactory.class) File tempDir) throws SQLException { String stageName = "testStage" + SnowflakeUtil.randomAlphaNumeric(10); try (Connection connection = getConnection(); Statement statement = connection.createStatement()) { @@ -902,11 +905,7 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQ new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String tempDirPath = - tempDir - .getPath() - .replace("\\", "/") - .replaceFirst("^~", System.getProperty("user.home")); + String tempDirPath = tempDir.getPath().replace("\\", "/"); String getCommand = "GET @" + stageName + " file:///" + tempDirPath; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); @@ -917,4 +916,11 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) throws SQ } } } + + static class GetTempDirFactory implements TempDirFactory { + @Override + public Path createTempDirectory(AnnotatedElementContext elementContext, ExtensionContext extensionContext) throws Exception { + return Files.createTempDirectory(extensionContext.getRequiredTestMethod().getName()); + } + } } From c8ab55091b14048717fcf9eee321de655d3d839b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Tue, 4 Feb 2025 17:07:57 +0100 Subject: [PATCH 10/11] use canonical path --- .../net/snowflake/client/jdbc/FileUploaderLatestIT.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 8a7d09abb..3dd665d37 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -891,7 +891,8 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { } @Test - public void testUploadWithTripleSlashFilePrefix(@TempDir(factory = GetTempDirFactory.class) File tempDir) throws SQLException { + public void testUploadWithTripleSlashFilePrefix( + @TempDir(factory = GetTempDirFactory.class) File tempDir) throws SQLException, IOException { String stageName = "testStage" + SnowflakeUtil.randomAlphaNumeric(10); try (Connection connection = getConnection(); Statement statement = connection.createStatement()) { @@ -905,7 +906,7 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir(factory = GetTempDirFac new SnowflakeFileTransferAgent(command, sfSession, new SFStatement(sfSession)); assertTrue(sfAgent.execute()); - String tempDirPath = tempDir.getPath().replace("\\", "/"); + String tempDirPath = tempDir.getCanonicalPath().replace("\\", "/"); String getCommand = "GET @" + stageName + " file:///" + tempDirPath; SnowflakeFileTransferAgent sfAgent1 = new SnowflakeFileTransferAgent(getCommand, sfSession, new SFStatement(sfSession)); @@ -919,7 +920,9 @@ public void testUploadWithTripleSlashFilePrefix(@TempDir(factory = GetTempDirFac static class GetTempDirFactory implements TempDirFactory { @Override - public Path createTempDirectory(AnnotatedElementContext elementContext, ExtensionContext extensionContext) throws Exception { + public Path createTempDirectory( + AnnotatedElementContext elementContext, ExtensionContext extensionContext) + throws Exception { return Files.createTempDirectory(extensionContext.getRequiredTestMethod().getName()); } } From 8a695747376dff3eddb0ef3c06e42247b971844b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Kubik?= Date: Wed, 5 Feb 2025 09:19:40 +0100 Subject: [PATCH 11/11] remove tempdir factory as it's obsolete --- .../client/jdbc/FileUploaderLatestIT.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java index 3dd665d37..c75fa063a 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderLatestIT.java @@ -49,10 +49,7 @@ import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.AnnotatedElementContext; -import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.api.io.TempDirFactory; /** Tests for SnowflakeFileTransferAgent that require an active connection */ @Tag(TestTags.OTHERS) @@ -891,8 +888,8 @@ public void testUploadWithTildeInPath() throws SQLException, IOException { } @Test - public void testUploadWithTripleSlashFilePrefix( - @TempDir(factory = GetTempDirFactory.class) File tempDir) throws SQLException, IOException { + public void testUploadWithTripleSlashFilePrefix(@TempDir File tempDir) + throws SQLException, IOException { String stageName = "testStage" + SnowflakeUtil.randomAlphaNumeric(10); try (Connection connection = getConnection(); Statement statement = connection.createStatement()) { @@ -917,13 +914,4 @@ public void testUploadWithTripleSlashFilePrefix( } } } - - static class GetTempDirFactory implements TempDirFactory { - @Override - public Path createTempDirectory( - AnnotatedElementContext elementContext, ExtensionContext extensionContext) - throws Exception { - return Files.createTempDirectory(extensionContext.getRequiredTestMethod().getName()); - } - } }