diff --git a/cpp/FileMetadataInitializer.cpp b/cpp/FileMetadataInitializer.cpp index 5d1047ce20..46eaf7d8a5 100755 --- a/cpp/FileMetadataInitializer.cpp +++ b/cpp/FileMetadataInitializer.cpp @@ -8,6 +8,7 @@ #include "snowflake/platform.h" #include "snowflake/SnowflakeTransferException.hpp" #include +#include "boost/filesystem.hpp" #define COMPRESSION_AUTO "AUTO" #define COMPRESSION_AUTO_DETECT "AUTO_DETECT" @@ -20,7 +21,7 @@ #include #endif - +using namespace boost::filesystem; Snowflake::Client::FileMetadataInitializer::FileMetadataInitializer( std::vector &smallFileMetadata, @@ -34,16 +35,16 @@ Snowflake::Client::FileMetadataInitializer::FileMetadataInitializer( } void -Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::string &fileDir, const char *fileName, +Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::string &fileNameFull, + const std::string &destPath, + const std::string &fileName, size_t fileSize, size_t threshold) { - std::string fileNameFull = fileDir; - fileNameFull += fileName; - FileMetadata fileMetadata; fileMetadata.srcFileName = m_stmtPutGet->platformStringToUTF8(fileNameFull); fileMetadata.srcFileSize = fileSize; - fileMetadata.destFileName = m_stmtPutGet->platformStringToUTF8(std::string(fileName)); + fileMetadata.destPath = m_stmtPutGet->platformStringToUTF8(destPath); + fileMetadata.destFileName = m_stmtPutGet->platformStringToUTF8(fileName); // process compression type initCompressionMetadata(fileMetadata); @@ -56,9 +57,54 @@ Snowflake::Client::FileMetadataInitializer::initUploadFileMetadata(const std::st void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(std::string &sourceLocation, size_t putThreshold) +{ + // looking for files on disk. + std::string srcLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(sourceLocation); + replaceStrAll(srcLocationPlatform, "/", std::string() + PATH_SEP); + size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP); + std::string basePath = srcLocationPlatform.substr(0, dirSep + 1); + + std::vector fileList; + if (!listFiles(srcLocationPlatform, fileList)) + { + CXX_LOG_ERROR("Failed on finding files for uploading."); + return; + } + + for (auto file = fileList.begin(); file != fileList.end(); file++) + { + path p(*file); + size_t fileSize = file_size(p); + std::string fileNameFull = p.string(); + std::string fileName = p.filename().string(); + //make the path on stage by removing base path and file name from full path + std::string destPath = fileNameFull.substr(basePath.length(), + fileNameFull.length() - basePath.length() - fileName.length()); + initUploadFileMetadata(fileNameFull, destPath, fileName, fileSize, putThreshold); + } +} + +void Snowflake::Client::FileMetadataInitializer::includeSubfolderFilesRecursive(const std::string &folderPath, + std::vector & fileList) +{ + for (auto const& entry : recursive_directory_iterator(folderPath)) + { + if (is_regular_file(entry)) + { + fileList.push_back(entry.path().string()); + } + } +} + +bool Snowflake::Client::FileMetadataInitializer::listFiles(const std::string &sourceLocation, + std::vector & fileList) { // looking for files on disk. std::string srcLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(sourceLocation); + size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP); + std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1); + std::string filePattern = srcLocationPlatform.substr(dirSep + 1); + bool includeSubfolder = filePattern == "**"; #ifdef _WIN32 WIN32_FIND_DATA fdd; @@ -71,8 +117,7 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st { CXX_LOG_ERROR("No file matching pattern %s has been found. Error: %d", sourceLocation.c_str(), dwError); - FindClose(hFind); - return; + return false; } else if (dwError != ERROR_SUCCESS) { @@ -85,37 +130,29 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st do { if (!(fdd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ) { - std::string fileFullPath = std::string(fdd.cFileName); - size_t dirSep = srcLocationPlatform.find_last_of(PATH_SEP); - if (dirSep == std::string::npos) - { - dirSep = sourceLocation.find_last_of(ALTER_PATH_SEP); - } - if (dirSep != std::string::npos) + fileList.push_back(dirPath + fdd.cFileName); + } + else + { + if (includeSubfolder && + (std::string(fdd.cFileName) != ".") && + (std::string(fdd.cFileName) != "..")) { - std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1); - LARGE_INTEGER fileSize; - fileSize.LowPart = fdd.nFileSizeLow; - fileSize.HighPart = fdd.nFileSizeHigh; - initUploadFileMetadata(dirPath, (char *)fdd.cFileName, (size_t)fileSize.QuadPart, putThreshold); + includeSubfolderFilesRecursive(dirPath + fdd.cFileName, fileList); } } } while (FindNextFile(hFind, &fdd) != 0); DWORD dwError = GetLastError(); + FindClose(hFind); if (dwError != ERROR_NO_MORE_FILES) { CXX_LOG_ERROR("Failed on FindNextFile. Error: %d", dwError); throw SnowflakeTransferException(TransferError::DIR_OPEN_ERROR, srcLocationPlatform.c_str(), dwError); } - FindClose(hFind); #else - unsigned long dirSep = srcLocationPlatform.find_last_of(PATH_SEP); - std::string dirPath = srcLocationPlatform.substr(0, dirSep + 1); - std::string filePattern = srcLocationPlatform.substr(dirSep + 1); - DIR * dir = nullptr; struct dirent * dir_entry; if ((dir = opendir(dirPath.c_str())) != NULL) @@ -130,8 +167,14 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st if (!ret) { if (S_ISREG(fileStatus.st_mode)) { - initUploadFileMetadata(dirPath, dir_entry->d_name, - (size_t) fileStatus.st_size, putThreshold); + fileList.push_back(dirPath + dir_entry->d_name); + } + else if (includeSubfolder && + (S_ISDIR(fileStatus.st_mode)) && + (std::string(dir_entry->d_name) != ".") && + (std::string(dir_entry->d_name) != "..")) + { + includeSubfolderFilesRecursive(dirPath + dir_entry->d_name, fileList); } } else @@ -153,6 +196,7 @@ void Snowflake::Client::FileMetadataInitializer::populateSrcLocUploadMetadata(st dirPath.c_str(), errno); } #endif + return true; } void Snowflake::Client::FileMetadataInitializer::initCompressionMetadata( @@ -168,8 +212,10 @@ void Snowflake::Client::FileMetadataInitializer::initCompressionMetadata( { // guess CXX_LOG_INFO("Auto detect on compression type"); - fileMetadata.sourceCompression = FileCompressionType::guessCompressionType( + std::string srcFileNamePlatform = m_stmtPutGet->UTF8ToPlatformString( fileMetadata.srcFileName); + fileMetadata.sourceCompression = FileCompressionType::guessCompressionType( + srcFileNamePlatform); } else if (!sf_strncasecmp(m_sourceCompression, COMPRESSION_NONE, sizeof(COMPRESSION_NONE))) @@ -253,8 +299,9 @@ populateSrcLocDownloadMetadata(std::string &sourceLocation, size_t getThreshold) { std::string fullPath = *remoteLocation + sourceLocation; - size_t dirSep = fullPath.find_last_of('/'); - std::string dstFileName = fullPath.substr(dirSep + 1); + size_t dirSep = sourceLocation.find_last_of('/'); + std::string dstFileName = sourceLocation.substr(dirSep + 1); + std::string dstPath = sourceLocation.substr(0, dirSep + 1); FileMetadata fileMetadata; fileMetadata.presignedUrl = presignedUrl; @@ -271,6 +318,7 @@ populateSrcLocDownloadMetadata(std::string &sourceLocation, metaListToPush.push_back(fileMetadata); metaListToPush.back().srcFileName = fullPath; metaListToPush.back().destFileName = dstFileName; + metaListToPush.back().destPath = dstPath; if (encMat) { EncryptionProvider::decryptFileKey(&(metaListToPush.back()), encMat, getRandomDev()); @@ -284,4 +332,29 @@ populateSrcLocDownloadMetadata(std::string &sourceLocation, return outcome; } +void Snowflake::Client::FileMetadataInitializer:: +replaceStrAll(std::string& stringToReplace, + std::string const& oldValue, + std::string const& newValue) +{ + size_t oldValueLen = oldValue.length(); + size_t newValueLen = newValue.length(); + if (0 == oldValueLen) + { + return; + } + + size_t index = 0; + while (true) { + /* Locate the substring to replace. */ + index = stringToReplace.find(oldValue, index); + if (index == std::string::npos) break; + + /* Make the replacement. */ + stringToReplace.replace(index, oldValueLen, newValue); + + /* Advance index forward so the next iteration doesn't pick it up as well. */ + index += newValueLen; + } +} diff --git a/cpp/FileMetadataInitializer.hpp b/cpp/FileMetadataInitializer.hpp index f60239d1f8..2a6cb7df9e 100755 --- a/cpp/FileMetadataInitializer.hpp +++ b/cpp/FileMetadataInitializer.hpp @@ -30,11 +30,38 @@ class FileMetadataInitializer IStatementPutGet *stmtPutGet); /** - * Given a source locations, find all files that match the location pattern, + * Given a source location, find all files that match the location pattern, * init file metadata, and divide them into different vector according to size */ void populateSrcLocUploadMetadata(std::string &sourceLocation, size_t putThreshold); + + /** + * Utility function to replace all matching instances in a string. + */ + static void replaceStrAll(std::string& stringToReplace, std::string const& oldValue, + std::string const& newValue); + /** + * Given a source location, find all files match the partern, recursively include + * all subfolders if the pattern is ** + * Utility function called from populateSrcLocUploadMetadata. + * + * @param sourceLocation The source location could have pattern at the end. + * @param fileList Output the files with the full path. + * + * @return True when succeeded, false when no file matches with the source location. + * @throw SnowflakeTransferException on unexpected error. + */ + bool listFiles(const std::string &sourceLocation, std::vector & fileList); + + /** + * Given a full path of a folder, add all files in the folder recursively including subfolders. + * + * @param folderPath The full path of a folder. + * @param fileList Output the files in the folder recursively including subfolders. + */ + void includeSubfolderFilesRecursive(const std::string &folderPath, std::vector & fileList); + /** * Given a source location, find out file size to determine use parallel * download or not. @@ -79,7 +106,8 @@ class FileMetadataInitializer * Given file name, populate metadata * @param fileName */ - void initUploadFileMetadata(const std::string &fileDir, const char *fileName, size_t fileSize, size_t threshold); + void initUploadFileMetadata(const std::string &fileNameFull, const std::string &destPath, + const std::string &fileName, size_t fileSize, size_t threshold); /** * init compression metadata diff --git a/cpp/FileTransferAgent.cpp b/cpp/FileTransferAgent.cpp index 0323874b1b..ccc76a106f 100755 --- a/cpp/FileTransferAgent.cpp +++ b/cpp/FileTransferAgent.cpp @@ -33,31 +33,6 @@ using ::Snowflake::Client::RemoteStorageRequestOutcome; namespace { const std::string FILE_PROTOCOL = "file://"; - - void replaceStrAll(std::string& stringToReplace, - std::string const& oldValue, - std::string const& newValue) - { - size_t oldValueLen = oldValue.length(); - size_t newValueLen = newValue.length(); - if (0 == oldValueLen) - { - return; - } - - size_t index = 0; - while (true) { - /* Locate the substring to replace. */ - index = stringToReplace.find(oldValue, index); - if (index == std::string::npos) break; - - /* Make the replacement. */ - stringToReplace.replace(index, oldValueLen, newValue); - - /* Advance index forward so the next iteration doesn't pick it up as well. */ - index += newValueLen; - } - } } Snowflake::Client::FileTransferAgent::FileTransferAgent( @@ -468,6 +443,12 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::uploadSingleFi fileMetadata->srcFileToUpload = fileMetadata->srcFileName; fileMetadata->srcFileToUploadSize = fileMetadata->srcFileSize; } + + // after compress replace PATH_SEP with / in destPath as that's + // what needed on stage side + FileMetadataInitializer::replaceStrAll(fileMetadata->destPath, std::string() + PATH_SEP, "/"); + fileMetadata->destFileName = fileMetadata->destPath + fileMetadata->destFileName; + CXX_LOG_TRACE("Update File digest metadata start"); // calculate digest @@ -614,6 +595,24 @@ void Snowflake::Client::FileTransferAgent::compressSourceFile( } std::string stagingFile(tempDir); + + if (!fileMetadata->destPath.empty()) + { + std::string subfolder = fileMetadata->destPath; + FileMetadataInitializer::replaceStrAll(subfolder, "/", std::string() + PATH_SEP); + std::string subfolderPlatform = m_stmtPutGet->UTF8ToPlatformString(subfolder); + subfolder = std::string(tempDir) + subfolder; + subfolderPlatform = std::string(tempDir) + subfolderPlatform; + + int ret = sf_create_directory_if_not_exists_recursive(subfolderPlatform.c_str()); + if (ret != 0) + { + CXX_LOG_ERROR("Failed to create temporary folder %s. Errno: %d", subfolder, errno); + throw SnowflakeTransferException(TransferError::FILE_OPEN_ERROR, subfolder, -1); + } + stagingFile = subfolderPlatform; + } + stagingFile += m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destFileName); std::string srcFileNamePlatform = m_stmtPutGet->UTF8ToPlatformString(fileMetadata->srcFileName); @@ -647,12 +646,13 @@ void Snowflake::Client::FileTransferAgent::download(string *command) m_executionResults = new FileTransferExecutionResult(CommandType::DOWNLOAD, m_largeFilesMeta.size() + m_smallFilesMeta.size()); - int ret = sf_create_directory_if_not_exists((const char *)response.localLocation); + std::string localLocationPlatform = m_stmtPutGet->UTF8ToPlatformString(response.localLocation); + int ret = sf_create_directory_if_not_exists_recursive((const char *)localLocationPlatform.c_str()); if (ret != 0) { CXX_LOG_ERROR("Filed to create directory %s", response.localLocation); throw SnowflakeTransferException(TransferError::MKDIR_ERROR, - response.localLocation, ret); + localLocationPlatform, ret); } if (m_largeFilesMeta.size() > 0) @@ -829,11 +829,28 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle FileMetadata *fileMetadata, size_t resultIndex) { + RemoteStorageRequestOutcome outcome = RemoteStorageRequestOutcome::FAILED; + + // create subfolder first befor adding file name + FileMetadataInitializer::replaceStrAll(fileMetadata->destPath, "/", std::string() + PATH_SEP); fileMetadata->destPath = std::string(response.localLocation) + PATH_SEP + - fileMetadata->destFileName; + fileMetadata->destPath; std::string destPathPlatform = m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destPath); + int ret = sf_create_directory_if_not_exists_recursive(destPathPlatform.c_str()); + if (ret != 0) + { + char* str_error = sf_strerror(errno); + CXX_LOG_DEBUG("Filed to create directory: %s", + fileMetadata->destPath.c_str(), str_error); + sf_free_s(str_error); + m_executionResults->SetTransferOutCome(outcome, resultIndex); + + return outcome; + } + + fileMetadata->destPath += fileMetadata->destFileName; + destPathPlatform += m_stmtPutGet->UTF8ToPlatformString(fileMetadata->destFileName); - RemoteStorageRequestOutcome outcome = RemoteStorageRequestOutcome::FAILED; RetryContext getRetryCtx(fileMetadata->srcFileName, m_maxGetRetries); do { @@ -846,7 +863,6 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle std::ios_base::out | std::ios_base::binary); } catch (...) { - std::string err = "Could not open file " + fileMetadata->destPath + " to downoad"; char* str_error = sf_strerror(errno); CXX_LOG_DEBUG("Could not open file %s to downoad: %s", fileMetadata->destPath.c_str(), str_error); @@ -856,7 +872,6 @@ RemoteStorageRequestOutcome Snowflake::Client::FileTransferAgent::downloadSingle } if (!dstFile.is_open()) { - std::string err = "Could not open file " + fileMetadata->destPath + " to downoad"; char* str_error = sf_strerror(errno); CXX_LOG_DEBUG("Could not open file %s to downoad: %s", fileMetadata->destPath.c_str(), str_error); @@ -894,7 +909,7 @@ void Snowflake::Client::FileTransferAgent::getPresignedUrlForUploading( // need to replace file://mypath/myfile?.csv with file://mypath/myfile1.csv.gz std::string presignedUrlCommand = command; std::string localFilePath = getLocalFilePathFromCommand(command, false); - replaceStrAll(presignedUrlCommand, localFilePath, fileMetadata.destFileName); + FileMetadataInitializer::replaceStrAll(presignedUrlCommand, localFilePath, fileMetadata.destFileName); // then hand that to GS to get the actual presigned URL we'll use PutGetParseResponse rsp; if (!m_stmtPutGet->parsePutGetCommand(&presignedUrlCommand, &rsp)) @@ -940,7 +955,7 @@ std::string Snowflake::Client::FileTransferAgent::getLocalFilePathFromCommand( // unescape backslashes to match the file name from GS if (unescape) { - replaceStrAll(localFilePath, "\\\\\\\\", "\\\\"); + FileMetadataInitializer::replaceStrAll(localFilePath, "\\\\\\\\", "\\\\"); } } else diff --git a/tests/test_simple_put.cpp b/tests/test_simple_put.cpp index 0812268f99..0074947a3e 100755 --- a/tests/test_simple_put.cpp +++ b/tests/test_simple_put.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -118,7 +119,8 @@ void test_simple_put_core(const char * fileName, int compressLevel = -1, bool overwrite = false, SF_CONNECT * connection = nullptr, - bool testUnicode = false) + bool testUnicode = false, + bool uploadFolder = false) { /* init */ SF_STATUS status; @@ -146,10 +148,19 @@ void test_simple_put_core(const char * fileName, ", c2 number, c3 string)"); create_table = create_table_temp ; createOnlyOnce = false; + } + else if (uploadFolder) { + std::string create_table_temp("create table if not exists test_small_put(c1 number" + ", c2 number, c3 string)"); + create_table = create_table_temp; } else if(! createDupTable ){ std::string create_table_temp("create or replace table test_small_put(c1 number" ", c2 number, c3 string)"); create_table = create_table_temp ; + }else if(! createDupTable ){ + std::string create_table_temp("create or replace table test_small_put(c1 number" + ", c2 number, c3 string)"); + create_table = create_table_temp ; } if(! create_table.empty()) { ret = snowflake_query(sfstmt, create_table.c_str(), create_table.size()); @@ -158,6 +169,11 @@ void test_simple_put_core(const char * fileName, std::string dataDir = TestSetup::getDataDir(); std::string file = dataDir + fileName; + if (uploadFolder) + { + file += PATH_SEP; + file += "**"; + } std::string putCommand = "put file://" + file + " @%test_small_put"; if (testUnicode) { @@ -171,7 +187,7 @@ void test_simple_put_core(const char * fileName, } else if (createSubfolder) { - putCommand = "put file://" + file + " @%test_small_put/subfolder"; + putCommand += "/subfolder"; } if (!autoCompress) @@ -230,29 +246,34 @@ void test_simple_put_core(const char * fileName, } ITransferResult * results = agent.execute(&putCommand); - assert_int_equal(1, results->getResultSize()); + if (!uploadFolder) + { + assert_int_equal(1, results->getResultSize()); + } while(results->next()) { std::string value; - results->getColumnAsString(0, value); // source - assert_string_equal( sf_filename_from_path(fileName), value.c_str()); - - std::string expectedTarget = (autoCompress && !strstr(fileName, ".gz")) ? - std::string(fileName) + ".gz" : - std::string(fileName); - results->getColumnAsString(1, value); // get target - assert_string_equal(sf_filename_from_path(expectedTarget.c_str()), value.c_str()); - - std::string expectedSourceCompression = !strstr(fileName, ".gz") ? - "none" : "gzip"; - results->getColumnAsString(4, value); // get source_compression - assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); - - std::string expectedTargetCompression = (!autoCompress && - !strstr(fileName, ".gz")) ? "none" : "gzip"; - results->getColumnAsString(5, value); // get target_compression - assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); + if (!uploadFolder) + { + results->getColumnAsString(0, value); // source + assert_string_equal(sf_filename_from_path(fileName), value.c_str()); + std::string expectedTarget = (autoCompress && !strstr(fileName, ".gz")) ? + std::string(fileName) + ".gz" : + std::string(fileName); + results->getColumnAsString(1, value); // get target + assert_string_equal(sf_filename_from_path(expectedTarget.c_str()), value.c_str()); + + std::string expectedSourceCompression = !strstr(fileName, ".gz") ? + "none" : "gzip"; + results->getColumnAsString(4, value); // get source_compression + assert_string_equal(expectedSourceCompression.c_str(), value.c_str()); + + std::string expectedTargetCompression = (!autoCompress && + !strstr(fileName, ".gz")) ? "none" : "gzip"; + results->getColumnAsString(5, value); // get target_compression + assert_string_equal(expectedTargetCompression.c_str(), value.c_str()); + } results->getColumnAsString(6, value); // get encryption assert_string_equal("UPLOADED", value.c_str()); @@ -352,13 +373,22 @@ static int teardown(void **unused) } void test_simple_get_data(const char *getCommand, const char *size, - long getThreshold = 0, bool testUnicode = false) + long getThreshold = 0, bool testUnicode = false, + SF_CONNECT * connection = nullptr) { /* init */ SF_STATUS status; - SF_CONNECT *sf = setup_snowflake_connection(); - status = snowflake_connect(sf); - assert_int_equal(SF_STATUS_SUCCESS, status); + SF_CONNECT *sf; + + if (!connection) { + sf = setup_snowflake_connection(); + status = snowflake_connect(sf); + assert_int_equal(SF_STATUS_SUCCESS, status); + } + else { + // use the connection passed from test case + sf = connection; + } SF_STMT *sfstmt = NULL; SF_STATUS ret; @@ -405,8 +435,9 @@ void test_simple_get_data(const char *getCommand, const char *size, snowflake_stmt_term(sfstmt); /* close and term */ - snowflake_term(sf); // purge snowflake context - + if (!connection) { + snowflake_term(sf); // purge snowflake context + } } void test_large_put_auto_compress(void **unused) @@ -852,6 +883,171 @@ void test_large_get_threshold(void **unused) test_simple_get_data(tempPath, "5166848", 1000000); } +/* + * 1. generate local files with nested subfolders + * 2. put command to upload files with subfolders + * 3. get command to download uploaded files with subfolders + * 4. put command to upload downloaded files with subfolders + * 5. list command to confirm all expected files on stage + */ +void test_putget_subfolder_core(bool testUnicode) +{ + std::string dataDir = TestSetup::getDataDir(); + std::string filename = "small_file.csv"; + if (testUnicode) + { + filename = PLATFORM_STR + ".csv"; + copy_file(dataDir + "small_file.csv", dataDir + filename, copy_option::overwrite_if_exists); + filename = UTF8_STR + ".csv"; + } + + SF_CONNECT *sf = setup_snowflake_connection(); + snowflake_set_attribute(sf, SF_CON_PROXY, ""); + SF_STATUS status = snowflake_connect(sf); + if (status != SF_STATUS_SUCCESS) { + dump_error(&(sf->error)); + } + assert_int_equal(status, SF_STATUS_SUCCESS); + + // generate files on stage with subfolders + // create a simple file on stage first + test_simple_put_core(filename.c_str(), // filename + "auto", //source compression + true, // auto compress + false, // copyUploadFile + false, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64 * 1024 * 1024, // customThreshold + false, // useDevUrand + false, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, // compressLevel + false, // overwrite + sf, // connection + testUnicode + ); + + // use get command to have local files in nested subfolder + std::string cmd = "get '@%test_small_put/" + filename + ".gz' "; + std::string sub2 = "sub2"; + std::string localPath = "file://"; + + localPath += dataDir + "subfoldertest" + PATH_SEP + "sub1"; + test_simple_get_data((cmd + localPath).c_str(), "48", 0, testUnicode, sf); + + if (testUnicode) + { + sub2 += UTF8_STR; + } + localPath += PATH_SEP; + localPath += sub2; + if (testUnicode) + { + localPath = "'" + localPath + "'"; + replaceInPlace(localPath, "\\", "\\\\"); + } + test_simple_get_data((cmd + localPath).c_str(), "48", 0, testUnicode, sf); + + // the files returned in alphabetical order, use set to get expected files sorted + std::set expectedFiles; + // put command to upload files with subfolders to stage + // uploadFolder=true + // createSubfolder=true as well so the uploaded files will be in path of + // "subfolder" which would be easier to check later + std::string basePath = "subfolder/"; + test_simple_put_core("subfoldertest", // filename + "auto", //source compression + true, // auto compress + false, // copyUploadFile + false, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64 * 1024 * 1024, // customThreshold + false, // useDevUrand + true, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, // compressLevel + false, // overwrite + sf, // connection + testUnicode, // testUnicode + true // uploadFolder + ); + expectedFiles.insert(basePath + "sub1/" + filename + ".gz"); + expectedFiles.insert(basePath + "sub1/" + sub2 + "/" + filename + ".gz"); + + // get command to download uploaded files into a new folder sub3 + cmd = "get @%test_small_put/subfolder file://" + + dataDir + "subfoldertest" + PATH_SEP + "put2" + PATH_SEP + "sub3"; + test_simple_get_data(cmd.c_str(), "48", 0, testUnicode, sf); + + // put command to upload downloaded file again + std::string put2Path = std::string("subfoldertest") + PATH_SEP + "put2"; + test_simple_put_core(put2Path.c_str(), // filename + "auto", //source compression + true, // auto compress + false, // copyUploadFile + false, // verifyCopyUploadFile + false, // copyTableToStaging + false, // createDupTable + false, // setCustomThreshold + 64 * 1024 * 1024, // customThreshold + false, // useDevUrand + true, // createSubfolder + nullptr, // tmpDir + false, // useS3regionalUrl + -1, // compressLevel + false, // overwrite + sf, // connection + testUnicode, // testUnicode + true // uploadFolder + ); + expectedFiles.insert(basePath + "sub3/subfolder/sub1/" + filename + ".gz"); + expectedFiles.insert(basePath + "sub3/subfolder/sub1/" + sub2 + "/" + filename + ".gz"); + + // run list command to check result + SF_STMT *sfstmt = NULL; + SF_STATUS ret; + + /* query */ + sfstmt = snowflake_stmt(sf); + std::string listFiles = "list @%test_small_put/subfolder"; + ret = snowflake_query(sfstmt, listFiles.c_str(), listFiles.size()); + assert_int_equal(SF_STATUS_SUCCESS, ret); + + assert_int_equal(4, snowflake_num_rows(sfstmt)); + auto expectedItr = expectedFiles.begin(); + for (int i = 0; i < 4; i++) + { + const char* value; + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_SUCCESS, ret); + snowflake_column_as_const_str(sfstmt, 1, &value); + assert_string_equal(expectedItr->c_str(), value); + expectedItr++; + } + ret = snowflake_fetch(sfstmt); + assert_int_equal(SF_STATUS_EOF, ret); + + snowflake_stmt_term(sfstmt); + snowflake_term(sf); + remove_all(dataDir + "subfoldertest"); +} + +void test_putget_subfolder(void **unused) +{ + test_putget_subfolder_core(false); +} + +void test_putget_subfolder_with_unicode(void **unused) +{ + test_putget_subfolder_core(true); +} + static int gr_setup(void **unused) { initialize_test(SF_BOOLEAN_FALSE); @@ -1643,6 +1839,8 @@ int main(void) { } const struct CMUnitTest tests[] = { + cmocka_unit_test_teardown(test_putget_subfolder, teardown), + cmocka_unit_test_teardown(test_putget_subfolder_with_unicode, teardown), cmocka_unit_test_teardown(test_simple_put_auto_compress, teardown), cmocka_unit_test_teardown(test_simple_put_config_temp_dir, teardown), cmocka_unit_test_teardown(test_simple_put_auto_detect_gzip, teardown), diff --git a/tests/test_unit_file_metadata_init.cpp b/tests/test_unit_file_metadata_init.cpp index 592098da62..c62d548243 100755 --- a/tests/test_unit_file_metadata_init.cpp +++ b/tests/test_unit_file_metadata_init.cpp @@ -101,6 +101,8 @@ void test_file_pattern_match_core(std::vector *expectedFiles, for (auto i = expectedFiles->begin(); i != expectedFiles->end(); i++) { std::string expectedFileFull = testDir + *i; + // the src file path would use platform path separator + replaceStrAll(expectedFileFull, "/", std::string() + PATH_SEP); expectedFilesFull.insert(expectedFileFull); }