Change data dir and add options (#35)

Signed-off-by: sunby <[email protected]>
milvus-io · Aug 3, 2023 · d6da825 · d6da825
1 parent 2ad6158
commit d6da825
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 3 deletions.
diff --git a/cpp/src/common/constants.h b/cpp/src/common/constants.h
@@ -7,6 +7,9 @@ const int kReadBatchSize = 1024;
 const std::string kManifestTempFileSuffix = ".manifest.tmp";
 const std::string kManifestFileSuffix = ".manifest";
 const std::string kManifestsDir = "versions";
+const std::string kScalarDataDir = "scalar";
+const std::string kVectorDataDir = "vector";
+const std::string kDeleteDataDir = "delete";
 const std::string kParquetDataFileSuffix = ".parquet";
 const std::string kOffsetFieldName = "__offset";
 

diff --git a/cpp/src/common/utils.cpp b/cpp/src/common/utils.cpp
@@ -231,6 +231,18 @@ std::string GetNewParquetFilePath(const std::string& path) {
   return path + boost::uuids::to_string(scalar_file_id) + kParquetDataFileSuffix;
 }
 
+std::string GetScalarDataDir(const std::string& path) {
+  return arrow::fs::internal::JoinAbstractPath(std::vector<std::string_view>{path, kScalarDataDir});
+}
+
+std::string GetVectorDataDir(const std::string& path) {
+  return arrow::fs::internal::JoinAbstractPath(std::vector<std::string_view>{path, kVectorDataDir});
+}
+
+std::string GetDeleteDataDir(const std::string& path) {
+  return arrow::fs::internal::JoinAbstractPath(std::vector<std::string_view>{path, kDeleteDataDir});
+}
+
 std::string GetManifestFilePath(const std::string& path, const int64_t version) {
   return arrow::fs::internal::JoinAbstractPath(
       std::vector<std::string_view>{path, kManifestsDir, std::to_string(version) + kManifestFileSuffix});

diff --git a/cpp/src/common/utils.h b/cpp/src/common/utils.h
@@ -24,4 +24,7 @@ Result<std::shared_ptr<arrow::Schema>> ProjectSchema(std::shared_ptr<arrow::Sche
 int64_t ParseVersionFromFileName(const std::string& path);
 
 std::string GetManifestDir(const std::string& path);
+std::string GetScalarDataDir(const std::string& path);
+std::string GetVectorDataDir(const std::string& path);
+std::string GetDeleteDataDir(const std::string& path);
 }  // namespace milvus_storage
diff --git a/cpp/src/storage/space.cpp b/cpp/src/storage/space.cpp
@@ -80,14 +80,14 @@ Status Space::Write(arrow::RecordBatchReader* reader, WriteOption* option) {
     auto vector_record = arrow::RecordBatch::Make(vector_schema, batch->num_rows(), vector_cols);
 
     if (scalar_writer == nullptr) {
-      auto scalar_file_path = GetNewParquetFilePath(path_);
+      auto scalar_file_path = GetNewParquetFilePath(GetScalarDataDir(path_));
       scalar_writer = new ParquetFileWriter(scalar_schema, fs_, scalar_file_path);
       RETURN_NOT_OK(scalar_writer->Init());
       scalar_fragment.add_file(scalar_file_path);
     }
 
     if (vector_writer == nullptr) {
-      auto vector_file_path = GetNewParquetFilePath(path_);
+      auto vector_file_path = GetNewParquetFilePath(GetVectorDataDir(path_));
       vector_writer = new ParquetFileWriter(vector_schema, fs_, vector_file_path);
       RETURN_NOT_OK(vector_writer->Init());
       vector_fragment.add_file(vector_file_path);
@@ -137,7 +137,7 @@ Status Space::Delete(arrow::RecordBatchReader* reader) {
     }
 
     if (!writer) {
-      delete_file = GetNewParquetFilePath(path_);
+      delete_file = GetNewParquetFilePath(GetDeleteDataDir(path_));
       writer = new ParquetFileWriter(manifest_->schema()->delete_schema(), fs_, delete_file);
       RETURN_NOT_OK(writer->Init());
     }