Add blob interfaces

go/common/constant/constant.go

@@ -4,6 +4,7 @@ const KReadBatchSize = 1024
 const KManifestTempFileSuffix = ".manifest.tmp"
 const KManifestFileSuffix = ".manifest"
 const KManifestDir = "versions"
+const KBlobDir = "blobs"
 const KParquetDataFileSuffix = ".parquet"
 const KParquetDataDir = "data"
 const KOffsetFieldName = "__offset"

go/common/utils/utils.go

@@ -347,11 +347,20 @@ func GetManifestTmpFilePath(path string, version int64) string {
 	return path
 }
 
+func GetBlobFilePath(path string) string {
+	blobId := uuid.New()
+	return filepath.Join(GetBlobDir(path), blobId.String())
+}
+
 func GetManifestDir(path string) string {
 	path = filepath.Join(path, constant.KManifestDir)
 	return path
 }
 
+func GetBlobDir(path string) string {
+	return filepath.Join(path, constant.KBlobDir)
+}
+
 func ParseVersionFromFileName(path string) int64 {
 	pos := strings.Index(path, constant.KManifestFileSuffix)
 	if pos == -1 || !strings.HasSuffix(path, constant.KManifestFileSuffix) {

go/file/blob/blob.go

@@ -0,0 +1,25 @@
+package blob
+
+import "github.com/milvus-io/milvus-storage-format/proto/manifest_proto"
+
+type Blob struct {
+	Name string
+	Size int64
+	File string
+}
+
+func (b Blob) ToProtobuf() *manifest_proto.Blob {
+	blob := &manifest_proto.Blob{}
+	blob.Name = b.Name
+	blob.Size = b.Size
+	blob.File = b.File
+	return blob
+}
+
+func FromProtobuf(blob *manifest_proto.Blob) Blob {
+	return Blob{
+		Name: blob.Name,
+		Size: blob.Size,
+		File: blob.File,
+	}
+}

go/proto/manifest.proto

@@ -1,7 +1,7 @@
 syntax = "proto3";
 import "schema.proto";
 package manifest_proto;
-option go_package = "github.com/milvus-io/milvus-storage-format/proto/manifest_proto;manifest_proto";
+option go_package = "github.com/milvus-io/milvus-storage-format/proto/manifest_proto";
 
 message Options { string uri = 1; }
 
@@ -12,9 +12,16 @@ message Manifest {
   repeated Fragment scalar_fragments = 4;
   repeated Fragment vector_fragments = 5;
   repeated Fragment delete_fragments = 6;
+  repeated Blob blobs = 7;
 }
 
 message Fragment {
   int64 id = 1;
   repeated string files = 2;
 }
+
+message Blob {
+  string name = 1;
+  int64 size = 2;
+  string file = 3;
+}

go/proto/schema.proto

@@ -1,7 +1,7 @@
 syntax = "proto3";
 
 package schema_proto;
-option go_package = "github.com/milvus-io/milvus-storage-format/proto/schema_proto;schema_proto";
+option go_package = "github.com/milvus-io/milvus-storage-format/proto/schema_proto";
 
 enum LogicType {
   NA = 0;
@@ -53,23 +53,17 @@ enum Endianness {
   Big = 1;
 }
 
-message FixedSizeBinaryType {
-  int32 byte_width = 1;
-}
+message FixedSizeBinaryType { int32 byte_width = 1; }
 
-message FixedSizeListType {
-  int32 list_size = 1;
-}
+message FixedSizeListType { int32 list_size = 1; }
 
 message DictionaryType {
   DataType index_type = 1;
   DataType value_type = 2;
   bool ordered = 3;
 }
 
-message MapType {
-  bool keys_sorted = 1;
-}
+message MapType { bool keys_sorted = 1; }
 
 message DataType {
   oneof type_related_values {