From 1a44517beb7d19ef226ff698a76b480d24dc8767 Mon Sep 17 00:00:00 2001 From: sunby Date: Sun, 8 Oct 2023 17:20:56 +0800 Subject: [PATCH] aa --- go/io/fs/minio_fs.go | 52 ++++++--------------------------------- go/io/fs/minio_fs_test.go | 30 +++++++++++----------- 2 files changed, 22 insertions(+), 60 deletions(-) diff --git a/go/io/fs/minio_fs.go b/go/io/fs/minio_fs.go index 47958623..7213374e 100644 --- a/go/io/fs/minio_fs.go +++ b/go/io/fs/minio_fs.go @@ -5,9 +5,7 @@ import ( "fmt" "io" "net/url" - "strings" - "github.com/milvus-io/milvus-storage/go/common/errors" "github.com/milvus-io/milvus-storage/go/common/log" "github.com/milvus-io/milvus-storage/go/io/fs/file" "github.com/minio/minio-go/v7" @@ -20,48 +18,24 @@ type MinioFs struct { bucketName string } -func ExtractFileName(path string) (string, error) { - p := strings.Index(path, "/") - if p == -1 { - return "", errors.ErrInvalidPath - } - return path[p+1:], nil -} - func (fs *MinioFs) OpenFile(path string) (file.File, error) { - fileName, err := ExtractFileName(path) - if err != nil { - return nil, err - } - return file.NewMinioFile(fs.client, fileName, fs.bucketName) + return file.NewMinioFile(fs.client, path, fs.bucketName) } func (fs *MinioFs) Rename(src string, dst string) error { - srcFileName, err := ExtractFileName(src) - if err != nil { - return err - } - dstFileName, err := ExtractFileName(dst) - if err != nil { - return err - } - _, err = fs.client.CopyObject(context.TODO(), minio.CopyDestOptions{Bucket: fs.bucketName, Object: dstFileName}, minio.CopySrcOptions{Bucket: fs.bucketName, Object: srcFileName}) + _, err := fs.client.CopyObject(context.TODO(), minio.CopyDestOptions{Bucket: fs.bucketName, Object: dst}, minio.CopySrcOptions{Bucket: fs.bucketName, Object: src}) if err != nil { return err } - err = fs.client.RemoveObject(context.TODO(), fs.bucketName, srcFileName, minio.RemoveObjectOptions{}) + err = fs.client.RemoveObject(context.TODO(), fs.bucketName, src, minio.RemoveObjectOptions{}) if err != nil { - log.Warn("failed to remove source object", log.String("source", srcFileName)) + log.Warn("failed to remove source object", log.String("source", src)) } return nil } func (fs *MinioFs) DeleteFile(path string) error { - fileName, err := ExtractFileName(path) - if err != nil { - return err - } - return fs.client.RemoveObject(context.TODO(), fs.bucketName, fileName, minio.RemoveObjectOptions{}) + return fs.client.RemoveObject(context.TODO(), fs.bucketName, path, minio.RemoveObjectOptions{}) } func (fs *MinioFs) CreateDir(path string) error { @@ -69,10 +43,6 @@ func (fs *MinioFs) CreateDir(path string) error { } func (fs *MinioFs) List(path string) ([]FileEntry, error) { - path, err := ExtractFileName(path) - if err != nil { - return nil, err - } ret := make([]FileEntry, 0) for objInfo := range fs.client.ListObjects(context.TODO(), fs.bucketName, minio.ListObjectsOptions{Prefix: path, Recursive: true}) { if objInfo.Err != nil { @@ -85,10 +55,6 @@ func (fs *MinioFs) List(path string) ([]FileEntry, error) { } func (fs *MinioFs) ReadFile(path string) ([]byte, error) { - path, err := ExtractFileName(path) - if err != nil { - return nil, err - } obj, err := fs.client.GetObject(context.TODO(), fs.bucketName, path, minio.GetObjectOptions{}) if err != nil { return nil, err @@ -111,11 +77,7 @@ func (fs *MinioFs) ReadFile(path string) ([]byte, error) { } func (fs *MinioFs) Exist(path string) (bool, error) { - path, err := ExtractFileName(path) - if err != nil { - return false, err - } - _, err = fs.client.StatObject(context.TODO(), fs.bucketName, path, minio.StatObjectOptions{}) + _, err := fs.client.StatObject(context.TODO(), fs.bucketName, path, minio.StatObjectOptions{}) if err != nil { resp := minio.ToErrorResponse(err) if resp.Code == "NoSuchKey" { @@ -127,7 +89,7 @@ func (fs *MinioFs) Exist(path string) (bool, error) { } func (fs *MinioFs) Path() string { - return fs.bucketName + return "" } // uri should be s3://accessKey:secretAceessKey@endpoint/bucket/ diff --git a/go/io/fs/minio_fs_test.go b/go/io/fs/minio_fs_test.go index d84b45a0..de58766b 100644 --- a/go/io/fs/minio_fs_test.go +++ b/go/io/fs/minio_fs_test.go @@ -22,14 +22,14 @@ func (suite *MinioFsTestSuite) SetupSuite() { } func (suite *MinioFsTestSuite) TestMinioOpenFile() { - file, err := suite.fs.OpenFile("default/a") + file, err := suite.fs.OpenFile("a") suite.NoError(err) n, err := file.Write([]byte{1}) suite.NoError(err) suite.Equal(1, n) suite.NoError(file.Close()) - file, err = suite.fs.OpenFile("default/a") + file, err = suite.fs.OpenFile("a") suite.NoError(err) buf := make([]byte, 10) n, err = file.Read(buf) @@ -39,17 +39,17 @@ func (suite *MinioFsTestSuite) TestMinioOpenFile() { } func (suite *MinioFsTestSuite) TestMinioRename() { - file, err := suite.fs.OpenFile("default/a") + file, err := suite.fs.OpenFile("a") suite.NoError(err) n, err := file.Write([]byte{1}) suite.NoError(err) suite.Equal(1, n) suite.NoError(file.Close()) - err = suite.fs.Rename("default/a", "default/b") + err = suite.fs.Rename("a", "b") suite.NoError(err) - file, err = suite.fs.OpenFile("default/b") + file, err = suite.fs.OpenFile("b") suite.NoError(err) buf := make([]byte, 10) n, err = file.Read(buf) @@ -59,59 +59,59 @@ func (suite *MinioFsTestSuite) TestMinioRename() { } func (suite *MinioFsTestSuite) TestMinioFsDeleteFile() { - file, err := suite.fs.OpenFile("default/a") + file, err := suite.fs.OpenFile("a") suite.NoError(err) n, err := file.Write([]byte{1}) suite.NoError(err) suite.Equal(1, n) suite.NoError(file.Close()) - err = suite.fs.DeleteFile("default/a") + err = suite.fs.DeleteFile("a") suite.NoError(err) - exist, err := suite.fs.Exist("default/a") + exist, err := suite.fs.Exist("a") suite.NoError(err) suite.False(exist) } func (suite *MinioFsTestSuite) TestMinioFsList() { - file, err := suite.fs.OpenFile("default/a/b/c") + file, err := suite.fs.OpenFile("a/b/c") suite.NoError(err) _, err = file.Write([]byte{1}) suite.NoError(err) suite.NoError(file.Close()) - entries, err := suite.fs.List("default/a/") + entries, err := suite.fs.List("a/") suite.NoError(err) suite.EqualValues([]fs.FileEntry{{Path: "a/b/c"}}, entries) } func (suite *MinioFsTestSuite) TestMinioFsReadFile() { - file, err := suite.fs.OpenFile("default/a") + file, err := suite.fs.OpenFile("a") suite.NoError(err) n, err := file.Write([]byte{1}) suite.NoError(err) suite.Equal(1, n) suite.NoError(file.Close()) - content, err := suite.fs.ReadFile("default/a") + content, err := suite.fs.ReadFile("a") suite.NoError(err) suite.EqualValues([]byte{1}, content) } func (suite *MinioFsTestSuite) TestMinioFsExist() { - exist, err := suite.fs.Exist("default/nonexist") + exist, err := suite.fs.Exist("nonexist") suite.NoError(err) suite.False(exist) - file, err := suite.fs.OpenFile("default/exist") + file, err := suite.fs.OpenFile("exist") suite.NoError(err) n, err := file.Write([]byte{1}) suite.NoError(err) suite.Equal(1, n) suite.NoError(file.Close()) - exist, err = suite.fs.Exist("default/exist") + exist, err = suite.fs.Exist("exist") suite.NoError(err) suite.True(exist) }