Skip to content

Commit

Permalink
aa
Browse files Browse the repository at this point in the history
  • Loading branch information
@sunby
sunby committed Oct 8, 2023
1 parent f9cad59 commit 1a44517
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 60 deletions.
52 changes: 7 additions & 45 deletions go/io/fs/minio_fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@ import (
"fmt"
"io"
"net/url"
"strings"

"github.com/milvus-io/milvus-storage/go/common/errors"
"github.com/milvus-io/milvus-storage/go/common/log"
"github.com/milvus-io/milvus-storage/go/io/fs/file"
"github.com/minio/minio-go/v7"
Expand All @@ -20,59 +18,31 @@ type MinioFs struct {
bucketName string
}

func ExtractFileName(path string) (string, error) {
p := strings.Index(path, "/")
if p == -1 {
return "", errors.ErrInvalidPath
}
return path[p+1:], nil
}

func (fs *MinioFs) OpenFile(path string) (file.File, error) {
fileName, err := ExtractFileName(path)
if err != nil {
return nil, err
}
return file.NewMinioFile(fs.client, fileName, fs.bucketName)
return file.NewMinioFile(fs.client, path, fs.bucketName)
}

func (fs *MinioFs) Rename(src string, dst string) error {
srcFileName, err := ExtractFileName(src)
if err != nil {
return err
}
dstFileName, err := ExtractFileName(dst)
if err != nil {
return err
}
_, err = fs.client.CopyObject(context.TODO(), minio.CopyDestOptions{Bucket: fs.bucketName, Object: dstFileName}, minio.CopySrcOptions{Bucket: fs.bucketName, Object: srcFileName})
_, err := fs.client.CopyObject(context.TODO(), minio.CopyDestOptions{Bucket: fs.bucketName, Object: dst}, minio.CopySrcOptions{Bucket: fs.bucketName, Object: src})
if err != nil {
return err
}
err = fs.client.RemoveObject(context.TODO(), fs.bucketName, srcFileName, minio.RemoveObjectOptions{})
err = fs.client.RemoveObject(context.TODO(), fs.bucketName, src, minio.RemoveObjectOptions{})
if err != nil {
log.Warn("failed to remove source object", log.String("source", srcFileName))
log.Warn("failed to remove source object", log.String("source", src))
}
return nil
}

func (fs *MinioFs) DeleteFile(path string) error {
fileName, err := ExtractFileName(path)
if err != nil {
return err
}
return fs.client.RemoveObject(context.TODO(), fs.bucketName, fileName, minio.RemoveObjectOptions{})
return fs.client.RemoveObject(context.TODO(), fs.bucketName, path, minio.RemoveObjectOptions{})
}

func (fs *MinioFs) CreateDir(path string) error {
return nil
}

func (fs *MinioFs) List(path string) ([]FileEntry, error) {
path, err := ExtractFileName(path)
if err != nil {
return nil, err
}
ret := make([]FileEntry, 0)
for objInfo := range fs.client.ListObjects(context.TODO(), fs.bucketName, minio.ListObjectsOptions{Prefix: path, Recursive: true}) {
if objInfo.Err != nil {
Expand All @@ -85,10 +55,6 @@ func (fs *MinioFs) List(path string) ([]FileEntry, error) {
}

func (fs *MinioFs) ReadFile(path string) ([]byte, error) {
path, err := ExtractFileName(path)
if err != nil {
return nil, err
}
obj, err := fs.client.GetObject(context.TODO(), fs.bucketName, path, minio.GetObjectOptions{})
if err != nil {
return nil, err
Expand All @@ -111,11 +77,7 @@ func (fs *MinioFs) ReadFile(path string) ([]byte, error) {
}

func (fs *MinioFs) Exist(path string) (bool, error) {
path, err := ExtractFileName(path)
if err != nil {
return false, err
}
_, err = fs.client.StatObject(context.TODO(), fs.bucketName, path, minio.StatObjectOptions{})
_, err := fs.client.StatObject(context.TODO(), fs.bucketName, path, minio.StatObjectOptions{})
if err != nil {
resp := minio.ToErrorResponse(err)
if resp.Code == "NoSuchKey" {
Expand All @@ -127,7 +89,7 @@ func (fs *MinioFs) Exist(path string) (bool, error) {
}

func (fs *MinioFs) Path() string {
return fs.bucketName
return ""
}

// uri should be s3://accessKey:secretAceessKey@endpoint/bucket/
Expand Down
30 changes: 15 additions & 15 deletions go/io/fs/minio_fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@ func (suite *MinioFsTestSuite) SetupSuite() {
}

func (suite *MinioFsTestSuite) TestMinioOpenFile() {
file, err := suite.fs.OpenFile("default/a")
file, err := suite.fs.OpenFile("a")
suite.NoError(err)
n, err := file.Write([]byte{1})
suite.NoError(err)
suite.Equal(1, n)
suite.NoError(file.Close())

file, err = suite.fs.OpenFile("default/a")
file, err = suite.fs.OpenFile("a")
suite.NoError(err)
buf := make([]byte, 10)
n, err = file.Read(buf)
Expand All @@ -39,17 +39,17 @@ func (suite *MinioFsTestSuite) TestMinioOpenFile() {
}

func (suite *MinioFsTestSuite) TestMinioRename() {
file, err := suite.fs.OpenFile("default/a")
file, err := suite.fs.OpenFile("a")
suite.NoError(err)
n, err := file.Write([]byte{1})
suite.NoError(err)
suite.Equal(1, n)
suite.NoError(file.Close())

err = suite.fs.Rename("default/a", "default/b")
err = suite.fs.Rename("a", "b")
suite.NoError(err)

file, err = suite.fs.OpenFile("default/b")
file, err = suite.fs.OpenFile("b")
suite.NoError(err)
buf := make([]byte, 10)
n, err = file.Read(buf)
Expand All @@ -59,59 +59,59 @@ func (suite *MinioFsTestSuite) TestMinioRename() {
}

func (suite *MinioFsTestSuite) TestMinioFsDeleteFile() {
file, err := suite.fs.OpenFile("default/a")
file, err := suite.fs.OpenFile("a")
suite.NoError(err)
n, err := file.Write([]byte{1})
suite.NoError(err)
suite.Equal(1, n)
suite.NoError(file.Close())

err = suite.fs.DeleteFile("default/a")
err = suite.fs.DeleteFile("a")
suite.NoError(err)

exist, err := suite.fs.Exist("default/a")
exist, err := suite.fs.Exist("a")
suite.NoError(err)
suite.False(exist)
}

func (suite *MinioFsTestSuite) TestMinioFsList() {
file, err := suite.fs.OpenFile("default/a/b/c")
file, err := suite.fs.OpenFile("a/b/c")
suite.NoError(err)
_, err = file.Write([]byte{1})
suite.NoError(err)
suite.NoError(file.Close())

entries, err := suite.fs.List("default/a/")
entries, err := suite.fs.List("a/")
suite.NoError(err)
suite.EqualValues([]fs.FileEntry{{Path: "a/b/c"}}, entries)
}

func (suite *MinioFsTestSuite) TestMinioFsReadFile() {
file, err := suite.fs.OpenFile("default/a")
file, err := suite.fs.OpenFile("a")
suite.NoError(err)
n, err := file.Write([]byte{1})
suite.NoError(err)
suite.Equal(1, n)
suite.NoError(file.Close())

content, err := suite.fs.ReadFile("default/a")
content, err := suite.fs.ReadFile("a")
suite.NoError(err)
suite.EqualValues([]byte{1}, content)
}

func (suite *MinioFsTestSuite) TestMinioFsExist() {
exist, err := suite.fs.Exist("default/nonexist")
exist, err := suite.fs.Exist("nonexist")
suite.NoError(err)
suite.False(exist)

file, err := suite.fs.OpenFile("default/exist")
file, err := suite.fs.OpenFile("exist")
suite.NoError(err)
n, err := file.Write([]byte{1})
suite.NoError(err)
suite.Equal(1, n)
suite.NoError(file.Close())

exist, err = suite.fs.Exist("default/exist")
exist, err = suite.fs.Exist("exist")
suite.NoError(err)
suite.True(exist)
}
Expand Down

0 comments on commit 1a44517

Please sign in to comment.