Skip to content

Commit

Permalink
add tests
Browse files Browse the repository at this point in the history
Signed-off-by: Alexandr Stefurishin <[email protected]>
  • Loading branch information
Alexandr Stefurishin committed Dec 3, 2024
1 parent 61a51e8 commit ba25c69
Show file tree
Hide file tree
Showing 9 changed files with 400 additions and 20 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ require (
go.uber.org/zap v1.19.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/mod v0.22.0
golang.org/x/oauth2 v0.22.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down
4 changes: 2 additions & 2 deletions pkg/nfs/controllerserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName())

klog.V(2).Infof("tar %v -> %v", srcPath, dstPath)
err = tarPack(dstPath, srcPath, true)
err = TarPack(srcPath, dstPath, true)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v", err)
}
Expand Down Expand Up @@ -557,7 +557,7 @@ func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.Creat
dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol)
klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath)

err = tarUnpack(snapPath, dstPath, true)
err = TarUnpack(snapPath, dstPath, true)
if err != nil {
return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v", err)
}
Expand Down
39 changes: 21 additions & 18 deletions pkg/nfs/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,33 @@ import (
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
)

func tarPack(dstFilePath, srcPath string, enableCompression bool) error {
func TarPack(srcDirPath string, dstPath string, enableCompression bool) error {
// normalize all paths to be absolute and clean
dstFilePath, err := filepath.Abs(dstFilePath)
dstPath, err := filepath.Abs(dstPath)
if err != nil {
return fmt.Errorf("normalizing destination path: %w", err)
}

srcPath, err = filepath.Abs(srcPath)
srcDirPath, err = filepath.Abs(srcDirPath)
if err != nil {
return fmt.Errorf("normalizing source path: %w", err)
}

if strings.Index(dstFilePath, srcPath) == 0 {
return fmt.Errorf("destination file %s cannot be under source directory %s", dstFilePath, srcPath)
if strings.Index(path.Dir(dstPath), srcDirPath) == 0 {
return fmt.Errorf("destination file %s cannot be under source directory %s", dstPath, srcDirPath)
}

tarFile, err := os.Create(dstFilePath)
tarFile, err := os.Create(dstPath)
if err != nil {
return fmt.Errorf("creating destination file: %w", err)
}
defer func() {
err = errors.Join(err, closeAndWrapErr(tarFile, "closing destination file %s: %w", dstFilePath))
err = errors.Join(err, closeAndWrapErr(tarFile, "closing destination file %s: %w", dstPath))
}()

var tarDst io.Writer = tarFile
Expand All @@ -52,11 +53,13 @@ func tarPack(dstFilePath, srcPath string, enableCompression bool) error {

// recursively visit every file and write it
if err = filepath.Walk(
srcPath,
srcDirPath,
func(srcSubPath string, fileInfo fs.FileInfo, walkErr error) error {
return tarVisitFileToPack(tarWriter, srcPath, srcSubPath, fileInfo, walkErr)
return tarVisitFileToPack(tarWriter, srcDirPath, srcSubPath, fileInfo, walkErr)
},
); err != nil {
ee := err.Error()
_ = ee
return fmt.Errorf("walking source directory: %w", err)
}

Expand Down Expand Up @@ -115,24 +118,24 @@ func tarVisitFileToPack(
return nil
}

func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error) {
func TarUnpack(srcPath, dstDirPath string, enableCompression bool) (err error) {
// normalize all paths to be absolute and clean
archivePath, err = filepath.Abs(archivePath)
srcPath, err = filepath.Abs(srcPath)
if err != nil {
return fmt.Errorf("normalizing archive path: %w", err)
}

dstPath, err = filepath.Abs(dstPath)
dstDirPath, err = filepath.Abs(dstDirPath)
if err != nil {
return fmt.Errorf("normalizing archive destination path: %w", err)
}

tarFile, err := os.Open(archivePath)
tarFile, err := os.Open(srcPath)
if err != nil {
return fmt.Errorf("opening archive %s: %w", archivePath, err)
return fmt.Errorf("opening archive %s: %w", srcPath, err)
}
defer func() {
err = errors.Join(err, closeAndWrapErr(tarFile, "closing archive %s: %w", archivePath))
err = errors.Join(err, closeAndWrapErr(tarFile, "closing archive %s: %w", srcPath))
}()

var tarDst io.Reader = tarFile
Expand All @@ -146,7 +149,7 @@ func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error)
err = errors.Join(err, closeAndWrapErr(gzipReader, "closing gzip reader: %w"))
}()

tarDst = tar.NewReader(gzipReader)
tarDst = gzipReader
}

tarReader := tar.NewReader(tarDst)
Expand All @@ -158,12 +161,12 @@ func tarUnpack(archivePath, dstPath string, enableCompression bool) (err error)
break
}
if err != nil {
return fmt.Errorf("reading tar header of %s: %w", archivePath, err)
return fmt.Errorf("reading tar header of %s: %w", srcPath, err)
}

fileInfo := tarHeader.FileInfo()

filePath := filepath.Join(dstPath, tarHeader.Name)
filePath := filepath.Join(dstDirPath, tarHeader.Name)

fileDirPath := filePath
if !fileInfo.Mode().IsDir() {
Expand Down
187 changes: 187 additions & 0 deletions test/tar/tar_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
package tar

import (
"bytes"
"fmt"
"maps"
"math"
"os"
"os/exec"
"path"
"slices"
"strings"
"testing"

"github.com/kubernetes-csi/csi-driver-nfs/pkg/nfs"
"golang.org/x/mod/sumdb/dirhash"
)

const (
code packApi = '0'
cli packApi = '1'
)

type packApi byte

const archiveFileExt = ".tar.gz"

func TestPackUnpack(t *testing.T) {
inputPath := t.TempDir()
generateFileSystem(t, inputPath)

outputPath := t.TempDir()

// produced file names (without extensions) have a suffix,
// which determine the last operation:
// "0" means that it was produced from code
// "1" means that it was produced from CLI
// e.g.: "testdata011.tar.gz" - was packed from code,
// then unpacked from cli and packed again from cli

pathsBySuffix := make(map[string]string)

// number of pack/unpack operations
opNum := 4

// generate all operation combinations
fileNum := int(math.Pow(2, float64(opNum)))
for i := 0; i < fileNum; i++ {
binStr := fmt.Sprintf("%b", i)

// left-pad with zeroes
binStr = strings.Repeat("0", opNum-len(binStr)) + binStr

// copy slices to satisfy type system
ops := make([]packApi, opNum)
for opIdx := 0; opIdx < opNum; opIdx++ {
ops[opIdx] = packApi(binStr[opIdx])
}

// produce folders and archives
produce(t, pathsBySuffix, inputPath, outputPath, ops...)
}

// byte-compare archives with the same last step
paths := slices.Collect(maps.Values(pathsBySuffix))
assertUnpackedFilesEqual(t, inputPath, paths)
}

func produce(
t *testing.T,
results map[string]string,
inputDirPath string,
outputDirPath string,
ops ...packApi,
) {
baseName := path.Base(inputDirPath)

for i := 0; i < len(ops); i++ {
packing := i%2 == 0

srcPath := inputDirPath
if i > 0 {
prevSuffix := string(ops[:i])
srcPath = path.Join(outputDirPath, baseName+prevSuffix)
if !packing {
srcPath += archiveFileExt
}
}

suffix := string(ops[:i+1])
dstPath := path.Join(outputDirPath, baseName+suffix)
if packing {
dstPath += archiveFileExt
}

if _, ok := results[suffix]; ok {
continue
}

switch {
case packing && ops[i] == code:
// packing from code
if err := nfs.TarPack(srcPath, dstPath, true); err != nil {
t.Fatalf("packing '%s' with TarPack into '%s': %v", srcPath, dstPath, err)
}
case packing && ops[i] == cli:
// packing from CLI
if out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput(); err != nil {
t.Log("TAR OUTPUT:", string(out))
t.Fatalf("packing '%s' with tar into '%s': %v", srcPath, dstPath, err)
}
case !packing && ops[i] == code:
// unpacking from code
if err := nfs.TarUnpack(srcPath, dstPath, true); err != nil {
t.Fatalf("unpacking '%s' with TarUnpack into '%s': %v", srcPath, dstPath, err)
}
case !packing && ops[i] == cli:
// unpacking from CLI
// tar requires destination directory to exist
if err := os.MkdirAll(dstPath, 0755); err != nil {
t.Fatalf("making dir '%s' for unpacking with tar: %v", dstPath, err)
}
if out, err := exec.Command("tar", "-xzvf", srcPath, "-C", dstPath).CombinedOutput(); err != nil {
t.Log("TAR OUTPUT:", string(out))
t.Fatalf("unpacking '%s' with tar into '%s': %v", srcPath, dstPath, err)
}
default:
t.Fatalf("unknown suffix: %s", string(ops[i]))
}

results[suffix] = dstPath
}
}

func assertUnpackedFilesEqual(t *testing.T, originalDir string, paths []string) {
originalDirHash, err := dirhash.HashDir(originalDir, "_", dirhash.DefaultHash)
if err != nil {
t.Fatal("failed hashing original dir ", err)
}

for _, p := range paths {
if strings.HasSuffix(p, archiveFileExt) {
// archive, not a directory
continue
}

// unpacked directory
hs, err := dirhash.HashDir(p, "_", dirhash.DefaultHash)
if err != nil {
t.Fatal("failed hashing dir ", err)
}

if hs != originalDirHash {
t.Errorf("expected '%s' to have the same hash as '%s', got different", originalDir, p)
}
}
}

func generateFileSystem(t *testing.T, inputPath string) {
// empty directory
if err := os.MkdirAll(path.Join(inputPath, "empty_dir"), 0755); err != nil {
t.Fatalf("generating empty directory: %v", err)
}

// deep empty directories
deepEmptyDirPath := path.Join(inputPath, "deep_empty_dir", strings.Repeat("/0/1/2", 20))
if err := os.MkdirAll(deepEmptyDirPath, 0755); err != nil {
t.Fatalf("generating deep empty directory '%s': %v", deepEmptyDirPath, err)
}

// empty file
f, err := os.Create(path.Join(inputPath, "empty_file"))
if err != nil {
t.Fatalf("generating empty file: %v", err)
}
f.Close()

// big (100MB) file
bigFilePath := path.Join(inputPath, "big_file")
for i := byte(0); i < 100; i++ {
// write 1MB
err := os.WriteFile(bigFilePath, bytes.Repeat([]byte{i}, 1024*1024), 0755)
if err != nil {
t.Fatalf("generating empty file: %v", err)
}
}
}
27 changes: 27 additions & 0 deletions vendor/golang.org/x/mod/LICENSE

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 22 additions & 0 deletions vendor/golang.org/x/mod/PATENTS

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit ba25c69

Please sign in to comment.