Make klone more atomic

cmd/sync.go

@@ -1,14 +1,9 @@
 package cmd
 
 import (
-	"fmt"
-	"os"
 	"path/filepath"
 
-	"github.com/cert-manager/klone/pkg/cache"
-	"github.com/cert-manager/klone/pkg/download/git"
-	"github.com/cert-manager/klone/pkg/mod"
-
+	"github.com/cert-manager/klone/pkg/sync"
 	"github.com/spf13/cobra"
 )
 
@@ -23,48 +18,7 @@ func NewSyncCommand() *cobra.Command {
 				return err
 			}
 
-			wrkDir := mod.WorkDir(workDirPath)
-			if err := wrkDir.FetchTargets(
-				func(_ string, _ string, src *mod.KloneSource) error {
-					src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))
-
-					if src.RepoHash == "" {
-						hash, err := git.GetHash(src.RepoURL, src.RepoRef)
-						if err != nil {
-							return err
-						}
-
-						src.RepoHash = hash
-					}
-
-					return nil
-				},
-				func(target string, srcs mod.KloneFolder) error {
-					if err := os.RemoveAll(filepath.Join(workDirPath, target)); err != nil {
-						return err
-					}
-
-					if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
-						return err
-					}
-
-					for _, src := range srcs {
-						if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
-							return err
-						}
-					}
-
-					return nil
-				},
-			); err != nil {
-				return fmt.Errorf("failed to fetch targets: %w", err)
-			}
-
-			if err := cache.CleanupOldCacheItems(); err != nil {
-				return fmt.Errorf("failed to cleanup old cache items: %w", err)
-			}
-
-			return nil
+			return sync.SyncFolder(workDirPath, false)
 		},
 	}
 

cmd/upgrade.go

@@ -1,14 +1,9 @@
 package cmd
 
 import (
-	"fmt"
-	"os"
 	"path/filepath"
 
-	"github.com/cert-manager/klone/pkg/cache"
-	"github.com/cert-manager/klone/pkg/download/git"
-	"github.com/cert-manager/klone/pkg/mod"
-
+	"github.com/cert-manager/klone/pkg/sync"
 	"github.com/spf13/cobra"
 )
 
@@ -23,46 +18,7 @@ func NewUpgradeCommand() *cobra.Command {
 				return err
 			}
 
-			wrkDir := mod.WorkDir(workDirPath)
-			if err := wrkDir.FetchTargets(
-				func(_ string, _ string, src *mod.KloneSource) error {
-					src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))
-
-					hash, err := git.GetHash(src.RepoURL, src.RepoRef)
-					if err != nil {
-						return err
-					}
-
-					src.RepoHash = hash
-
-					return nil
-				},
-				func(target string, srcs mod.KloneFolder) error {
-					if err := os.RemoveAll(filepath.Join(workDirPath, target)); err != nil {
-						return err
-					}
-
-					if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
-						return err
-					}
-
-					for _, src := range srcs {
-						if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
-							return err
-						}
-					}
-
-					return nil
-				},
-			); err != nil {
-				return fmt.Errorf("failed to fetch targets: %w", err)
-			}
-
-			if err := cache.CleanupOldCacheItems(); err != nil {
-				return fmt.Errorf("failed to cleanup old cache items: %w", err)
-			}
-
-			return nil
+			return sync.SyncFolder(workDirPath, true)
 		},
 	}
 

go.mod

@@ -3,7 +3,6 @@ module github.com/cert-manager/klone
 go 1.21.1
 
 require (
-	github.com/otiai10/copy v1.12.0
 	github.com/rogpeppe/go-internal v1.11.0
 	github.com/spf13/cobra v1.7.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -12,5 +11,4 @@ require (
 require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
 )

go.sum

@@ -1,19 +1,13 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
-github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
-github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
-github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/cache/clone.go

@@ -3,16 +3,17 @@ package cache
 import (
 	"crypto/sha256"
 	"fmt"
+	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"time"
 
 	"github.com/cert-manager/klone/pkg/mod"
-	cp "github.com/otiai10/copy"
 )
 
 func calculateCacheKey(src mod.KloneSource) string {
-	return fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s-%s-%s", src.RepoURL, src.RepoHash, src.RepoPath))))[:30]
+	return fmt.Sprintf("cache-%x", sha256.Sum256([]byte(fmt.Sprintf("%s-%s-%s", src.RepoURL, src.RepoHash, src.RepoPath))))[:30]
 }
 
 func getCacheDir() (string, error) {
@@ -28,14 +29,6 @@ func getCacheDir() (string, error) {
 	return filepath.Abs(filepath.Clean(filepath.Join(home, ".cache", "klone")))
 }
 
-func getTempDir() (string, error) {
-	if tempDir := os.Getenv("KLONE_TEMP_DIR"); tempDir != "" {
-		return filepath.Abs(filepath.Clean(tempDir))
-	}
-
-	return os.TempDir(), nil
-}
-
 func CloneWithCache(
 	destPath string,
 	src mod.KloneSource,
@@ -46,17 +39,16 @@ func CloneWithCache(
 		return err
 	}
 
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return err
+	}
+
 	cachePath := filepath.Join(cacheDir, calculateCacheKey(src))
 
 	if _, err := os.Stat(cachePath); err != nil && !os.IsNotExist(err) {
 		return err
 	} else if err != nil {
-		tmpParentDir, err := getTempDir()
-		if err != nil {
-			return err
-		}
-
-		tempDir, err := os.MkdirTemp(tmpParentDir, "klone-*")
+		tempDir, err := os.MkdirTemp(cacheDir, "temp-*")
 		if err != nil {
 			return err
 		}
@@ -86,17 +78,33 @@ func CloneWithCache(
 		return err
 	}
 
-	if err := os.RemoveAll(destPath); err != nil {
+	if err := os.MkdirAll(destPath, 0755); err != nil {
 		return err
 	}
 
-	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
+	if err := runRsyncCmd(cachePath, os.Stdout, os.Stderr, "-aEq", ".", destPath); err != nil {
 		return err
 	}
 
-	if err := cp.Copy(cachePath, destPath); err != nil {
+	return nil
+}
+
+func runRsyncCmd(root string, stdout io.Writer, stderr io.Writer, args ...string) error {
+	cmd := exec.Command("rsync", args...)
+
+	cmd.Dir = root
+	cmd.Env = append(os.Environ(), cmd.Env...)
+
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+
+	if err := cmd.Start(); err != nil {
 		return err
 	}
 
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("rsync command failed: %v", err)
+	}
+
 	return nil
 }

pkg/sync/sync.go

@@ -0,0 +1,119 @@
+package sync
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/cert-manager/klone/pkg/cache"
+	"github.com/cert-manager/klone/pkg/download/git"
+	"github.com/cert-manager/klone/pkg/mod"
+)
+
+func SyncFolder(workDirPath string, forceUpgrade bool) error {
+	wrkDir := mod.WorkDir(workDirPath)
+	if err := wrkDir.FetchTargets(
+		func(_ string, _ string, src *mod.KloneSource) error {
+			src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))
+
+			if src.RepoHash == "" || forceUpgrade {
+				hash, err := git.GetHash(src.RepoURL, src.RepoRef)
+				if err != nil {
+					return err
+				}
+
+				src.RepoHash = hash
+			}
+
+			return nil
+		},
+		func(target string, srcs mod.KloneFolder) error {
+			folders := newTreeNode()
+			for _, src := range srcs {
+				folders.Add(filepath.SplitList(src.FolderName)...)
+			}
+
+			if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
+				return err
+			}
+
+			// 1) Remove all folders that are not defined in srcs
+			if err := folders.Cleanup(filepath.Join(workDirPath, target)); err != nil {
+				return err
+			}
+
+			// 2) Sync all folders with cached files
+			for _, src := range srcs {
+				if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		},
+	); err != nil {
+		return fmt.Errorf("failed to fetch targets: %w", err)
+	}
+
+	if err := cache.CleanupOldCacheItems(); err != nil {
+		return fmt.Errorf("failed to cleanup old cache items: %w", err)
+	}
+
+	return nil
+
+}
+
+type treeNode struct {
+	isLeaf   bool
+	children map[string]*treeNode
+}
+
+func newTreeNode() *treeNode {
+	return &treeNode{
+		isLeaf:   false,
+		children: make(map[string]*treeNode),
+	}
+}
+
+func (tn *treeNode) Add(pathSegments ...string) {
+	if len(pathSegments) == 0 {
+		tn.isLeaf = true
+		return
+	}
+
+	if _, ok := tn.children[pathSegments[0]]; !ok {
+		tn.children[pathSegments[0]] = newTreeNode()
+	}
+
+	tn.children[pathSegments[0]].Add(pathSegments[1:]...)
+}
+
+func (tn treeNode) Cleanup(root string) error {
+	if tn.isLeaf {
+		return nil
+	}
+
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		entryName := entry.Name()
+		if _, ok := tn.children[entryName]; ok {
+			continue
+		}
+
+		if err := os.RemoveAll(filepath.Join(root, entryName)); err != nil {
+			return err
+		}
+	}
+
+	for name, node := range tn.children {
+		if err := node.Cleanup(filepath.Join(root, name)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}