From 92876fbe8f3605aecc64c829f18c8f3d9199822e Mon Sep 17 00:00:00 2001 From: Kimmo Lehto Date: Wed, 23 Mar 2022 13:26:02 +0200 Subject: [PATCH] Avoid reuploading unchanged files (#352) --- go.sum | 1 - phase/upload_binaries.go | 22 ++++++++++++-- phase/uploadfiles.go | 17 +++++++++-- .../v1beta1/cluster/host.go | 30 +++++++++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 7b56da75..6f930a5f 100644 --- a/go.sum +++ b/go.sum @@ -471,7 +471,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/phase/upload_binaries.go b/phase/upload_binaries.go index 73d58304..821beb3b 100644 --- a/phase/upload_binaries.go +++ b/phase/upload_binaries.go @@ -1,6 +1,9 @@ package phase import ( + "fmt" + "os" + "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" "github.com/k0sproject/rig/exec" @@ -38,15 +41,28 @@ func (p *UploadBinaries) Run() error { } func (p *UploadBinaries) uploadBinary(h *cluster.Host) error { - log.Infof("%s: uploading k0s binary from %s", h, h.UploadBinaryPath) - if err := h.Upload(h.UploadBinaryPath, h.Configurer.K0sBinaryPath(), exec.Sudo(h)); err != nil { - return err + stat, err := os.Stat(h.UploadBinaryPath) + if err != nil { + return fmt.Errorf("failed to stat %s: %w", h.UploadBinaryPath, err) + } + if h.FileChanged(h.UploadBinaryPath, h.Configurer.K0sBinaryPath()) { + log.Infof("%s: uploading k0s binary from %s", h, h.UploadBinaryPath) + if err := h.Upload(h.UploadBinaryPath, h.Configurer.K0sBinaryPath(), exec.Sudo(h)); err != nil { + return err + } + } else { + log.Infof("%s: k0s binary %s already exists on the target and hasn't been changed, skipping upload", h, h.UploadBinaryPath) } if err := h.Configurer.Chmod(h, h.Configurer.K0sBinaryPath(), "0700", exec.Sudo(h)); err != nil { return err } + log.Debugf("%s: touching %s", h, h.Configurer.K0sBinaryPath()) + if err := h.Configurer.Touch(h, h.Configurer.K0sBinaryPath(), stat.ModTime(), exec.Sudo(h)); err != nil { + return fmt.Errorf("failed to touch %s: %w", h.Configurer.K0sBinaryPath(), err) + } + h.Metadata.K0sBinaryVersion = p.Config.Spec.K0s.Version return nil diff --git a/phase/uploadfiles.go b/phase/uploadfiles.go index bf352db9..d332875c 100644 --- a/phase/uploadfiles.go +++ b/phase/uploadfiles.go @@ -2,6 +2,7 @@ package phase import ( "fmt" + "os" "path" "github.com/alessio/shellescape" @@ -107,8 +108,12 @@ func (p *UploadFiles) uploadFile(h *cluster.Host, f *cluster.UploadFile) error { return err } - if err := h.Upload(path.Join(f.Base, s.Path), dest, exec.Sudo(h)); err != nil { - return err + if h.FileChanged(src, dest) { + if err := h.Upload(path.Join(f.Base, s.Path), dest, exec.Sudo(h)); err != nil { + return err + } + } else { + log.Infof("%s: file already exists and hasn't been changed, skipping upload", h) } if owner != "" { @@ -121,6 +126,14 @@ func (p *UploadFiles) uploadFile(h *cluster.Host, f *cluster.UploadFile) error { if err := h.Configurer.Chmod(h, dest, s.PermMode, exec.Sudo(h)); err != nil { return err } + stat, err := os.Stat(src) + if err != nil { + return fmt.Errorf("failed to stat %s: %s", src, err) + } + log.Debugf("%s: touching %s", h, dest) + if err := h.Configurer.Touch(h, dest, stat.ModTime(), exec.Sudo(h)); err != nil { + return fmt.Errorf("failed to touch %s: %w", dest, err) + } } return nil diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go index 245e2895..852f6c0d 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/host.go @@ -3,6 +3,7 @@ package cluster import ( "encoding/json" "fmt" + gos "os" "regexp" "strconv" "strings" @@ -112,6 +113,8 @@ type configurer interface { TempFile(os.Host) (string, error) UpdateServiceEnvironment(os.Host, string, map[string]string) error CleanupServiceEnvironment(os.Host, string) error + Stat(os.Host, string, ...exec.Option) (*os.FileInfo, error) + Touch(os.Host, string, time.Time, ...exec.Option) error } // HostMetadata resolved metadata for host @@ -494,3 +497,30 @@ func (h *Host) WaitKubeAPIReady(port int) error { // thus we need to accept both 200 and 401 as valid statuses when checking kube api return h.WaitHTTPStatus(fmt.Sprintf("https://localhost:%d/version", port), 200, 401) } + +// FileChanged returns true when a remote file has different size or mtime compared to local +// or if an error occurs +func (h *Host) FileChanged(lpath, rpath string) bool { + lstat, err := gos.Stat(lpath) + if err != nil { + log.Debugf("%s: local stat failed: %s", h, err) + return true + } + rstat, err := h.Configurer.Stat(h, rpath, exec.Sudo(h)) + if err != nil { + log.Debugf("%s: remote stat failed: %s", h, err) + return true + } + + if lstat.Size() != rstat.Size() { + log.Debugf("%s: file sizes for %s differ (%d vs %d)", h, lpath, lstat.Size(), rstat.Size()) + return true + } + + if !lstat.ModTime().Equal(rstat.ModTime()) { + log.Debugf("%s: file modtimes for %s differ (%s vs %s)", h, lpath, lstat.ModTime(), rstat.ModTime()) + return true + } + + return false +}