diff --git a/go.mod b/go.mod index 860a3b930..f8ba19f3e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/cppforlife/color v1.9.1-0.20200716202919-6706ac40b835 github.com/cppforlife/go-cli-ui v0.0.0-20220425131040-94f26b16bc14 github.com/cppforlife/go-patch v0.2.0 + github.com/google/go-cmp v0.5.9 github.com/hashicorp/go-version v1.6.0 github.com/k14s/difflib v0.0.0-20201117154628-0c031775bf57 github.com/k14s/ytt v0.36.0 @@ -33,7 +34,6 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.12 // indirect diff --git a/pkg/kapp/cmd/app/deploy.go b/pkg/kapp/cmd/app/deploy.go index 13c00a213..53bfdfd91 100644 --- a/pkg/kapp/cmd/app/deploy.go +++ b/pkg/kapp/cmd/app/deploy.go @@ -5,12 +5,18 @@ package app import ( "fmt" + "io/fs" "os" "sort" "strings" "github.com/cppforlife/go-cli-ui/ui" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + ctlapp "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/app" ctlcap "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/clusterapply" cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core" @@ -23,10 +29,6 @@ import ( ctllogs "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/logs" ctlres "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/resources" ctlresm "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/resourcesmisc" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes" ) const ( @@ -47,6 +49,8 @@ type DeployOptions struct { DeployFlags DeployFlags ResourceTypesFlags ResourceTypesFlags LabelFlags LabelFlags + + FileSystem fs.FS } func NewDeployOptions(ui ui.UI, depsFactory cmdcore.DepsFactory, logger logger.Logger) *DeployOptions { @@ -329,7 +333,7 @@ func (o *DeployOptions) newResourcesFromFiles() ([]ctlres.Resource, error) { return nil, fmt.Errorf("Expected at least one --file (-f) specified with a file or directory path") } for _, file := range o.FileFlags.Files { - fileRs, err := ctlres.NewFileResources(file) + fileRs, err := ctlres.NewFileResources(o.FileSystem, file) if err != nil { return nil, err } diff --git a/pkg/kapp/cmd/tools/diff.go b/pkg/kapp/cmd/tools/diff.go index 87ce94e7b..7e7e74e8f 100644 --- a/pkg/kapp/cmd/tools/diff.go +++ b/pkg/kapp/cmd/tools/diff.go @@ -4,8 +4,11 @@ package tools import ( + "io/fs" + "github.com/cppforlife/go-cli-ui/ui" "github.com/spf13/cobra" + ctlcap "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/clusterapply" cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core" ctldiff "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/diff" @@ -19,6 +22,8 @@ type DiffOptions struct { FileFlags FileFlags FileFlags2 FileFlags2 DiffFlags DiffFlags + + FileSystem fs.FS } func NewDiffOptions(ui ui.UI, depsFactory cmdcore.DepsFactory) *DiffOptions { @@ -71,7 +76,7 @@ func (o *DiffOptions) fileResources(files []string) ([]ctlres.Resource, error) { var newResources []ctlres.Resource for _, file := range files { - fileRs, err := ctlres.NewFileResources(file) + fileRs, err := ctlres.NewFileResources(o.FileSystem, file) if err != nil { return nil, err } diff --git a/pkg/kapp/cmd/tools/inspect.go b/pkg/kapp/cmd/tools/inspect.go index 38341026b..9efdb790e 100644 --- a/pkg/kapp/cmd/tools/inspect.go +++ b/pkg/kapp/cmd/tools/inspect.go @@ -4,8 +4,11 @@ package tools import ( + "io/fs" + "github.com/cppforlife/go-cli-ui/ui" "github.com/spf13/cobra" + cmdcore "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core" ctlres "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/resources" ) @@ -17,6 +20,8 @@ type InspectOptions struct { FileFlags FileFlags ResourceFilterFlags ResourceFilterFlags Raw bool + + FileSystem fs.FS } func NewInspectOptions(ui ui.UI, depsFactory cmdcore.DepsFactory) *InspectOptions { @@ -49,7 +54,7 @@ func (o *InspectOptions) inspectFiles() error { } for _, file := range o.FileFlags.Files { - fileRs, err := ctlres.NewFileResources(file) + fileRs, err := ctlres.NewFileResources(o.FileSystem, file) if err != nil { return err } diff --git a/pkg/kapp/resources/file_resources.go b/pkg/kapp/resources/file_resources.go index 6b2994219..3311745a6 100644 --- a/pkg/kapp/resources/file_resources.go +++ b/pkg/kapp/resources/file_resources.go @@ -5,6 +5,7 @@ package resources import ( "fmt" + "io/fs" "os" "path/filepath" "sort" @@ -19,7 +20,12 @@ type FileResource struct { fileSrc FileSource } -func NewFileResources(file string) ([]FileResource, error) { +// NewFileResources inspects file and returns a slice of FileResource objects. If file is "-", a FileResource for STDIN +// is returned. If it is prefixed with either http:// or https://, a FileResource that supports an HTTP transport is +// returned. If file is a directory, one FileResource object is returned for each file in the directory with an allowed +// extension (.json, .yml, .yaml). If file is not a directory, a FileResource object is returned for that one file. If +// fsys is nil, NewFileResources uses the OS's file system. Otherwise, it uses the passed in file system. +func NewFileResources(fsys fs.FS, file string) ([]FileResource, error) { var fileRs []FileResource switch { @@ -30,16 +36,22 @@ func NewFileResources(file string) ([]FileResource, error) { fileRs = append(fileRs, NewFileResource(NewHTTPFileSource(file))) default: - fileInfo, err := os.Stat(file) + dir, err := isDir(fsys, file) if err != nil { - return nil, fmt.Errorf("Checking file: %v", err) + return nil, err } - if fileInfo.IsDir() { - var paths []string + if dir { + // The typical command line invocation won't set fsys. If it comes in nil, create a new DirFS rooted at + // file, then set file to '.' (current working directory) so the fs.WalkDir call below works correctly. + if fsys == nil { + fsys = os.DirFS(file) + file = "." + } - err := filepath.Walk(file, func(path string, fi os.FileInfo, err error) error { - if err != nil || fi.IsDir() { + var paths []string + err := fs.WalkDir(fsys, file, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { return err } ext := filepath.Ext(path) @@ -51,16 +63,16 @@ func NewFileResources(file string) ([]FileResource, error) { return nil }) if err != nil { - return nil, fmt.Errorf("Listing files '%s'", file) + return nil, fmt.Errorf("error listing file %q", file) } sort.Strings(paths) for _, path := range paths { - fileRs = append(fileRs, NewFileResource(NewLocalFileSource(path))) + fileRs = append(fileRs, NewFileResource(NewLocalFileSource(fsys, path))) } } else { - fileRs = append(fileRs, NewFileResource(NewLocalFileSource(file))) + fileRs = append(fileRs, NewFileResource(NewLocalFileSource(fsys, file))) } } @@ -94,3 +106,38 @@ func (r FileResource) Resources() ([]Resource, error) { return resources, nil } + +// isDir returns if path is a directory. If fsys is nil, isDir calls os.Stat(path); otherwise, it checks path inside +// fsys. +func isDir(fsys fs.FS, path string) (bool, error) { + if fsys == nil { + fileInfo, err := os.Stat(path) + if err != nil { + return false, fmt.Errorf("error stat'ing file %q: %v", path, err) + } + return fileInfo.IsDir(), nil + } + + switch t := fsys.(type) { + case fs.StatFS: + fileInfo, err := t.Stat(path) + if err != nil { + return false, fmt.Errorf("error stat'ing file %q: %v", path, err) + } + return fileInfo.IsDir(), nil + case fs.FS: + f, err := t.Open(path) + if err != nil { + return false, fmt.Errorf("error opening file %q: %v", path, err) + } + defer f.Close() + + fileInfo, err := f.Stat() + if err != nil { + return false, fmt.Errorf("error stat'ing file %q: %v", path, err) + } + return fileInfo.IsDir(), nil + default: + return false, fmt.Errorf("error determining if %q is a directory: unexpected FS type %T", path, fsys) + } +} diff --git a/pkg/kapp/resources/file_sources.go b/pkg/kapp/resources/file_sources.go index 2873b4211..a5f4cadaa 100644 --- a/pkg/kapp/resources/file_sources.go +++ b/pkg/kapp/resources/file_sources.go @@ -6,6 +6,7 @@ package resources import ( "fmt" "io" + "io/fs" "net/http" "os" ) @@ -34,14 +35,31 @@ func (s StdinSource) Description() string { return "stdin" } func (s StdinSource) Bytes() ([]byte, error) { return io.ReadAll(os.Stdin) } type LocalFileSource struct { + fsys fs.FS path string } var _ FileSource = LocalFileSource{} -func NewLocalFileSource(path string) LocalFileSource { return LocalFileSource{path} } -func (s LocalFileSource) Description() string { return fmt.Sprintf("file '%s'", s.path) } -func (s LocalFileSource) Bytes() ([]byte, error) { return os.ReadFile(s.path) } +func NewLocalFileSource(fsys fs.FS, path string) LocalFileSource { + return LocalFileSource{fsys: fsys, path: path} +} +func (s LocalFileSource) Description() string { return fmt.Sprintf("file '%s'", s.path) } +func (s LocalFileSource) Bytes() ([]byte, error) { + switch t := s.fsys.(type) { + case fs.ReadFileFS: + return t.ReadFile(s.path) + case fs.FS: + f, err := t.Open(s.path) + if err != nil { + return nil, err + } + defer f.Close() + return fs.ReadFile(s.fsys, s.path) + default: + return os.ReadFile(s.path) + } +} type HTTPFileSource struct { url string diff --git a/test/e2e/deploy_filesystem_test.go b/test/e2e/deploy_filesystem_test.go new file mode 100644 index 000000000..e210852cd --- /dev/null +++ b/test/e2e/deploy_filesystem_test.go @@ -0,0 +1,161 @@ +// Copyright 2023 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "fmt" + "testing" + "testing/fstest" + "time" + + "github.com/cppforlife/go-cli-ui/ui" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/app" + "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/cmd/core" + "github.com/vmware-tanzu/carvel-kapp/pkg/kapp/logger" +) + +func TestDeployFilesystem(t *testing.T) { + env := BuildEnv(t) + testLogger := Logger{} + kapp := Kapp{t, env.Namespace, env.KappBinaryPath, testLogger} + appName := "test-deploy-filesystem" + cleanUp := func() { + kapp.Run([]string{"delete", "-a", appName}) + } + + cleanUp() + t.Cleanup(cleanUp) + + theUI := ui.NewConfUI(ui.NewNoopLogger()) + theUI.EnableNonInteractive() + + configFactory := core.NewConfigFactoryImpl() + configFactory.ConfigureClient(100, 200) + + // We don't need to customize any of these, but we'll get panics if they're nil + configFactory.ConfigurePathResolver(func() (string, error) { + return "", nil + }) + configFactory.ConfigureContextResolver(func() (string, error) { + return "", nil + }) + configFactory.ConfigureYAMLResolver(func() (string, error) { + return "", nil + }) + + depsFactory := core.NewDepsFactoryImpl(configFactory, theUI) + log := logger.NewUILogger(theUI) + + deployOptions := app.NewDeployOptions(theUI, depsFactory, log) + deployOptions.AppFlags.NamespaceFlags.Name = env.Namespace + deployOptions.AppFlags.Name = appName + deployOptions.FileFlags.Files = []string{ + "dir1", // directory + "file1.yaml", // file in root of fs + "dir2/file2.yaml", // file in subdir + "dir3", // another directory + } + + flagsFactory := core.NewFlagsFactory(configFactory, depsFactory) + + deployCmd := app.NewDeployCmd(deployOptions, flagsFactory) + + now := time.Now().Unix() + labelSelector := fmt.Sprintf("now=%d", now) + + inMemFS := newFSBuilder(). + file( + "dir1/cm1.yaml", + testConfigMap(env.Namespace, "cm1", now), + ). + file( + "dir1/cm2.yaml", + testConfigMap(env.Namespace, "cm2", now), + ). + file( + "file1.yaml", + testConfigMap(env.Namespace, "cm3", now), + ). + file( + "dir2/file2.yaml", + testConfigMap(env.Namespace, "cm4", now), + ). + file( + "dir3/cm5.yaml", + testConfigMap(env.Namespace, "cm5", now), + ). + file( + "dir3/cm6.yaml", + testConfigMap(env.Namespace, "cm6", now), + ). + toFS() + + deployOptions.FileSystem = inMemFS + + err := deployCmd.Execute() + require.NoError(t, err) + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + restConfig, err := kubeConfig.ClientConfig() + require.NoError(t, err, "error creating rest config") + + kubeClient, err := kubernetes.NewForConfig(restConfig) + require.NoError(t, err, "error creating k8s clientset") + + ctx := context.Background() + const allNamespaces = "" + configMaps, err := kubeClient.CoreV1().ConfigMaps(allNamespaces).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err, "error listing ConfigMaps") + require.Len(t, configMaps.Items, 6) + expectedNames := sets.New[string]("cm1", "cm2", "cm3", "cm4", "cm5", "cm6") + actualNames := sets.New[string]() + for _, cm := range configMaps.Items { + actualNames.Insert(cm.Name) + } + require.Empty(t, cmp.Diff(expectedNames, actualNames)) +} + +type fsBuilder struct { + mapFS fstest.MapFS +} + +func newFSBuilder() *fsBuilder { + return &fsBuilder{ + mapFS: fstest.MapFS{}, + } +} + +func (b *fsBuilder) file(name, contents string) *fsBuilder { + b.mapFS[name] = &fstest.MapFile{ + Data: []byte(contents), + } + return b +} + +func (b *fsBuilder) toFS() fstest.MapFS { + return b.mapFS +} + +func testConfigMap(ns, name string, label int64) string { + return fmt.Sprintf(` +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: %s + name: %s + labels: + now: "%d" +`, ns, name, label) +}