From cc059fc960bc4086550cb79315ea2cc78f48a519 Mon Sep 17 00:00:00 2001 From: Christopher Hunter Date: Mon, 11 Dec 2023 10:44:00 -0800 Subject: [PATCH] convert compile command tests to ginkgo style I prefer standard tests but I rewrote the tests to match existing test conventions. This commit also adds test coverage for compilation and release parsing failure cases. I sorted the imports manually and applied gofumpt I also updated to yaml version in the test to v3 (using v2 was a mistake) --- go.mod | 2 + go.sum | 4 + releasetarball/compile.go | 55 ++-- releasetarball/compile_test.go | 508 ++++++++++++++++++--------------- vendor/modules.txt | 6 + 5 files changed, 321 insertions(+), 254 deletions(-) diff --git a/go.mod b/go.mod index dc29acc0b..d69cbaac2 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( code.cloudfoundry.org/tlsconfig v0.0.0-20231017135636-f0e44068c22f github.com/Microsoft/hcsshim v0.8.14 github.com/charlievieth/fs v0.0.3 + github.com/cloudfoundry/bosh-cli v6.4.1+incompatible github.com/cloudfoundry/bosh-davcli v0.0.297 github.com/cloudfoundry/bosh-utils v0.0.416 github.com/cloudfoundry/gosigar v1.3.37 @@ -33,6 +34,7 @@ require ( golang.org/x/net v0.19.0 golang.org/x/sys v0.15.0 golang.org/x/tools v0.16.0 + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 inet.af/wf v0.0.0-20221017222439-36129f591884 ) diff --git a/go.sum b/go.sum index f0f691691..74ed5fea8 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/charlievieth/fs v0.0.3 h1:3lZQXTj4PbE81CVPwALSn+JoyCNXkZgORHN6h2XHGlg github.com/charlievieth/fs v0.0.3/go.mod h1:hD4sRzto1Hw8zCua76tNVKZxaeZZr1RiKftjAJQRLLo= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= +github.com/cloudfoundry/bosh-cli v6.4.1+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= github.com/cloudfoundry/bosh-davcli v0.0.297 h1:9mOe5OtE3y69zz6YsoQO1zocc0CQWzvcNFGvM4Jylyk= github.com/cloudfoundry/bosh-davcli v0.0.297/go.mod h1:O8QPMRoumd/7+5Pv/hUnyXcW28DYS+dBOgB3hpYACpY= github.com/cloudfoundry/bosh-utils v0.0.416 h1:hAmXTL1B6YepkIIxdQFfhzj4GP22Demz3Ugzex+A3+Q= @@ -397,6 +399,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/releasetarball/compile.go b/releasetarball/compile.go index e1a322f5f..d9b99e01d 100644 --- a/releasetarball/compile.go +++ b/releasetarball/compile.go @@ -29,6 +29,15 @@ import ( "gopkg.in/yaml.v3" "code.cloudfoundry.org/clock" + + "github.com/cloudfoundry/bosh-cli/release/manifest" + + boshblob "github.com/cloudfoundry/bosh-utils/blobstore" + boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" + boshcmd "github.com/cloudfoundry/bosh-utils/fileutil" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + boshsys "github.com/cloudfoundry/bosh-utils/system" + boshmodels "github.com/cloudfoundry/bosh-agent/agent/applier/models" boshap "github.com/cloudfoundry/bosh-agent/agent/applier/packages" boshagentblobstore "github.com/cloudfoundry/bosh-agent/agent/blobstore" @@ -37,13 +46,6 @@ import ( "github.com/cloudfoundry/bosh-agent/agent/httpblobprovider" "github.com/cloudfoundry/bosh-agent/agent/httpblobprovider/blobstore_delegator" "github.com/cloudfoundry/bosh-agent/settings/directories" - boshblob "github.com/cloudfoundry/bosh-utils/blobstore" - boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" - boshcmd "github.com/cloudfoundry/bosh-utils/fileutil" - boshlog "github.com/cloudfoundry/bosh-utils/logger" - boshsys "github.com/cloudfoundry/bosh-utils/system" - - "github.com/cloudfoundry/bosh-cli/release/manifest" ) const ( @@ -77,14 +79,14 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, log.Printf("Reading BOSH Release Manifest from tarball %s", boshReleaseTarballPath) m, err := Manifest(boshReleaseTarballPath) if err != nil { - return "", err + return "", fmt.Errorf("failed to parse release manifest: %w", err) } log.Printf("Release %s/%s has %d packages", m.Name, m.Version, len(m.Packages)) log.Printf("Extracting packages") blobstoreIDs, err := extractPackages(m, blobsDirectory, boshReleaseTarballPath) if err != nil { - return "", err + return "", fmt.Errorf("failed to extract packages from tarball: %w", err) } log.Printf("Starting packages compilation") @@ -95,7 +97,10 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, } var compiledPackages []boshmodels.Package for _, p := range packages { - compiledPackages = compilePackage(p, blobstoreIDs, compiledPackages, compiler) + compiledPackages, err = compilePackage(compiledPackages, p, blobstoreIDs, compiler) + if err != nil { + return "", fmt.Errorf("failed to compile package %s/%s: %w", p.Name, p.Version, err) + } } log.Printf("Finished packages compilation after %s", time.Since(start)) @@ -103,11 +108,11 @@ func Compile(compiler boshcomp.Compiler, boshReleaseTarballPath, blobsDirectory, return writeCompiledRelease(m, outputDirectory, stemcellSlug, blobsDirectory, boshReleaseTarballPath, compiledPackages) } -func compilePackage(p manifest.PackageRef, blobstoreIDs map[string]string, compiledPackages []boshmodels.Package, compiler boshcomp.Compiler) []boshmodels.Package { +func compilePackage(compiledPackages []boshmodels.Package, p manifest.PackageRef, blobstoreIDs map[string]string, compiler boshcomp.Compiler) ([]boshmodels.Package, error) { log.Printf("Compiling package %s/%s", p.Name, p.Version) digest, err := boshcrypto.ParseMultipleDigest(p.SHA1) if err != nil { - log.Fatal(err) + return nil, err } pkg := boshcomp.Package{ BlobstoreID: path.Base(blobstoreIDs[p.SHA1]), @@ -124,7 +129,7 @@ func compilePackage(p manifest.PackageRef, blobstoreIDs map[string]string, compi } compiledBlobID, compiledDigest, err := compiler.Compile(pkg, modelsDeps) if err != nil { - log.Fatal(err) + return nil, err } log.Printf("Finished compiling release %s/%s BlobstoreID=%s", p.Name, p.Version, compiledBlobID) return append(compiledPackages, boshmodels.Package{ @@ -134,7 +139,7 @@ func compilePackage(p manifest.PackageRef, blobstoreIDs map[string]string, compi Sha1: compiledDigest, BlobstoreID: compiledBlobID, }, - }) + }), nil } func extractPackages(m manifest.Manifest, blobsDirectory, releaseTarballPath string) (map[string]string, error) { @@ -147,7 +152,7 @@ func extractPackages(m manifest.Manifest, blobsDirectory, releaseTarballPath str return p.Name+".tgz" == path.Base(h.Name) }) if pkgIndex < 0 { - return true, nil + return true, fmt.Errorf("package not found in release manifest") } p := m.Packages[pkgIndex] dstFilepath := filepath.Join(blobsDirectory, p.Name+".tgz") @@ -277,21 +282,25 @@ func Manifest(releaseFilePath string) (manifest.Manifest, error) { m manifest.Manifest found = false ) - err := walkTarballFiles(releaseFilePath, func(name string, _ *tar.Header, r io.Reader) (bool, error) { + err := walkTarballFiles(releaseFilePath, readReleaseManifest(&found, &m)) + if !found { + return m, fmt.Errorf("failed to find %s in tarball", releaseManifestFilename) + } + return m, err +} + +func readReleaseManifest(found *bool, m *manifest.Manifest) func(string, *tar.Header, io.Reader) (bool, error) { + return func(name string, _ *tar.Header, r io.Reader) (bool, error) { if path.Base(name) != "release.MF" { return true, nil } - found = true + *found = true buf, err := io.ReadAll(r) if err != nil { return false, err } - return false, yaml.Unmarshal(buf, &m) - }) - if !found { - return m, fmt.Errorf("failed to find %s in tarball", releaseManifestFilename) + return false, yaml.Unmarshal(buf, m) } - return m, err } type tarballWalkFunc func(fullName string, h *tar.Header, r io.Reader) (bool, error) @@ -312,7 +321,7 @@ func walkTarballFiles(releaseFilePath string, file tarballWalkFunc) error { for { h, err := r.Next() if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { return nil } return err diff --git a/releasetarball/compile_test.go b/releasetarball/compile_test.go index 57186951a..2ba3186b8 100644 --- a/releasetarball/compile_test.go +++ b/releasetarball/compile_test.go @@ -13,15 +13,15 @@ import ( "maps" "os" "path/filepath" - "reflect" - "slices" - "strings" "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/cloudfoundry/bosh-cli/release/manifest" boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/cloudfoundry/bosh-agent/agent/applier/models" "github.com/cloudfoundry/bosh-agent/agent/compiler" @@ -30,211 +30,324 @@ import ( "github.com/cloudfoundry/bosh-agent/settings/directories" ) -func init() { +func TestSuite(t *testing.T) { log.Default().SetOutput(io.Discard) + RegisterFailHandler(Fail) + RunSpecs(t, "Release Compilation Suite") } -func TestCompiler(t *testing.T) { - d := directories.NewProvider(t.TempDir()) +var _ = Describe("NewCompiler", func() { + When("initialized", func() { + var ( + temporaryDirectory string + setupErr error + ) - _, err := releasetarball.NewCompiler(d) - if err != nil { - log.Fatal(err) - } -} + BeforeEach(func() { + temporaryDirectory, setupErr = os.MkdirTemp("", "") + }) + AfterEach(func() { + setupErr = errors.Join(setupErr, os.RemoveAll(temporaryDirectory)) + }) + + It("returns a result and no error", func() { + Expect(setupErr).NotTo(HaveOccurred()) + + d := directories.NewProvider(temporaryDirectory) + err := os.MkdirAll(d.BlobsDir(), 0o766) + Expect(err).NotTo(HaveOccurred()) + + result, err := releasetarball.NewCompiler(d) + Expect(err).NotTo(HaveOccurred()) + Expect(result).NotTo(BeNil()) + }) + }) +}) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -fake-name Compiler -o internal/fakes/compiler.go github.com/cloudfoundry/bosh-agent/agent/compiler.Compiler -func TestCompile(t *testing.T) { - t.Run("a release with multiple source packages", func(t *testing.T) { - d := temporaryDirectoriesProvider(t) - releasesOutputDir := t.TempDir() +var _ = Describe("Compile", func() { + const stemcellSlug = "banana-slug/1.23" + var ( + setupErr error + + releasesOutputDir string + + pkgCompiler *fakes.Compiler + + sourceTarballPath string - pkgCompiler := new(fakes.Compiler) + d directories.Provider + ) + BeforeEach(func() { + d = directories.NewProvider(GinkgoT().TempDir()) + err := os.MkdirAll(d.BlobsDir(), 0o766) + Expect(err).NotTo(HaveOccurred()) + releasesOutputDir = GinkgoT().TempDir() + pkgCompiler = new(fakes.Compiler) pkgCompiler.CompileCalls(fakeCompilation(d)) + }) - const stemcellSlug = "banana-slug/1.23" - sourceBOSHReleaseFilepath := filepath.Join("testdata", "log-cache-release-3.0.9.tgz") - resultPath, err := releasetarball.Compile(pkgCompiler, sourceBOSHReleaseFilepath, d.BlobsDir(), releasesOutputDir, stemcellSlug) - if err != nil { - t.Error(err) - } - if exp := filepath.Join(releasesOutputDir, "log-cache-3.0.9-banana-slug-1.23.tgz"); exp != resultPath { - t.Errorf("unexpected resultPath: got %q expected %q", resultPath, exp) - } + When("compiling a tarball with compiled packages", func() { + BeforeEach(func() { + sourceTarballPath = filepath.Join("testdata", "log-cache-3.0.9-banana-slug-1.23.tgz") + }) - const ( - expectedUname = "root" - expectedUID = 0 - ) + It("does not compile any of the packages", func() { + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).NotTo(HaveOccurred()) + Expect(pkgCompiler.CompileCallCount()).To(BeZero()) + }) + }) - infos := listFileNamesInTarball(t, resultPath) - const expectedFiles = 15 - if got, exp := len(infos), expectedFiles; got != exp { - t.Fatalf("got %d files expected %d", got, exp) - } - for i, tt := range [expectedFiles]struct { - Name string - Mode int64 - }{ - {Name: "release.MF", Mode: 0o0644}, - {Name: "jobs/", Mode: 0o0755}, - {Name: "jobs/log-cache-gateway.tgz", Mode: 0o0644}, - {Name: "jobs/log-cache.tgz", Mode: 0o0644}, - {Name: "jobs/log-cache-syslog-server.tgz", Mode: 0o0644}, - {Name: "jobs/log-cache-cf-auth-proxy.tgz", Mode: 0o0644}, - {Name: "compiled_packages/", Mode: 0o0755}, - {Name: "compiled_packages/golang-1.20-linux.tgz", Mode: 0o0644}, - {Name: "compiled_packages/log-cache.tgz", Mode: 0o0644}, - {Name: "compiled_packages/log-cache-cf-auth-proxy.tgz", Mode: 0o0644}, - {Name: "compiled_packages/log-cache-gateway.tgz", Mode: 0o0644}, - {Name: "compiled_packages/log-cache-syslog-server.tgz", Mode: 0o0644}, - {Name: "license.tgz", Mode: 0o0644}, - {Name: "LICENSE", Mode: 0o0644}, - {Name: "NOTICE", Mode: 0o0644}, - } { - t.Run(strings.TrimPrefix(tt.Name, "./"), func(t *testing.T) { - h := infos[i] - if got, exp := h.Name, "./"+tt.Name; got != exp { - t.Errorf("expected file header name at index %d to be %q but got %q", i, got, exp) - } - requireOctal(t, h.Mode, tt.Mode) - requireTime(t, h.ChangeTime, time.Time{}) - requireUnameAndUID(t, h, expectedUname, expectedUID) + When("compiling a tarball with compiled packages", func() { + BeforeEach(func() { + releaseInputDir := GinkgoT().TempDir() + sourceTarballPath = filepath.Join(releaseInputDir, "banana.tgz") + + releaseMF, _ := yaml.Marshal(manifest.Manifest{ + Packages: []manifest.PackageRef{ + {Name: "A", Dependencies: []string{"B"}}, + {Name: "B", Dependencies: []string{"C"}}, + {Name: "C", Dependencies: []string{"A"}}, + }, }) - } + tgz, err := createTGZ(simpleFile("release.MF", releaseMF, 0o0644)) + Expect(err).NotTo(HaveOccurred()) + if err := os.WriteFile(sourceTarballPath, tgz, 0o0644); err != nil { + panic(err) + } + }) - sourceManifest, err := releasetarball.Manifest(sourceBOSHReleaseFilepath) - if err != nil { - t.Fatal(err) - } - compiledManifest, err := releasetarball.Manifest(resultPath) - if err != nil { - t.Fatal(err) - } - if got, exp := compiledManifest.CommitHash, sourceManifest.CommitHash; got != exp { - t.Errorf("wrong commit hash got %q expected %q", got, exp) - } - if got, exp := compiledManifest.Name, sourceManifest.Name; got != exp { - t.Errorf("wrong release name got %q expected %q", got, exp) - } - if got, exp := compiledManifest.Version, sourceManifest.Version; got != exp { - t.Errorf("wrong release version got %q expected %q", got, exp) - } - requireEqualJobs(t, compiledManifest.Jobs, sourceManifest.Jobs) - if n := len(compiledManifest.Packages); n != 0 { - t.Errorf("manifest has %d (source aka non-compiled) packages", n) - } - requirePackageCompilation(t, compiledManifest.CompiledPkgs, sourceManifest.Packages, stemcellSlug) + It("does not compile any of the packages", func() { + const stemcellSlug = "banana-slug/1.23" + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).To(MatchError(ContainSubstring("cycle detected"))) + }) }) - t.Run("a tarball with compiled packages", func(t *testing.T) { - d := temporaryDirectoriesProvider(t) - releasesOutputDir := t.TempDir() - sourceTarballPath := filepath.Join("testdata", "log-cache-3.0.9-banana-slug-1.23.tgz") - pkgCompiler := new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(d)) + When("the release manifest is invalid", func() { + BeforeEach(func() { + releaseInputDir := GinkgoT().TempDir() + sourceTarballPath = filepath.Join(releaseInputDir, "banana.tgz") - const stemcellSlug = "banana-slug/1.23" - _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) - if got := pkgCompiler.CompileCallCount(); got != 0 { - t.Errorf("compiler called %d times expected %d", got, 0) - } - if err != nil { - t.Error(err) - } + releaseMF := []byte(`{"name": ["wrong type for name field"]}`) + tgz, err := createTGZ(simpleFile("release.MF", releaseMF, 0o0644)) + Expect(err).NotTo(HaveOccurred()) + if err := os.WriteFile(sourceTarballPath, tgz, 0o0644); err != nil { + panic(err) + } + }) + + It("does not compile any of the packages", func() { + const stemcellSlug = "banana-slug/1.23" + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).To(MatchError(ContainSubstring("failed to parse release manifest"))) + }) }) - t.Run("a tarball without packages", func(t *testing.T) { - d := temporaryDirectoriesProvider(t) - releasesOutputDir := t.TempDir() - sourceTarballPath := filepath.Join("testdata", "bosh-dns-aliases-release-0.0.4.tgz") - pkgCompiler := new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(d)) + When("the compiler returns an error", func() { + BeforeEach(func() { + sourceTarballPath = filepath.Join("testdata", "log-cache-release-3.0.9.tgz") + pkgCompiler.CompileReturns("", nil, fmt.Errorf("banana")) + }) - const stemcellSlug = "banana-slug/1.23" - _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) - if got := pkgCompiler.CompileCallCount(); got != 0 { - t.Errorf("compiler called %d times expected %d", got, 0) - } - if err != nil { - t.Error(err) - } + It("does not compile any of the packages", func() { + const stemcellSlug = "banana-slug/1.23" + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).To(MatchError(ContainSubstring("banana"))) + }) }) - t.Run("a tarball with recursive package dependency", func(t *testing.T) { - releaseMF, _ := yaml.Marshal(manifest.Manifest{ - Packages: []manifest.PackageRef{ - {Name: "A", Dependencies: []string{"B"}}, - {Name: "B", Dependencies: []string{"C"}}, - {Name: "C", Dependencies: []string{"A"}}, - }, + + When("the compiler returns an error", func() { + BeforeEach(func() { + sourceTarballPath = filepath.Join("testdata", "log-cache-release-3.0.9.tgz") + pkgCompiler.CompileReturns("", nil, fmt.Errorf("banana")) }) - tgz, err := createTGZ("release.MF", releaseMF, 0o0644) - if err != nil { - log.Fatal(err) - } - d := temporaryDirectoriesProvider(t) - releasesOutputDir := t.TempDir() - releaseInputDir := t.TempDir() - sourceTarballPath := filepath.Join(releaseInputDir, "banana.tgz") - if err := os.WriteFile(sourceTarballPath, tgz, 0o0644); err != nil { - t.Fatal(err) - } + It("does not compile any of the packages", func() { + const stemcellSlug = "banana-slug/1.23" + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).To(MatchError(ContainSubstring("banana"))) + }) + }) - pkgCompiler := new(fakes.Compiler) - pkgCompiler.CompileCalls(fakeCompilation(d)) + When("package tarball is not found in release manifest", func() { + BeforeEach(func() { + releaseInputDir := GinkgoT().TempDir() + sourceTarballPath = filepath.Join(releaseInputDir, "banana.tgz") + releaseMF, _ := yaml.Marshal(manifest.Manifest{}) + + tgz, err := createTGZ( + simpleFile("release.MF", releaseMF, 0o0644), + simpleFile("packages/a.tgz", nil, 0o0644), + ) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(sourceTarballPath, tgz, 0o0644) + Expect(err).NotTo(HaveOccurred()) + }) - const stemcellSlug = "banana-slug/1.23" - _, err = releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) - if err == nil || !strings.Contains(err.Error(), "cycle detected") { - t.Errorf("expected a recursive tarball error got: %s", err) - } + It("returns a helpful error", func() { + const stemcellSlug = "banana-slug/1.23" + _, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).To(MatchError(ContainSubstring("package not found in release manifest"))) + }) }) -} -func temporaryDirectoriesProvider(t *testing.T) directories.Provider { - t.Helper() - dir := t.TempDir() - dp := directories.NewProvider(dir) + When("compiling a release with multiple source packages", func() { + BeforeEach(func() { + sourceTarballPath = filepath.Join("testdata", "log-cache-release-3.0.9.tgz") + }) - if err := os.MkdirAll(dp.BlobsDir(), 0766); err != nil { - log.Fatal(err) - } + It("writes a compiled release tarball", func() { + resultPath, err := releasetarball.Compile(pkgCompiler, sourceTarballPath, d.BlobsDir(), releasesOutputDir, stemcellSlug) + Expect(err).NotTo(HaveOccurred()) - return dp -} + By("generating a useful filename", func() { + Expect(resultPath).To(Equal(filepath.Join(releasesOutputDir, "log-cache-3.0.9-banana-slug-1.23.tgz"))) + }) + + By("mutating the release manifest", func() { + sourceManifest, err := releasetarball.Manifest(sourceTarballPath) + Expect(err).NotTo(HaveOccurred()) + + compiledManifest, err := releasetarball.Manifest(resultPath) + Expect(err).NotTo(HaveOccurred()) + + Expect(compiledManifest.CommitHash).To(Equal(sourceManifest.CommitHash), "it must not change the commit sha") + Expect(compiledManifest.Name).To(Equal(sourceManifest.Name), "it must not change the name") + Expect(compiledManifest.Version).To(Equal(sourceManifest.Version), "it must not change the version") + + Expect(compiledManifest.Jobs).To(Equal(sourceManifest.Jobs), "it must not change the jobs") + Expect(compiledManifest.Packages).To(HaveLen(0), "it must not leave any source packages") + + Expect(compiledManifest.CompiledPkgs).To(HaveLen(len(sourceManifest.Packages)), "it should convert all the source packages to compiled packages") + + Expect(compiledManifest.CompiledPkgs[0].Name).To(Equal(sourceManifest.Packages[0].Name)) + Expect(compiledManifest.CompiledPkgs[0].Version).To(Equal(sourceManifest.Packages[0].Version)) + Expect(compiledManifest.CompiledPkgs[0].OSVersionSlug).To(Equal(stemcellSlug)) + Expect(compiledManifest.CompiledPkgs[0].SHA1).NotTo(BeZero()) + Expect(compiledManifest.CompiledPkgs[0].Dependencies).To(HaveLen(0)) + + Expect(compiledManifest.CompiledPkgs[1].Name).To(Equal(sourceManifest.Packages[1].Name)) + Expect(compiledManifest.CompiledPkgs[1].Version).To(Equal(sourceManifest.Packages[1].Version)) + Expect(compiledManifest.CompiledPkgs[1].OSVersionSlug).To(Equal(stemcellSlug)) + Expect(compiledManifest.CompiledPkgs[1].SHA1).NotTo(BeZero()) + Expect(compiledManifest.CompiledPkgs[1].Dependencies).To(HaveLen(0)) + + Expect(compiledManifest.CompiledPkgs[2].Name).To(Equal(sourceManifest.Packages[2].Name)) + Expect(compiledManifest.CompiledPkgs[2].Version).To(Equal(sourceManifest.Packages[2].Version)) + Expect(compiledManifest.CompiledPkgs[2].OSVersionSlug).To(Equal(stemcellSlug)) + Expect(compiledManifest.CompiledPkgs[2].SHA1).NotTo(BeZero()) + Expect(compiledManifest.CompiledPkgs[2].Dependencies).To(HaveLen(0)) + + Expect(compiledManifest.CompiledPkgs[3].Name).To(Equal(sourceManifest.Packages[3].Name)) + Expect(compiledManifest.CompiledPkgs[3].Version).To(Equal(sourceManifest.Packages[3].Version)) + Expect(compiledManifest.CompiledPkgs[3].OSVersionSlug).To(Equal(stemcellSlug)) + Expect(compiledManifest.CompiledPkgs[3].SHA1).NotTo(BeZero()) + Expect(compiledManifest.CompiledPkgs[3].Dependencies).To(HaveLen(0)) + + Expect(compiledManifest.CompiledPkgs[4].Name).To(Equal(sourceManifest.Packages[4].Name)) + Expect(compiledManifest.CompiledPkgs[4].Version).To(Equal(sourceManifest.Packages[4].Version)) + Expect(compiledManifest.CompiledPkgs[4].OSVersionSlug).To(Equal(stemcellSlug)) + Expect(compiledManifest.CompiledPkgs[4].SHA1).NotTo(BeZero()) + Expect(compiledManifest.CompiledPkgs[4].Dependencies).To(HaveLen(0)) + }) + + infos := listFileNamesInTarball(GinkgoT(), resultPath) + const expectedFiles = 15 + + Expect(infos).To(HaveLen(expectedFiles)) + const ( + expectedUname = "root" + expectedUID = 0 + ) + for i, tt := range [expectedFiles]struct { + Name string + Mode int64 + }{ + {Name: "release.MF", Mode: 0o0644}, + {Name: "jobs/", Mode: 0o0755}, + {Name: "jobs/log-cache-gateway.tgz", Mode: 0o0644}, + {Name: "jobs/log-cache.tgz", Mode: 0o0644}, + {Name: "jobs/log-cache-syslog-server.tgz", Mode: 0o0644}, + {Name: "jobs/log-cache-cf-auth-proxy.tgz", Mode: 0o0644}, + {Name: "compiled_packages/", Mode: 0o0755}, + {Name: "compiled_packages/golang-1.20-linux.tgz", Mode: 0o0644}, + {Name: "compiled_packages/log-cache.tgz", Mode: 0o0644}, + {Name: "compiled_packages/log-cache-cf-auth-proxy.tgz", Mode: 0o0644}, + {Name: "compiled_packages/log-cache-gateway.tgz", Mode: 0o0644}, + {Name: "compiled_packages/log-cache-syslog-server.tgz", Mode: 0o0644}, + {Name: "license.tgz", Mode: 0o0644}, + {Name: "LICENSE", Mode: 0o0644}, + {Name: "NOTICE", Mode: 0o0644}, + } { + h := infos[i] + Expect(h.Name).To(Equal("./"+tt.Name), fmt.Sprintf("expected file header name at index %d", i)) + Expect(h.Mode).To(Equal(tt.Mode)) + Expect(h.ChangeTime.IsZero()).To(BeTrue()) + Expect(h.ChangeTime.IsZero()).To(BeTrue()) + Expect(h.Uname).To(Equal(expectedUname)) + Expect(h.Uid).To(Equal(expectedUID)) + } + }) + }) + + It("returns a result and no error", func() { + Expect(setupErr).NotTo(HaveOccurred()) + }) +}) func closeAndIgnoreError(c io.Closer) { _ = c.Close() } -func createTGZ(fileName string, content []byte, mode int64) ([]byte, error) { +type writeTarballFileFunc func(tw *tar.Writer) error + +func createTGZ(functions ...writeTarballFileFunc) ([]byte, error) { fn := func(w io.Writer) error { gw := gzip.NewWriter(w) defer closeAndIgnoreError(gw) tw := tar.NewWriter(gw) defer closeAndIgnoreError(tw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: fileName, - Size: int64(len(content)), - ModTime: time.Time{}, - AccessTime: time.Time{}, - ChangeTime: time.Time{}, - Mode: mode, - }); err != nil { - return err + for _, fn := range functions { + if err := fn(tw); err != nil { + return err + } } - _, err := tw.Write(content) - return err + return nil } w := new(bytes.Buffer) err := fn(w) return w.Bytes(), err } -func listFileNamesInTarball(t *testing.T, filePath string) []tar.Header { +func simpleFile(name string, content []byte, mode int64) writeTarballFileFunc { + return func(tw *tar.Writer) error { + if err := tw.WriteHeader(newSimpleFileHeader(name, int64(len(content)), mode)); err != nil { + return err + } + _, err := tw.Write(content) + if err != nil { + return err + } + return nil + } +} + +func newSimpleFileHeader(name string, length, mode int64) *tar.Header { + return &tar.Header{ + Typeflag: tar.TypeReg, + Name: name, + Size: length, + ModTime: time.Time{}, + AccessTime: time.Time{}, + ChangeTime: time.Time{}, + Mode: mode, + } +} + +func listFileNamesInTarball(t GinkgoTInterface, filePath string) []tar.Header { f, err := os.Open(filePath) if err != nil { t.Fatal(err) @@ -264,7 +377,7 @@ func listFileNamesInTarball(t *testing.T, filePath string) []tar.Header { func fakeCompilation(d directories.Provider) func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { return func(c compiler.Package, packages []models.Package) (string, boshcrypto.Digest, error) { - blobContent, err := createTGZ("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744) + blobContent, err := createTGZ(simpleFile("packaging", fmt.Appendf(nil, `"echo Compiled %q`, c.Name), 0o0744)) if err != nil { log.Fatal(err) } @@ -278,70 +391,3 @@ func fakeCompilation(d directories.Provider) func(c compiler.Package, packages [ return compiledBlobstoreID, digest, nil } } - -func requireOctal(t *testing.T, got, exp int64) { - t.Helper() - if got != exp { - t.Errorf("wrong mode expected 0o%04o got 0o%04o", exp, got) - } -} - -func requireTime(t *testing.T, got, exp time.Time) { - t.Helper() - if !got.Equal(exp) { - t.Errorf("wrong time expected %q got %q", exp, got) - } -} - -func requireEqualJobs(t *testing.T, gotJobs, expJobs []manifest.JobRef) { - t.Helper() - - isEqual := slices.EqualFunc(gotJobs, expJobs, func(got manifest.JobRef, exp manifest.JobRef) bool { - if reflect.DeepEqual(got, exp) { - return true - } - t.Logf("jobs are not equal:\n\tgot: %#v\n\texp: %#v", got, exp) - return false - }) - if !isEqual { - t.Fail() - } -} - -func requireUnameAndUID(t *testing.T, h tar.Header, expUname string, expUID int) { - t.Helper() - if got := h.Uname; got != expUname { - t.Errorf("wrong uname expected %q got %q", expUname, got) - } - if got := h.Uid; got != expUID { - t.Errorf("wrong uid expected %d got %d", expUID, got) - } -} - -func requirePackageCompilation(t *testing.T, compiled []manifest.CompiledPackageRef, source []manifest.PackageRef, stemcellSlug string) { - t.Helper() - if got, exp := len(compiled), len(source); got != exp { - t.Errorf("got %d compiled packages expected %d", got, exp) - } - for i := range compiled { - if got, exp := compiled[i].Name, source[i].Name; got != exp { - t.Errorf("compiled package at index %d is has name %q but is expected to have %q", i, got, exp) - } - if got, exp := compiled[i].Version, source[i].Version; got != exp { - t.Errorf("compiled package at index %d is has version %q but is expected to have %q", i, got, exp) - } - if got, exp := compiled[i].OSVersionSlug, stemcellSlug; got != exp { - t.Errorf("compiled package at index %d is has stemcell %q but is expected to have %q", i, got, exp) - } - if got := compiled[i].SHA1; got == "" { - t.Errorf("compiled package at index %d should have sha1 set %q", i, got) - } - if got, exp := compiled[i].Fingerprint, compiled[i].SHA1; got != exp { - // I believe this is incorrect. I am "just" setting the fingerprint to the sha1 sum. This is wrong. See: https://bosh.io/docs/managing-releases/#jobs-and-packages - t.Errorf("compiled package at index %d should have fingerprint %q to be identical to the sha1 sum %q", i, got, exp) - } - if got := compiled[i].Dependencies; len(got) != 0 { - t.Errorf("compiled package at index %d should have no dependencies got %#v", i, got) - } - } -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 02dbf1432..e4d49d4a9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -69,6 +69,9 @@ github.com/bodgit/windows # github.com/charlievieth/fs v0.0.3 ## explicit; go 1.18 github.com/charlievieth/fs +# github.com/cloudfoundry/bosh-cli v6.4.1+incompatible +## explicit +github.com/cloudfoundry/bosh-cli/release/manifest # github.com/cloudfoundry/bosh-davcli v0.0.297 ## explicit; go 1.21 github.com/cloudfoundry/bosh-davcli/client @@ -489,6 +492,9 @@ google.golang.org/protobuf/types/descriptorpb # gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ## explicit gopkg.in/tomb.v1 +# gopkg.in/yaml.v2 v2.4.0 +## explicit; go 1.15 +gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3