diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 25d8e264..7dbaf2ec 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,7 +25,7 @@ jobs: run: sudo apt update # This is needed for the container resolver dependencies - - name: Install libgpgme devel package + - name: Install test dependencies run: sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev podman - name: Build diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 00000000..54380e75 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,42 @@ +# Hacking on image-builder-cli + +Hacking on `image-builder` should be fun and is easy. + +We have unit tests and some integration testing. + +## Setup + +To work on bootc-image-builder one needs a working Go environment. See +[go.mod](go.mod). + +To run the testsuite install the test dependencies as outlined in the +[github action](./.github/workflows/go.yml) under +"Install test dependencies". + +## Code layout + +The go source code of image-builder-cli is under +`./cmd/image-builder`. It uses the +[images](https://github.com/osbuild/images) library internally to +generate the images. Unit tests (and integration tests where it makes +sense) are expected to be part of every PR but we are happy to help if +those are missing from a PR. + +## Build + +Build by running: +```console +$ go build ./cmd/image-builder/ +``` + +## Unit tests + +Run the unit tests via: +```console +$ go test -short ./... +``` + +There are some integration tests that can be run via: +```console +$ go test ./... +``` diff --git a/cmd/image-builder/distro.go b/cmd/image-builder/distro.go new file mode 100644 index 00000000..deaae7a8 --- /dev/null +++ b/cmd/image-builder/distro.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + + "github.com/osbuild/images/pkg/distro" +) + +var distroGetHostDistroName = distro.GetHostDistroName + +// findDistro will ensure that the given distro argument do not +// diverge. If no distro is set via the blueprint or the argument +// the host is used to derive the distro. +func findDistro(argDistroName, bpDistroName string) (string, error) { + switch { + case argDistroName != "" && bpDistroName != "" && argDistroName != bpDistroName: + return "", fmt.Errorf("error selecting distro name, cmdline argument %q is different from blueprint %q", argDistroName, bpDistroName) + case argDistroName != "": + return argDistroName, nil + case bpDistroName != "": + return bpDistroName, nil + } + // nothing selected by the user, derive from host + distroStr, err := distroGetHostDistroName() + if err != nil { + return "", fmt.Errorf("error deriving host distro %w", err) + } + fmt.Fprintf(osStderr, "No distro name specified, selecting %q based on host, use --distro to override", distroStr) + return distroStr, nil +} diff --git a/cmd/image-builder/distro_test.go b/cmd/image-builder/distro_test.go new file mode 100644 index 00000000..7c187f8b --- /dev/null +++ b/cmd/image-builder/distro_test.go @@ -0,0 +1,48 @@ +package main_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/image-builder-cli/cmd/image-builder" +) + +func TestFindDistro(t *testing.T) { + for _, tc := range []struct { + argDistro string + bpDistro string + expectedDistro string + expectedErr string + }{ + {"arg", "", "arg", ""}, + {"", "bp", "bp", ""}, + {"arg", "bp", "", `error selecting distro name, cmdline argument "arg" is different from blueprint "bp"`}, + // the argDistro,bpDistro == "" case is tested below + } { + distro, err := main.FindDistro(tc.argDistro, tc.bpDistro) + if tc.expectedErr != "" { + assert.Equal(t, tc.expectedErr, err.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expectedDistro, distro) + } + } +} + +func TestFindDistroAutoDetect(t *testing.T) { + var buf bytes.Buffer + restore := main.MockOsStderr(&buf) + defer restore() + + restore = main.MockDistroGetHostDistroName(func() (string, error) { + return "mocked-host-distro", nil + }) + defer restore() + + distro, err := main.FindDistro("", "") + assert.NoError(t, err) + assert.Equal(t, "mocked-host-distro", distro) + assert.Equal(t, `No distro name specified, selecting "mocked-host-distro" based on host, use --distro to override`, buf.String()) +} diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 0cdb7b34..5b3005f7 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -8,6 +8,12 @@ import ( "github.com/osbuild/images/pkg/reporegistry" ) +var ( + GetOneImage = getOneImage + Run = run + FindDistro = findDistro +) + func MockOsArgs(new []string) (restore func()) { saved := os.Args os.Args = append([]string{"argv0"}, new...) @@ -45,6 +51,10 @@ func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore } } -var ( - Run = run -) +func MockDistroGetHostDistroName(f func() (string, error)) (restore func()) { + saved := distroGetHostDistroName + distroGetHostDistroName = f + return func() { + distroGetHostDistroName = saved + } +} diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 480be1ae..24388e2b 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -1,6 +1,11 @@ package main import ( + "fmt" + "strings" + + "github.com/gobwas/glob" + "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/imagefilter" ) @@ -11,5 +16,48 @@ func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { if err != nil { return nil, err } + return imagefilter.New(fac, repos) } + +// should this be moved to images:imagefilter? +func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) { + imageFilter, err := newImageFilterDefault(dataDir) + if err != nil { + return nil, err + } + // strip prefixes to make ib copy/paste friendly when pasting output + // from "list-images" + distroName = strings.TrimPrefix(distroName, "distro:") + imgTypeStr = strings.TrimPrefix(imgTypeStr, "type:") + archStr = strings.TrimPrefix(archStr, "arch:") + + // error early when globs are used + for _, s := range []string{distroName, imgTypeStr, archStr} { + if glob.QuoteMeta(s) != s { + return nil, fmt.Errorf("cannot use globs in %q when getting a single image", s) + } + } + + filterExprs := []string{ + fmt.Sprintf("distro:%s", distroName), + fmt.Sprintf("arch:%s", archStr), + fmt.Sprintf("type:%s", imgTypeStr), + } + filteredResults, err := imageFilter.Filter(filterExprs...) + if err != nil { + return nil, err + } + switch len(filteredResults) { + case 0: + return nil, fmt.Errorf("cannot find image for: distro:%q type:%q arch:%q", distroName, imgTypeStr, archStr) + case 1: + return &filteredResults[0], nil + default: + // This condition should never be hit in practise as we + // disallow globs above. + // XXX: imagefilter.Result should have a String() method so + // that this output can actually show the results + return nil, fmt.Errorf("internal error: found %v results for %q %q %q", len(filteredResults), distroName, imgTypeStr, archStr) + } +} diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go new file mode 100644 index 00000000..2bf984a7 --- /dev/null +++ b/cmd/image-builder/filters_test.go @@ -0,0 +1,55 @@ +package main_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + testrepos "github.com/osbuild/images/test/data/repositories" + + "github.com/osbuild/image-builder-cli/cmd/image-builder" +) + +func TestGetOneImageHappy(t *testing.T) { + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + dataDir := "" + for _, tc := range []struct { + distro, imgType, arch string + }{ + {"centos-9", "qcow2", "x86_64"}, + {"distro:centos-9", "qcow2", "x86_64"}, + {"distro:centos-9", "type:qcow2", "x86_64"}, + {"distro:centos-9", "type:qcow2", "arch:x86_64"}, + } { + res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + assert.NoError(t, err) + assert.Equal(t, "centos-9", res.Distro.Name()) + assert.Equal(t, "qcow2", res.ImgType.Name()) + assert.Equal(t, "x86_64", res.Arch.Name()) + } +} + +func TestGetOneImageError(t *testing.T) { + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + dataDir := "" + for _, tc := range []struct { + distro, imgType, arch string + expectedErr string + }{ + { + "unknown", "qcow2", "x86_64", + `cannot find image for: distro:"unknown" type:"qcow2" arch:"x86_64"`, + }, + { + "centos*", "qcow2", "x86_64", + `cannot use globs in "centos*" when getting a single image`, + }, + } { + _, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + assert.EqualError(t, err, tc.expectedErr) + } +} diff --git a/cmd/image-builder/list.go b/cmd/image-builder/list.go index 81e357ee..9bf06d94 100644 --- a/cmd/image-builder/list.go +++ b/cmd/image-builder/list.go @@ -4,8 +4,8 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func listImages(output string, filterExprs []string, opts *cmdlineOpts) error { - imageFilter, err := newImageFilterDefault(opts.dataDir) +func listImages(dataDir, output string, filterExprs []string) error { + imageFilter, err := newImageFilterDefault(dataDir) if err != nil { return err } @@ -19,7 +19,7 @@ func listImages(output string, filterExprs []string, opts *cmdlineOpts) error { if err != nil { return err } - if err := fmter.Output(opts.out, filteredResult); err != nil { + if err := fmter.Output(osStdout, filteredResult); err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 81575c7f..39f24cb0 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -7,6 +7,11 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "github.com/osbuild/images/pkg/arch" + + "github.com/osbuild/image-builder-cli/internal/blueprintload" + "github.com/osbuild/image-builder-cli/internal/manifestgen" ) var ( @@ -14,11 +19,6 @@ var ( osStderr io.Writer = os.Stderr ) -type cmdlineOpts struct { - dataDir string - out io.Writer -} - func cmdListImages(cmd *cobra.Command, args []string) error { filter, err := cmd.Flags().GetStringArray("filter") if err != nil { @@ -33,11 +33,57 @@ func cmdListImages(cmd *cobra.Command, args []string) error { return err } - opts := &cmdlineOpts{ - out: osStdout, - dataDir: dataDir, + return listImages(dataDir, output, filter) +} + +func cmdManifest(cmd *cobra.Command, args []string) error { + dataDir, err := cmd.Flags().GetString("datadir") + if err != nil { + return err + } + archStr, err := cmd.Flags().GetString("arch") + if err != nil { + return err + } + if archStr == "" { + archStr = arch.Current().String() + } + distroStr, err := cmd.Flags().GetString("distro") + if err != nil { + return err + } + + var blueprintPath string + imgTypeStr := args[0] + if len(args) > 1 { + blueprintPath = args[1] + } + bp, err := blueprintload.Load(blueprintPath) + if err != nil { + return err + } + distroStr, err = findDistro(distroStr, bp.Distro) + if err != nil { + return err + } + + res, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr) + if err != nil { + return err } - return listImages(output, filter, opts) + repos, err := newRepoRegistry(dataDir) + if err != nil { + return err + } + // XXX: add --rpmmd/cachedir option like bib + mg, err := manifestgen.New(repos, &manifestgen.Options{ + Output: osStdout, + }) + if err != nil { + return err + } + + return mg.Generate(bp, res.Distro, res.ImgType, res.Arch, nil) } func run() error { @@ -70,6 +116,18 @@ operating sytsems like centos and RHEL with easy customizations support.`, listImagesCmd.Flags().String("output", "", "Output in a specific format (text, json)") rootCmd.AddCommand(listImagesCmd) + manifestCmd := &cobra.Command{ + Use: "manifest [blueprint]", + Short: "Build manifest for the given distro/image-type, e.g. centos-9 qcow2", + RunE: cmdManifest, + SilenceUsage: true, + Args: cobra.RangeArgs(1, 2), + Hidden: true, + } + manifestCmd.Flags().String("arch", "", `build manifest for a different architecture`) + manifestCmd.Flags().String("distro", "", `build manifest for a different distroname (e.g. centos-9)`) + rootCmd.AddCommand(manifestCmd) + return rootCmd.Execute() } diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index fbaad7b1..a265ef42 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -3,6 +3,8 @@ package main_test import ( "bytes" "encoding/json" + "os" + "path/filepath" "testing" "github.com/sirupsen/logrus" @@ -11,6 +13,7 @@ import ( testrepos "github.com/osbuild/images/test/data/repositories" "github.com/osbuild/image-builder-cli/cmd/image-builder" + "github.com/osbuild/image-builder-cli/internal/manifesttest" ) func init() { @@ -123,3 +126,102 @@ func TestListImagesErrorsOnExtraArgs(t *testing.T) { err := main.Run() assert.EqualError(t, err, `unknown command "extra-arg" for "image-builder list-images"`) } + +func hasDepsolveDnf() bool { + // XXX: expose images/pkg/depsolve:findDepsolveDnf() + _, err := os.Stat("/usr/libexec/osbuild-depsolve-dnf") + return err == nil +} + +var testBlueprint = `{ + "containers": [ + { + "source": "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal" + } + ], + "customizations": { + "user": [ + { + "name": "alice" + } + ] + } +}` + +func makeTestBlueprint(t *testing.T, testBlueprint string) string { + tmpdir := t.TempDir() + blueprintPath := filepath.Join(tmpdir, "blueprint.json") + err := os.WriteFile(blueprintPath, []byte(testBlueprint), 0644) + assert.NoError(t, err) + return blueprintPath +} + +// XXX: move to pytest like bib maybe? +func TestManifestIntegrationSmoke(t *testing.T) { + if testing.Short() { + t.Skip("manifest generation takes a while") + } + if !hasDepsolveDnf() { + t.Skip("no osbuild-depsolve-dnf binary found") + } + + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + restore = main.MockOsArgs([]string{ + "manifest", + "qcow2", + "--distro=centos-9", + makeTestBlueprint(t, testBlueprint), + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err := main.Run() + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes()) + assert.NoError(t, err) + assert.Contains(t, pipelineNames, "qcow2") + + // XXX: provide helpers in manifesttest to extract this in a nicer way + assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`) + assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`) +} + +func TestManifestIntegrationCrossArch(t *testing.T) { + if testing.Short() { + t.Skip("manifest generation takes a while") + } + if !hasDepsolveDnf() { + t.Skip("no osbuild-depsolve-dnf binary found") + } + + restore := main.MockNewRepoRegistry(testrepos.New) + defer restore() + + restore = main.MockOsArgs([]string{ + "manifest", + "tar", + "--distro", "centos-9", + "--arch", "s390x", + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err := main.Run() + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes()) + assert.NoError(t, err) + assert.Contains(t, pipelineNames, "archive") + + // XXX: provide helpers in manifesttest to extract this in a nicer way + assert.Contains(t, fakeStdout.String(), `.el9.s390x.rpm`) +} diff --git a/go.mod b/go.mod index 044652c7..ad5e2b85 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ module github.com/osbuild/image-builder-cli go 1.21.0 require ( + github.com/gobwas/glob v0.2.3 github.com/osbuild/images v0.98.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 @@ -50,7 +51,6 @@ require ( github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-openapi/validate v0.24.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/internal/blueprintload/blueprintload.go b/internal/blueprintload/blueprintload.go new file mode 100644 index 00000000..e06135cb --- /dev/null +++ b/internal/blueprintload/blueprintload.go @@ -0,0 +1,67 @@ +package blueprintload + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/BurntSushi/toml" + + "github.com/osbuild/images/pkg/blueprint" +) + +// XXX: move this helper into images, share with bib +func decodeToml(r io.Reader, what string) (*blueprint.Blueprint, error) { + dec := toml.NewDecoder(r) + + var conf blueprint.Blueprint + _, err := dec.Decode(&conf) + if err != nil { + return nil, fmt.Errorf("cannot decode %q: %w", what, err) + } + + return &conf, nil +} + +func decodeJson(r io.Reader, what string) (*blueprint.Blueprint, error) { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + + var conf blueprint.Blueprint + if err := dec.Decode(&conf); err != nil { + return nil, fmt.Errorf("cannot decode %q: %w", what, err) + } + if dec.More() { + return nil, fmt.Errorf("multiple configuration objects or extra data found in %q", what) + } + return &conf, nil +} + +func Load(path string) (*blueprint.Blueprint, error) { + var fp io.ReadCloser + var err error + + switch path { + case "": + return &blueprint.Blueprint{}, nil + case "-": + fp = os.Stdin + default: + fp, err = os.Open(path) + if err != nil { + return nil, err + } + defer fp.Close() + } + + switch { + case path == "-", filepath.Ext(path) == ".json": + return decodeJson(fp, path) + case filepath.Ext(path) == ".toml": + return decodeToml(fp, path) + default: + return nil, fmt.Errorf("unsupported file extension for %q (please use .toml or .json)", path) + } +} diff --git a/internal/blueprintload/blueprintload_test.go b/internal/blueprintload/blueprintload_test.go new file mode 100644 index 00000000..de7ad844 --- /dev/null +++ b/internal/blueprintload/blueprintload_test.go @@ -0,0 +1,72 @@ +package blueprintload_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/blueprint" + + "github.com/osbuild/image-builder-cli/internal/blueprintload" +) + +var testBlueprintJSON = `{ + "customizations": { + "user": [ + { + "name": "alice" + } + ] + } +}` + +var testBlueprintTOML = ` +[[customizations.user]] +name = "alice" +` + +var expectedBlueprint = &blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + User: []blueprint.UserCustomization{ + { + Name: "alice", + }, + }, + }, +} + +func makeTestBlueprint(t *testing.T, name, content string) string { + tmpdir := t.TempDir() + blueprintPath := filepath.Join(tmpdir, name) + err := os.WriteFile(blueprintPath, []byte(content), 0644) + assert.NoError(t, err) + return blueprintPath +} + +func TestBlueprintLoadJSON(t *testing.T) { + for _, tc := range []struct { + fname string + content string + + expectedBp *blueprint.Blueprint + expectedError string + }{ + {"bp.json", testBlueprintJSON, expectedBlueprint, ""}, + {"bp.toml", testBlueprintTOML, expectedBlueprint, ""}, + {"bp.toml", "wrong-content", nil, `cannot decode .*/bp.toml": toml: `}, + {"bp.json", "wrong-content", nil, `cannot decode .*/bp.json": invalid `}, + {"bp", "wrong-content", nil, `unsupported file extension for "/.*/bp"`}, + } { + blueprintPath := makeTestBlueprint(t, tc.fname, tc.content) + bp, err := blueprintload.Load(blueprintPath) + if tc.expectedError == "" { + assert.NoError(t, err) + assert.Equal(t, tc.expectedBp, bp) + } else { + assert.NotNil(t, err) + assert.Regexp(t, tc.expectedError, err.Error()) + } + } +} diff --git a/internal/manifestgen/manifestgen.go b/internal/manifestgen/manifestgen.go new file mode 100644 index 00000000..1c912eb8 --- /dev/null +++ b/internal/manifestgen/manifestgen.go @@ -0,0 +1,184 @@ +package manifestgen + +import ( + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" +) + +// XXX: all of the helpers below are duplicated from +// cmd/build/main.go:depsolve (and probably more places) should go +// into a common helper in "images" or images should do this on its +// own +func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range packageSets { + res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) + if err != nil { + return nil, nil, fmt.Errorf("error depsolving: %w", err) + } + depsolvedSets[name] = res.Packages + repoSets[name] = res.Repos + // the depsolve result also contains SBOM information, + // it is currently not used here though + } + return depsolvedSets, repoSets, nil +} + +func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { + resolver := container.NewResolver(archName) + + for _, c := range containers { + resolver.Add(c) + } + + return resolver.Finish() +} + +func defaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + specs, err := resolveContainers(sourceSpecs, archName) + if err != nil { + return nil, fmt.Errorf("error container resolving: %w", err) + } + containerSpecs[plName] = specs + } + return containerSpecs, nil +} + +func defaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + var err error + commitSpecs[idx], err = ostree.Resolve(commitSource) + if err != nil { + return nil, fmt.Errorf("error ostree commit resolving: %w", err) + } + } + commits[name] = commitSpecs + } + return commits, nil +} + +type ( + DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) + + ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) + + CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) +) + +// Options contains the optional settings for the manifest generation. +// For unset values defaults will be used. +type Options struct { + Cachedir string + Output io.Writer + Depsolver DepsolveFunc + ContainerResolver ContainerResolverFunc + CommitResolver CommitResolverFunc +} + +// Generator can generate an osbuild manifest from a given repository +// and options. +type Generator struct { + cacheDir string + out io.Writer + + depsolver DepsolveFunc + containerResolver ContainerResolverFunc + commitResolver CommitResolverFunc + + reporegistry *reporegistry.RepoRegistry +} + +// New will create a new manifest generator +func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) { + if opts == nil { + opts = &Options{} + } + mg := &Generator{ + reporegistry: reporegistry, + + cacheDir: opts.Cachedir, + out: opts.Output, + depsolver: opts.Depsolver, + containerResolver: opts.ContainerResolver, + commitResolver: opts.CommitResolver, + } + if mg.out == nil { + mg.out = os.Stdout + } + if mg.depsolver == nil { + mg.depsolver = defaultDepsolver + } + if mg.containerResolver == nil { + mg.containerResolver = defaultContainerResolver + } + if mg.commitResolver == nil { + mg.commitResolver = defaultCommitResolver + } + + return mg, nil +} + +// Generate will generate a new manifest for the given distro/imageType/arch +// combination. +func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error { + if imgOpts == nil { + imgOpts = &distro.ImageOptions{} + } + // we may allow to customize the seed in the future via imgOpts or + // an environment variable + // XXX: look into "images" so that it automatically seeds when pasing + // a "0" seed. + seed := time.Now().UnixNano() + + repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name()) + if err != nil { + return err + } + preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, seed) + if err != nil { + return err + } + if len(warnings) > 0 { + // XXX: what can we do here? for things like json output? + // what are these warnings? + return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) + } + packageSpecs, _, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name()) + if err != nil { + return err + } + containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name()) + if err != nil { + return err + } + commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs()) + if err != nil { + return err + } + mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil) + if err != nil { + return err + } + fmt.Fprintf(mg.out, "%s\n", mf) + + return nil +} diff --git a/internal/manifestgen/manifestgen_test.go b/internal/manifestgen/manifestgen_test.go new file mode 100644 index 00000000..91128cb0 --- /dev/null +++ b/internal/manifestgen/manifestgen_test.go @@ -0,0 +1,217 @@ +package manifestgen_test + +import ( + "bytes" + "crypto/sha256" + "fmt" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distrofactory" + "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/rpmmd" + testrepos "github.com/osbuild/images/test/data/repositories" + + "github.com/osbuild/image-builder-cli/internal/manifestgen" + "github.com/osbuild/image-builder-cli/internal/manifesttest" +) + +func init() { + // silence logrus by default, it is quite verbose + logrus.SetLevel(logrus.WarnLevel) +} + +func sha256For(s string) string { + h := sha256.New() + h.Write([]byte(s)) + bs := h.Sum(nil) + return fmt.Sprintf("sha256:%x", bs) +} + +func TestManifestGeneratorDepsolve(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: panicContainerResolver, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) + + // we expect at least a "kernel" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("kernel") + assert.Contains(t, osbuildManifest.String(), expectedSha256) +} + +func TestManifestGeneratorWithOstreeCommit(t *testing.T) { + var osbuildManifest bytes.Buffer + + repos, err := testrepos.New() + assert.NoError(t, err) + + fac := distrofactory.NewDefault() + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: fakeCommitResolver, + ContainerResolver: panicContainerResolver, + } + imageOpts := &distro.ImageOptions{ + OSTree: &ostree.ImageOptions{ + //ImageRef: "latest/1/x86_64/edge", + URL: "http://example.com/", + }, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts) + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames) + + // XXX: add testhelper to manifesttest for this + assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`) + // we expect at least a "glibc" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("glibc") + assert.Contains(t, osbuildManifest.String(), expectedSha256) +} + +func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSets := range packageSets { + var resolvedSet []rpmmd.PackageSpec + for _, pkgSet := range pkgSets { + for _, pkgName := range pkgSet.Include { + resolvedSet = append(resolvedSet, rpmmd.PackageSpec{ + Name: pkgName, + Checksum: sha256For(pkgName), + }) + } + } + depsolvedSets[name] = resolvedSet + } + return depsolvedSets, repoSets, nil +} + +func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + commitSpecs[idx] = ostree.CommitSpec{ + URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref), + } + } + commits[name] = commitSpecs + } + return commits, nil + +} + +func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + if len(commitSources) > 0 { + panic("panicCommitResolver") + } + return nil, nil +} + +func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + var containers []container.Spec + for _, spec := range sourceSpecs { + containers = append(containers, container.Spec{ + Source: fmt.Sprintf("resolved-cnt-%s", spec.Source), + Digest: "sha256:" + sha256For("digest:"+spec.Source), + ImageID: "sha256:" + sha256For("id:"+spec.Source), + }) + } + containerSpecs[plName] = containers + } + return containerSpecs, nil +} + +func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + if len(containerSources) > 0 { + panic("panicContainerResolver") + } + return nil, nil +} + +func TestManifestGeneratorContainers(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: fakeContainerResolver, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal" + bp := blueprint.Blueprint{ + Containers: []blueprint.Container{ + { + Source: fakeContainerSource, + }, + }, + } + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + // container is included + assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource) +} diff --git a/internal/manifesttest/manifesttest.go b/internal/manifesttest/manifesttest.go new file mode 100644 index 00000000..47b3e3ae --- /dev/null +++ b/internal/manifesttest/manifesttest.go @@ -0,0 +1,27 @@ +package manifesttest + +import ( + "encoding/json" + "fmt" +) + +// PipelineNamesFrom will return all pipeline names from an osbuild +// json manifest. It will error on missing pipelines. +// +// TODO: move to images:pkg/manifesttest +func PipelineNamesFrom(osbuildManifest []byte) ([]string, error) { + var manifest map[string]interface{} + + if err := json.Unmarshal(osbuildManifest, &manifest); err != nil { + return nil, fmt.Errorf("cannot unmarshal manifest: %w", err) + } + if manifest["pipelines"] == nil { + return nil, fmt.Errorf("cannot find any pipelines in %v", manifest) + } + pipelines := manifest["pipelines"].([]interface{}) + pipelineNames := make([]string, len(pipelines)) + for idx, pi := range pipelines { + pipelineNames[idx] = pi.(map[string]interface{})["name"].(string) + } + return pipelineNames, nil +} diff --git a/internal/manifesttest/manifesttest_test.go b/internal/manifesttest/manifesttest_test.go new file mode 100644 index 00000000..ac731308 --- /dev/null +++ b/internal/manifesttest/manifesttest_test.go @@ -0,0 +1,35 @@ +package manifesttest_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/image-builder-cli/internal/manifesttest" +) + +var fakeOsbuildManifest = `{ + "version": "2", + "pipelines": [ + { + "name": "noop" + }, + { + "name": "noop2" + } + ] +}` + +func TestPipelineNamesFrom(t *testing.T) { + names, err := manifesttest.PipelineNamesFrom([]byte(fakeOsbuildManifest)) + assert.NoError(t, err) + assert.Equal(t, []string{"noop", "noop2"}, names) +} + +func TestPipelineNamesFromSad(t *testing.T) { + _, err := manifesttest.PipelineNamesFrom([]byte("bad-json")) + assert.ErrorContains(t, err, "cannot unmarshal manifest: invalid char") + + _, err = manifesttest.PipelineNamesFrom([]byte("{}")) + assert.ErrorContains(t, err, "cannot find any pipelines in map[]") +}