manifest,sources: add librepo support to Serialize/GenSources

This commit enables librepo sources generation via a new
osbuild.RpmDownloader iota that is passed to to
`manifest.Serialize()` and `osbuild.GenSources()`.

We also need to pass the resolved repoConfig to `manifest.Serialize()`.
This is currently not type-safe, ideally we would look into how to do
this in a type-safe way. Serialize should also take less args
ideally.

Author: mvo5

cmd/build/main.go

@@ -82,7 +82,7 @@ func makeManifest(
 		return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %w", err)
 	}
 
-	mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil)
+	mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0)
 	if err != nil {
 		return nil, fmt.Errorf("[ERROR] manifest serialization failed: %w", err)
 	}

cmd/gen-manifests/main.go

@@ -251,7 +251,7 @@ func makeManifestJob(
 			commitSpecs = mockResolveCommits(manifest.GetOSTreeSourceSpecs())
 		}
 
-		mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs)
+		mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0)
 		if err != nil {
 			return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error())
 		}

cmd/osbuild-playground/playground.go

@@ -49,7 +49,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos
 		fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error())
 	}
 
-	bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil)
+	bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil, 0)
 	if err != nil {
 		panic("failed to serialize manifest: " + err.Error())
 	}

pkg/distro/distro_test.go

@@ -134,7 +134,7 @@ func TestImageTypePipelineNames(t *testing.T) {
 						}
 						commits[name] = commitSpecs
 					}
-					mf, err := m.Serialize(packageSets, containers, commits, repoSets)
+					mf, err := m.Serialize(packageSets, containers, commits, repoSets, 0)
 					assert.NoError(err)
 					pm := new(manifest)
 					err = json.Unmarshal(mf, pm)

pkg/image/bootc_disk_test.go

@@ -88,7 +88,7 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt
 		"image": []container.Spec{{Source: "other-src", Digest: makeFakeDigest(t), ImageID: makeFakeDigest(t)}},
 	}
 
-	osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil)
+	osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil, 0)
 	require.Nil(t, err)
 
 	return osbuildManifest

pkg/image/installer_image_test.go

@@ -364,7 +364,7 @@ func instantiateAndSerialize(t *testing.T, img image.ImageKind, packages map[str
 	_, err := img.InstantiateManifest(&mf, nil, &runner.CentOS{Version: 9}, rng)
 	assert.NoError(t, err)
 
-	mfs, err := mf.Serialize(packages, containers, commits, nil)
+	mfs, err := mf.Serialize(packages, containers, commits, nil, 0)
 	assert.NoError(t, err)
 
 	return string(mfs)

pkg/manifest/manifest.go

@@ -41,12 +41,7 @@ const (
 	DISTRO_FEDORA
 )
 
-type Inputs struct {
-	Packages   []rpmmd.PackageSpec
-	Containers []container.Spec
-	Commits    []ostree.CommitSpec
-	RpmRepos   []rpmmd.RepoConfig
-}
+type Inputs osbuild.SourceInputs
 
 // An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild
 type OSBuildManifest []byte
@@ -149,32 +144,32 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec {
 // only depsolved PackageSpecs/RepoConfigs are passed so that we
 // have a valid mapping of pkg.RepoID<->repo.Id which will be important
 // for librepo
-func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) {
-	pipelines := make([]osbuild.Pipeline, 0)
-	packages := make([]rpmmd.PackageSpec, 0)
-	commits := make([]ostree.CommitSpec, 0)
-	inline := make([]string, 0)
-	containers := make([]container.Spec, 0)
+func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, resolvedRpmRepos map[string][]rpmmd.RepoConfig, rpmDownloader osbuild.RpmDownloader) (OSBuildManifest, error) {
 	for _, pipeline := range m.pipelines {
 		pipeline.serializeStart(Inputs{
 			Packages:   packageSets[pipeline.Name()],
 			Containers: containerSpecs[pipeline.Name()],
 			Commits:    ostreeCommits[pipeline.Name()],
-			RpmRepos:   rpmRepos[pipeline.Name()],
+			RpmRepos:   resolvedRpmRepos[pipeline.Name()],
 		})
 	}
+
+	var pipelines []osbuild.Pipeline
+	var mergedInputs osbuild.SourceInputs
 	for _, pipeline := range m.pipelines {
-		commits = append(commits, pipeline.getOSTreeCommits()...)
 		pipelines = append(pipelines, pipeline.serialize())
-		packages = append(packages, packageSets[pipeline.Name()]...)
-		inline = append(inline, pipeline.getInline()...)
-		containers = append(containers, pipeline.getContainerSpecs()...)
+
+		mergedInputs.Commits = append(mergedInputs.Commits, pipeline.getOSTreeCommits()...)
+		mergedInputs.Packages = append(mergedInputs.Packages, packageSets[pipeline.Name()]...)
+		mergedInputs.RpmRepos = append(mergedInputs.RpmRepos, resolvedRpmRepos[pipeline.Name()]...)
+		mergedInputs.Containers = append(mergedInputs.Containers, pipeline.getContainerSpecs()...)
+		mergedInputs.InlineData = append(mergedInputs.InlineData, pipeline.getInline()...)
 	}
 	for _, pipeline := range m.pipelines {
 		pipeline.serializeEnd()
 	}
 
-	sources, err := osbuild.GenSources(packages, commits, inline, containers)
+	sources, err := osbuild.GenSources(mergedInputs, rpmDownloader)
 	if err != nil {
 		return nil, err
 	}

pkg/osbuild/source.go

@@ -3,12 +3,33 @@ package osbuild
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/ostree"
 	"github.com/osbuild/images/pkg/rpmmd"
 )
 
+// RpmDownloader specifies what backend to use for rpm downloads
+// Note that the librepo backend requires a newer osbuild.
+type RpmDownloader uint64
+
+const (
+	RpmDownloaderCurl    = iota
+	RpmDownloaderLibrepo = iota
+)
+
+// SourceInputs contains the inputs to generate osbuild.Sources
+// Note that for Packages/RpmRepos the depsolve resolved results
+// must be passed
+type SourceInputs struct {
+	Packages   []rpmmd.PackageSpec
+	Containers []container.Spec
+	Commits    []ostree.CommitSpec
+	RpmRepos   []rpmmd.RepoConfig
+	InlineData []string
+}
+
 // A Sources map contains all the sources made available to an osbuild run
 type Sources map[string]Source
 
@@ -54,25 +75,55 @@ func (sources *Sources) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec) (Sources, error) {
+func addPackagesCurl(sources Sources, packages []rpmmd.PackageSpec) error {
+	curl := NewCurlSource()
+	for _, pkg := range packages {
+		err := curl.AddPackage(pkg)
+		if err != nil {
+			return err
+		}
+	}
+	sources["org.osbuild.curl"] = curl
+	return nil
+}
+
+func addPackagesLibrepo(sources Sources, packages []rpmmd.PackageSpec, rpmRepos []rpmmd.RepoConfig) error {
+	librepo := NewLibrepoSource()
+	for _, pkg := range packages {
+		err := librepo.AddPackage(pkg, rpmRepos)
+		if err != nil {
+			return err
+		}
+	}
+	sources["org.osbuild.librepo"] = librepo
+	return nil
+}
+
+// GenSources generates the Sources from the given inputs. Note that
+// the packages and rpmRepos need to come from the *resolved* set.
+func GenSources(inputs SourceInputs, rpmDownloader RpmDownloader) (Sources, error) {
 	sources := Sources{}
 
 	// collect rpm package sources
-	if len(packages) > 0 {
-		curl := NewCurlSource()
-		for _, pkg := range packages {
-			err := curl.AddPackage(pkg)
-			if err != nil {
-				return nil, err
-			}
+	if len(inputs.Packages) > 0 {
+		var err error
+		switch rpmDownloader {
+		case RpmDownloaderCurl:
+			err = addPackagesCurl(sources, inputs.Packages)
+		case RpmDownloaderLibrepo:
+			err = addPackagesLibrepo(sources, inputs.Packages, inputs.RpmRepos)
+		default:
+			err = fmt.Errorf("unknown rpm downloader %v", rpmDownloader)
+		}
+		if err != nil {
+			return nil, err
 		}
-		sources["org.osbuild.curl"] = curl
 	}
 
 	// collect ostree commit sources
-	if len(ostreeCommits) > 0 {
+	if len(inputs.Commits) > 0 {
 		ostree := NewOSTreeSource()
-		for _, commit := range ostreeCommits {
+		for _, commit := range inputs.Commits {
 			ostree.AddItem(commit)
 		}
 		if len(ostree.Items) > 0 {
@@ -81,21 +132,21 @@ func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec,
 	}
 
 	// collect inline data sources
-	if len(inlineData) > 0 {
+	if len(inputs.InlineData) > 0 {
 		ils := NewInlineSource()
-		for _, data := range inlineData {
+		for _, data := range inputs.InlineData {
 			ils.AddItem(data)
 		}
 
 		sources["org.osbuild.inline"] = ils
 	}
 
 	// collect skopeo and local container sources
-	if len(containers) > 0 {
+	if len(inputs.Containers) > 0 {
 		skopeo := NewSkopeoSource()
 		skopeoIndex := NewSkopeoIndexSource()
 		localContainers := NewContainersStorageSource()
-		for _, c := range containers {
+		for _, c := range inputs.Containers {
 			if c.LocalStorage {
 				localContainers.AddItem(c.ImageID)
 			} else {

pkg/osbuild/source_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/osbuild/images/pkg/container"
+	"github.com/osbuild/images/pkg/rpmmd"
 )
 
 func TestSource_UnmarshalJSON(t *testing.T) {
@@ -119,7 +120,7 @@ func TestSource_UnmarshalJSON(t *testing.T) {
 }
 
 func TestGenSourcesTrivial(t *testing.T) {
-	sources, err := GenSources(nil, nil, nil, nil)
+	sources, err := GenSources(SourceInputs{}, 0)
 	assert.NoError(t, err)
 
 	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
@@ -135,7 +136,7 @@ func TestGenSourcesContainerStorage(t *testing.T) {
 			LocalStorage: true,
 		},
 	}
-	sources, err := GenSources(nil, nil, nil, containers)
+	sources, err := GenSources(SourceInputs{Containers: containers}, 0)
 	assert.NoError(t, err)
 
 	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
@@ -159,7 +160,7 @@ func TestGenSourcesSkopeo(t *testing.T) {
 			ImageID: imageID,
 		},
 	}
-	sources, err := GenSources(nil, nil, nil, containers)
+	sources, err := GenSources(SourceInputs{Containers: containers}, 0)
 	assert.NoError(t, err)
 
 	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
@@ -190,7 +191,7 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) {
 			ImageID:    imageID,
 		},
 	}
-	sources, err := GenSources(nil, nil, nil, containers)
+	sources, err := GenSources(SourceInputs{Containers: containers}, 0)
 	assert.NoError(t, err)
 
 	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
@@ -217,3 +218,92 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) {
   }
 }`)
 }
+
+// TODO: move into a common "rpmtest" package
+var opensslPkg = rpmmd.PackageSpec{
+	Name:           "openssl-libs",
+	RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	Checksum:       "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666",
+	Path:           "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	RepoID:         "repo_id_metalink",
+}
+
+var fakeRepos = []rpmmd.RepoConfig{
+	{
+		Id:       "repo_id_metalink",
+		Metalink: "http://example.com/metalink",
+	},
+}
+
+func TestGenSourcesRpmDefaultRpmDownloaderIsCurl(t *testing.T) {
+	inputs := SourceInputs{
+		Packages: []rpmmd.PackageSpec{opensslPkg},
+		RpmRepos: fakeRepos,
+	}
+	var defaultRpmDownloader RpmDownloader
+	sources, err := GenSources(inputs, defaultRpmDownloader)
+	assert.NoError(t, err)
+
+	assert.NotNil(t, sources["org.osbuild.curl"])
+	assert.Nil(t, sources["org.osbuild.librepo"])
+}
+
+func TestGenSourcesRpmWithLibcurl(t *testing.T) {
+	inputs := SourceInputs{
+		Packages: []rpmmd.PackageSpec{opensslPkg},
+		RpmRepos: fakeRepos,
+	}
+	sources, err := GenSources(inputs, RpmDownloaderCurl)
+	assert.NoError(t, err)
+
+	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, string(jsonOutput), `{
+  "org.osbuild.curl": {
+    "items": {
+      "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+        "url": "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm"
+      }
+    }
+  }
+}`)
+}
+
+func TestGenSourcesRpmWithLibrepo(t *testing.T) {
+	inputs := SourceInputs{
+		Packages: []rpmmd.PackageSpec{opensslPkg},
+		RpmRepos: fakeRepos,
+	}
+	sources, err := GenSources(inputs, RpmDownloaderLibrepo)
+	assert.NoError(t, err)
+
+	jsonOutput, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, string(jsonOutput), `{
+  "org.osbuild.librepo": {
+    "items": {
+      "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+        "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+        "mirror": "repo_id_metalink"
+      }
+    },
+    "options": {
+      "mirrors": {
+        "repo_id_metalink": {
+          "url": "http://example.com/metalink",
+          "type": "metalink"
+        }
+      }
+    }
+  }
+}`)
+}
+
+func TestGenSourcesRpmBad(t *testing.T) {
+	inputs := SourceInputs{
+		Packages: []rpmmd.PackageSpec{opensslPkg},
+		RpmRepos: fakeRepos,
+	}
+	_, err := GenSources(inputs, 99)
+	assert.EqualError(t, err, "unknown rpm downloader 99")
+}