Merge pull request #111 from deckhouse/feature/tagged-release-channel-propagation

[mirror] for `--deckhouse-tag` bundles, add release channels pointing to the tagged release to the bundle if possible

Author: name212

internal/mirror/cmd/pull/flags.go

@@ -65,7 +65,7 @@ func addFlags(flagSet *pflag.FlagSet) {
 		&DeckhouseTag,
 		"deckhouse-tag",
 		"",
-		"Specific Deckhouse build tag to pull. Conflicts with --since-version. WARNING!: Clusters installed with this option will not be able to automatically update due to lack of release-channels information in bundle and, as such, will require special attention and manual intervention during updates.",
+		"Specific Deckhouse build tag to pull. Conflicts with --since-version. If registry contains release channel image for specified tag, all release channels in the bundle will be pointed to it.",
 	)
 	flagSet.StringArrayVarP(
 		&ModulesWhitelist,

internal/mirror/cmd/push/push.go

@@ -139,7 +139,11 @@ func pushModules(pushParams *params.PushParams, logger params.Logger) error {
 			logger.WarnLn(err)
 		}
 	}
-	logger.InfoF("Modules pushed: %v", strings.Join(successfullyPushedModules, ", "))
+
+	if len(successfullyPushedModules) > 0 {
+		logger.InfoF("Modules pushed: %v", strings.Join(successfullyPushedModules, ", "))
+	}
+
 	return nil
 }
 

internal/mirror/operations/pull_dkp.go

@@ -18,6 +18,7 @@ package operations
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"maps"
@@ -65,7 +66,7 @@ func PullDeckhousePlatform(pullParams *params.PullParams, tagsToMirror []string)
 		return err
 	}
 
-	// We should not generate deckhousereleases.yaml manifest for bundles that don't have release channels
+	// We should not generate deckhousereleases.yaml manifest for tag-based pulls
 	if pullParams.DeckhouseTag == "" {
 		if err = generateDeckhouseReleaseManifests(pullParams, tagsToMirror, imageLayouts, logger); err != nil {
 			return err
@@ -88,12 +89,36 @@ func PullDeckhousePlatform(pullParams *params.PullParams, tagsToMirror []string)
 		return fmt.Errorf("Pull Deckhouse images: %w", err)
 	}
 
-	logger.InfoLn("Processing image indexes")
-	for _, l := range imageLayouts.Layouts() {
-		err = layouts.SortIndexManifests(l)
-		if err != nil {
-			return fmt.Errorf("Sorting index manifests of %s: %w", l, err)
+	err = logger.Process("Processing image indexes", func() error {
+		if pullParams.DeckhouseTag != "" {
+			// If we are pulling some build by tag, propagate release channel image of it to all channels if it exists.
+			releaseChannel, err := layouts.FindImageDescriptorByTag(imageLayouts.ReleaseChannel, pullParams.DeckhouseTag)
+			switch {
+			case errors.Is(err, layouts.ErrImageNotFound):
+				logger.WarnLn("Registry does not contain release channels, release channels images will not be added to bundle")
+				goto sortManifests
+			case err != nil:
+				return fmt.Errorf("Find release-%s channel descriptor: %w", pullParams.DeckhouseTag, err)
+			}
+
+			for _, channel := range []string{"alpha", "beta", "early-access", "stable", "rock-solid"} {
+				if err = layouts.TagImage(imageLayouts.ReleaseChannel, releaseChannel.Digest, channel); err != nil {
+					return fmt.Errorf("tag release channel: %w", err)
+				}
+			}
+		}
+
+	sortManifests:
+		for _, l := range imageLayouts.AsList() {
+			err = layouts.SortIndexManifests(l)
+			if err != nil {
+				return fmt.Errorf("Sorting index manifests of %s: %w", l, err)
+			}
 		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("Processing image indexes: %w", err)
 	}
 
 	if err = logger.Process("Pack Deckhouse images into platform.tar", func() error {

internal/mirror/operations/pull_modules.go

@@ -89,7 +89,7 @@ func PullModules(pullParams *params.PullParams, filter *modules.Filter) error {
 	}
 
 	logger.InfoLn("Processing image indexes")
-	for _, l := range imageLayouts.Layouts() {
+	for _, l := range imageLayouts.AsList() {
 		err = layouts.SortIndexManifests(l)
 		if err != nil {
 			return fmt.Errorf("Sorting index manifests of %s: %w", l, err)

internal/mirror/operations/pull_security.go

@@ -57,7 +57,7 @@ func PullSecurityDatabases(pullParams *params.PullParams) error {
 	}
 
 	logger.InfoLn("Processing image indexes")
-	for _, l := range imageLayouts.Layouts() {
+	for _, l := range imageLayouts.AsList() {
 		err = layouts.SortIndexManifests(l)
 		if err != nil {
 			return fmt.Errorf("Sorting index manifests of %s: %w", l, err)

pkg/libmirror/layouts/layouts.go

@@ -18,6 +18,7 @@ package layouts
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -75,8 +76,8 @@ func NewImageLayouts() *ImageLayouts {
 	}
 }
 
-// Layouts returns a list of layout.Path's in it. Undefined path's are not included in the list.
-func (l *ImageLayouts) Layouts() []layout.Path {
+// AsList returns a list of layout.Path's in it. Undefined path's are not included in the list.
+func (l *ImageLayouts) AsList() []layout.Path {
 	layoutsValue := reflect.ValueOf(l).Elem()
 	layoutPathType := reflect.TypeOf(layout.Path(""))
 
@@ -284,7 +285,7 @@ func FindDeckhouseModulesImages(
 		maps.Copy(moduleImageLayouts.ReleaseImages, releaseImages)
 
 		if len(moduleImageLayouts.ModuleImages) == 0 {
-			return fmt.Errorf("found no releases matching filter for %s", module.Name)
+			return fmt.Errorf("found no releases for %s", module.Name)
 		}
 
 		layouts.Modules[module.Name] = moduleImageLayouts
@@ -293,6 +294,8 @@ func FindDeckhouseModulesImages(
 	return nil
 }
 
+var ErrImageNotFound = errors.New("image not found")
+
 func FindImageByTag(l layout.Path, tag string) (v1.Image, error) {
 	index, err := l.ImageIndex()
 	if err != nil {
@@ -303,13 +306,62 @@ func FindImageByTag(l layout.Path, tag string) (v1.Image, error) {
 		return nil, err
 	}
 
-	for _, imageManifest := range indexManifest.Manifests {
-		for key, value := range imageManifest.Annotations {
+	for _, imageDescriptor := range indexManifest.Manifests {
+		for key, value := range imageDescriptor.Annotations {
+			if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) {
+				return index.Image(imageDescriptor.Digest)
+			}
+		}
+	}
+
+	return nil, ErrImageNotFound
+}
+
+func FindImageDescriptorByTag(l layout.Path, tag string) (v1.Descriptor, error) {
+	index, err := l.ImageIndex()
+	if err != nil {
+		return v1.Descriptor{}, err
+	}
+	indexManifest, err := index.IndexManifest()
+	if err != nil {
+		return v1.Descriptor{}, err
+	}
+
+	for _, imageDescriptor := range indexManifest.Manifests {
+		for key, value := range imageDescriptor.Annotations {
 			if key == "org.opencontainers.image.ref.name" && strings.HasSuffix(value, ":"+tag) {
-				return index.Image(imageManifest.Digest)
+				return imageDescriptor, nil
+			}
+		}
+	}
+
+	return v1.Descriptor{}, ErrImageNotFound
+}
+
+func TagImage(l layout.Path, imageDigest v1.Hash, tag string) error {
+	index, err := l.ImageIndex()
+	if err != nil {
+		return err
+	}
+	indexManifest, err := index.IndexManifest()
+	if err != nil {
+		return err
+	}
+
+	for _, imageDescriptor := range indexManifest.Manifests {
+		if imageDescriptor.Digest == imageDigest {
+			imageRepo, _, found := strings.Cut(imageDescriptor.Annotations["org.opencontainers.image.ref.name"], ":")
+			// If there is no ":" symbol in the image reference, then it must be a reference by digest and those are fine as is
+			if found {
+				imageDescriptor.Annotations["org.opencontainers.image.ref.name"] = imageRepo + ":" + tag
+			}
+			imageDescriptor.Annotations["io.deckhouse.image.short_tag"] = tag
+			if err = l.AppendDescriptor(imageDescriptor); err != nil {
+				return fmt.Errorf("append descriptor %s: %w", tag, err)
 			}
+			return nil
 		}
 	}
 
-	return nil, nil
+	return ErrImageNotFound
 }

pkg/libmirror/layouts/layouts_test.go

@@ -49,8 +49,8 @@ func TestImagesLayoutsAllLayouts(t *testing.T) {
 		},
 	}
 
-	layouts := il.Layouts()
-	require.Len(t, layouts, 7, "Layouts should return exactly the number of non-empty layouts defined within it")
+	layouts := il.AsList()
+	require.Len(t, layouts, 7, "AsList should return exactly the number of non-empty layouts defined within it")
 	require.Contains(t, layouts, il.Deckhouse, "All non-empty layouts should be returned")
 	require.Contains(t, layouts, il.ReleaseChannel, "All non-empty layouts should be returned")
 	require.Contains(t, layouts, il.InstallStandalone, "All non-empty layouts should be returned")