[mirror] module pull improvements

internal/mirror/cmd/pull/flags.go

@@ -72,7 +72,21 @@ func addFlags(flagSet *pflag.FlagSet) {
 		"include-module",
 		"i",
 		nil,
-		`Whitelist specific modules for downloading. Format is "module-name[@version]". Use one flag per each module. Disables blacklisting by --exclude-module."`,
+		`Whitelist specific modules for downloading. Use one flag per each module. Disables blacklisting by --exclude-module."
+
+Example:
+Available versions for <module-name>: v1.0.0, v1.1.0, v1.2.0, v1.3.0, v1.3.3, v1.4.1
+
+[email protected] → semver ^ constraint (^1.3.0): include v1.3.0, v1.3.3, v1.4.1
+
+module-name@~1.3.0 →  semver ~ constraint (>=1.3.0 <1.4.0): include only v1.3.0, v1.3.3
+
+module-name@=v1.3.0 → exact tag match: include only v1.3.0
+
+module-name@=bobV1 → exact tag match: include only bobV1
+
+module-name@=v1.3.0+stable → exact tag match: include only v1.3.0 and tag it as stable
+		`,
 	)
 	flagSet.StringArrayVarP(
 		&ModulesBlacklist,

internal/mirror/operations/pull_modules.go

@@ -95,6 +95,7 @@ func PullModules(pullParams *params.PullParams, filter *modules.Filter) error {
 	}
 
 	for name, layout := range imageLayouts.Modules {
+		ApplyChannelAliasesIfNeeded(name, layout, filter)
 		pkgName := "module-" + name + ".tar"
 		logger.InfoF("Packing %s", pkgName)
 
@@ -114,6 +115,15 @@ func PullModules(pullParams *params.PullParams, filter *modules.Filter) error {
 	return nil
 }
 
+func ApplyChannelAliasesIfNeeded(name string, layout layouts.ModuleImageLayout, filter *modules.Filter) {
+	if c, ok := filter.GetConstraint(name); ok && c.HasChannelAlias() {
+		ex, _ := c.(*modules.ExactTagConstraint)
+		if desc, err := layouts.FindImageDescriptorByTag(layout.ReleasesLayout, ex.Tag()); err == nil {
+			_ = layouts.TagImage(layout.ReleasesLayout, desc.Digest, ex.Channel())
+		}
+	}
+}
+
 func printModulesList(logger params.Logger, modulesData []modules.Module) {
 	logger.InfoF("Repo contains %d modules:", len(modulesData))
 	for i, module := range modulesData {

pkg/libmirror/modules/constraints.go

@@ -0,0 +1,80 @@
+package modules
+
+import (
+	"fmt"
+
+	"github.com/Masterminds/semver/v3"
+)
+
+type VersionConstraint interface {
+	Match(version interface{}) bool
+	IsExact() bool
+	HasChannelAlias() bool
+}
+
+type SemanticVersionConstraint struct {
+	constraint *semver.Constraints
+}
+
+func NewSemanticVersionConstraint(c string) (*SemanticVersionConstraint, error) {
+	constraint, err := semver.NewConstraint(c)
+	if err != nil {
+		return nil, fmt.Errorf("invalid semantic version constraint %q: %w", c, err)
+	}
+	return &SemanticVersionConstraint{constraint: constraint}, nil
+}
+
+func (s *SemanticVersionConstraint) HasChannelAlias() bool {
+	return false
+}
+
+func (s *SemanticVersionConstraint) Match(version interface{}) bool {
+	switch v := version.(type) {
+	case *semver.Version:
+		return s.constraint.Check(v)
+	default:
+		return false
+	}
+}
+
+func (s *SemanticVersionConstraint) IsExact() bool {
+	return false
+}
+
+type ExactTagConstraint struct {
+	tag     string
+	channel string
+}
+
+func (e *ExactTagConstraint) Tag() string {
+	return e.tag
+}
+
+func (e *ExactTagConstraint) Channel() string {
+	return e.channel
+}
+
+func NewExactTagConstraint(tag string) *ExactTagConstraint {
+	return &ExactTagConstraint{tag: tag}
+}
+
+func NewExactTagConstraintWithChannel(tag string, channel string) *ExactTagConstraint {
+	return &ExactTagConstraint{tag: tag, channel: channel}
+}
+
+func (e *ExactTagConstraint) Match(version interface{}) bool {
+	switch v := version.(type) {
+	case string:
+		return e.tag == v
+	default:
+		return false
+	}
+}
+
+func (e *ExactTagConstraint) IsExact() bool {
+	return true
+}
+
+func (e *ExactTagConstraint) HasChannelAlias() bool {
+	return e.channel != ""
+}

pkg/libmirror/modules/filter.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/Masterminds/semver/v3"
-
 	"github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params"
 )
 
@@ -32,11 +30,15 @@ const (
 	FilterTypeBlacklist
 )
 
+var validChannels = map[string]struct{}{
+	"alpha": {}, "beta": {}, "early-access": {}, "stable": {}, "rock-solid": {},
+}
+
 // Filter for modules by black and whitelists. Maps module names to minimal versions of these modules to be pulled.
 // By default, this is a whitelist filter, but that can be changed via SetType.
 type Filter struct {
 	_type   FilterType
-	modules map[string]*semver.Version
+	modules map[string]VersionConstraint
 	logger  params.Logger
 }
 
@@ -47,7 +49,7 @@ func NewFilter(filterExpressions []string, filterType FilterType) (*Filter, erro
 
 	filter := &Filter{
 		_type:   filterType,
-		modules: make(map[string]*semver.Version),
+		modules: make(map[string]VersionConstraint),
 	}
 	if len(filterExpressions) == 0 {
 		// Empty filter matches any module
@@ -56,25 +58,26 @@ func NewFilter(filterExpressions []string, filterType FilterType) (*Filter, erro
 	}
 
 	for _, filterExpr := range filterExpressions {
-		moduleName, moduleMinVersionString, validSplit := strings.Cut(strings.TrimSpace(filterExpr), "@")
+		moduleName, versionStr, hasVersion := strings.Cut(strings.TrimSpace(filterExpr), "@")
 		moduleName = strings.TrimSpace(moduleName)
 		if moduleName == "" {
 			return nil, fmt.Errorf("Malformed filter expression %q: empty module name", filterExpr)
 		}
 		if _, moduleRedeclared := filter.modules[moduleName]; moduleRedeclared {
 			return nil, fmt.Errorf("Malformed filter expression: module %s is declared multiple times", moduleName)
 		}
-		if !validSplit {
-			filter.modules[moduleName] = semver.New(0, 0, 0, "", "")
+		if !hasVersion {
+			constraint, _ := NewSemanticVersionConstraint(">=0.0.0")
+			filter.modules[moduleName] = constraint
 			continue
 		}
 
-		moduleMinVersion, err := semver.NewVersion(strings.TrimSpace(moduleMinVersionString))
+		constraint, err := parseVersionConstraint(versionStr)
 		if err != nil {
-			return nil, fmt.Errorf("Malformed filter expression %q: %w", filterExpr, err)
+			return nil, err
 		}
 
-		filter.modules[moduleName] = moduleMinVersion
+		filter.modules[moduleName] = constraint
 	}
 
 	return filter, nil
@@ -96,37 +99,85 @@ func (f *Filter) Match(mod *Module) bool {
 
 func (f *Filter) Len() int { return len(f.modules) }
 
-func (f *Filter) GetMinimalVersion(moduleName string) (*semver.Version, bool) {
-	v, found := f.modules[moduleName]
-	if found && v.Major() == 0 && v.Minor() == 0 && v.Patch() == 0 {
-		return nil, false
+func (f *Filter) GetConstraint(moduleName string) (VersionConstraint, bool) {
+	constraint, found := f.modules[moduleName]
+	return constraint, found
+}
+
+func parseVersionConstraint(v string) (VersionConstraint, error) {
+    v = strings.TrimSpace(v)
+    if v == "" {
+        return nil, fmt.Errorf("empty constraint")
+    }
+    switch v[0] {
+	// has user defined constraint (nothing to do)
+    case '=', '>', '<', '~', '^':
+    default:
+        // version without contraint (add ^ for backward compatibility)
+        v = "^" + v
+    }
+
+    // exact-match: "=1.2.3" or "=1.2.3+stable"
+    if v[0] == '=' {
+        return parseExact(v[1:])
+    }
+	// semver constraint
+    return parseSemver(v)
+}
+
+func parseExact(body string) (VersionConstraint, error) {
+	// exac match, console@=1.38.1 = registry.deckhouse.io/deckhouse/ce/modules/console:v1.38.1
+	tag, ch, _ := strings.Cut(body, "+")
+	if tag == "" {
+		return nil, fmt.Errorf("empty tag in %q", body)
+	}
+	if ch != "" {
+		if _, ok := validChannels[ch]; ok {
+			return NewExactTagConstraintWithChannel(tag, ch), nil
+		}
+	}
+	return NewExactTagConstraint(tag), nil
+}
+
+
+func parseSemver(v string) (VersionConstraint, error) {
+	// semver match, console@~1.38.1 = registry.deckhouse.io/deckhouse/ce/modules/console:v1.38.x
+	c, err := NewSemanticVersionConstraint(v)
+	if err != nil {
+		return nil, fmt.Errorf("invalid semver %q: %w", v, err)
 	}
-	return v, found
+	return c, nil
+}
+
+func (f *Filter) ShouldMirrorReleaseChannels(moduleName string) bool {
+    if c, ok := f.modules[moduleName]; ok && c.IsExact() {
+        return false
+    }
+    return true
 }
 
-func (f *Filter) FilterReleaseTagsAboveMinimal(mod *Module) {
-	moduleMinVersion, hasMinVersion := f.modules[mod.Name]
-	if !hasMinVersion {
-		return
+func (f *Filter) VersionsToMirror(mod *Module) []string {
+	c, ok := f.modules[mod.Name]
+	if !ok {
+		return nil
 	}
 
-	filteredReleases := make([]string, 0)
-	for _, tag := range mod.Releases {
-		v, err := semver.NewVersion(tag)
-		if err != nil {
-			if f.logger != nil {
-				f.logger.DebugLn("Failed to parse module release tag as semver", tag, err.Error())
-			}
-			filteredReleases = append(filteredReleases, tag) // This is probably a release channel, so just leave it
-			continue
+	if c.IsExact() {
+		if e, ok := c.(*ExactTagConstraint); ok {
+			return []string{e.Tag()}
 		}
+		return nil
+	}
 
-		if moduleMinVersion.GreaterThan(v) {
-			continue
+	sc, ok := c.(*SemanticVersionConstraint)
+	if !ok {
+		return nil
+	}
+	var tags []string
+	for _, v := range mod.Versions() {
+		if sc.Match(v) {
+			tags = append(tags, "v"+v.String())
 		}
-
-		filteredReleases = append(filteredReleases, tag)
 	}
-
-	mod.Releases = filteredReleases
+	return tags
 }