diff --git a/build/build.go b/build/build.go index 9dcd5c969f5f..7d7e3dcc4ddb 100644 --- a/build/build.go +++ b/build/build.go @@ -723,7 +723,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ return err } - dt, desc, err := itpull.Combine(ctx, srcs, indexAnnotations, false) + dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false) if err != nil { return err } diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 8a90e8babc4a..321f0ca3324d 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -7,6 +7,8 @@ import ( "os" "strings" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/platforms" "github.com/distribution/reference" "github.com/docker/buildx/builder" "github.com/docker/buildx/util/buildflags" @@ -14,6 +16,7 @@ import ( "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" + "github.com/moby/buildkit/util/attestation" "github.com/moby/buildkit/util/progress/progressui" "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -31,6 +34,7 @@ type createOptions struct { actionAppend bool progress string preferIndex bool + platforms []string } func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, args []string) error { @@ -67,6 +71,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return err } + platforms, err := parsePlatforms(in.platforms) + if err != nil { + return err + } + repos := map[string]struct{}{} for _, t := range tags { @@ -160,7 +169,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return errors.Wrapf(err, "failed to parse annotations") } - dt, desc, err := r.Combine(ctx, srcs, annotations, in.preferIndex) + dt, desc, srcMap, err := r.Combine(ctx, srcs, annotations, in.preferIndex) + if err != nil { + return err + } + + dt, desc, manifests, err := filterPlatforms(dt, desc, srcMap, platforms) if err != nil { return err } @@ -170,6 +184,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return nil } + // manifests can be nil only if pushing one single-platform desc directly + if manifests == nil { + manifests = []descWithSource{{Descriptor: desc, Source: srcs[0]}} + } + // new resolver cause need new auth r = imagetools.New(imageopt) @@ -187,17 +206,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg eg.Go(func() error { return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error { eg2, _ := errgroup.WithContext(ctx) - for _, s := range srcs { - if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) { - continue - } - s := s + for _, desc := range manifests { eg2.Go(func() error { - sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String())) - return r.Copy(ctx, s, t) + sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", desc.Digest.String(), desc.Source.Ref.String(), t.String())) + return r.Copy(ctx, desc.Source, t) }) } - if err := eg2.Wait(); err != nil { return err } @@ -216,6 +230,107 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return err } +type descWithSource struct { + ocispecs.Descriptor + Source *imagetools.Source +} + +func filterPlatforms(dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*imagetools.Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []descWithSource, error) { + if len(plats) == 0 { + return dt, desc, nil, nil + } + + matcher := platforms.Any(plats...) + + if !images.IsIndexType(desc.MediaType) { + var mfst ocispecs.Manifest + if err := json.Unmarshal(dt, &mfst); err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest") + } + if desc.Platform == nil { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information") + } + if !matcher.Match(*desc.Platform) { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform)) + } + return dt, desc, nil, nil + } + + var idx ocispecs.Index + if err := json.Unmarshal(dt, &idx); err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index") + } + + manifestMap := map[digest.Digest]ocispecs.Descriptor{} + for _, m := range idx.Manifests { + manifestMap[m.Digest] = m + } + references := map[digest.Digest]struct{}{} + for _, m := range idx.Manifests { + if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault { + dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest] + if !ok { + continue + } + dgst, err := digest.Parse(dgstStr) + if err != nil { + continue + } + subject, ok := manifestMap[dgst] + if !ok { + continue + } + if subject.Platform == nil || matcher.Match(*subject.Platform) { + references[m.Digest] = struct{}{} + } + } + } + + var mfsts []ocispecs.Descriptor + var mfstsWithSource []descWithSource + + for _, m := range idx.Manifests { + if _, isRef := references[m.Digest]; isRef || m.Platform == nil || matcher.Match(*m.Platform) { + src, ok := srcMap[m.Digest] + if !ok { + defaultSource, ok := srcMap[desc.Digest] + if !ok { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest) + } + src = defaultSource + } + mfsts = append(mfsts, m) + mfstsWithSource = append(mfstsWithSource, descWithSource{ + Descriptor: m, + Source: src, + }) + } + } + if len(mfsts) == len(idx.Manifests) { + // all platforms matched, no need to rewrite index + return dt, desc, mfstsWithSource, nil + } + + if len(mfsts) == 0 { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms") + } + + idx.Manifests = mfsts + idxBytes, err := json.MarshalIndent(&idx, "", " ") + if err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index") + } + + desc = ocispecs.Descriptor{ + MediaType: desc.MediaType, + Size: int64(len(idxBytes)), + Digest: digest.FromBytes(idxBytes), + Annotations: desc.Annotations, + } + + return idxBytes, desc, mfstsWithSource, nil +} + func parseSources(in []string) ([]*imagetools.Source, error) { out := make([]*imagetools.Source, len(in)) for i, in := range in { @@ -228,6 +343,26 @@ func parseSources(in []string) ([]*imagetools.Source, error) { return out, nil } +func parsePlatforms(in []string) ([]ocispecs.Platform, error) { + out := make([]ocispecs.Platform, 0, len(in)) + for _, p := range in { + if arr := strings.Split(p, ","); len(arr) > 1 { + v, err := parsePlatforms(arr) + if err != nil { + return nil, err + } + out = append(out, v...) + continue + } + plat, err := platforms.Parse(p) + if err != nil { + return nil, errors.Wrapf(err, "invalid platform %q", p) + } + out = append(out, plat) + } + return out, nil +} + func parseRefs(in []string) ([]reference.Named, error) { refs := make([]reference.Named, len(in)) for i, in := range in { @@ -291,6 +426,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command { flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty", "rawjson"). Use plain to show container output`) flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image") flags.BoolVar(&options.preferIndex, "prefer-index", true, "When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy") + flags.StringArrayVarP(&options.platforms, "platform", "p", []string{}, "Filter specified platforms of target image") return cmd } diff --git a/docs/reference/buildx_imagetools_create.md b/docs/reference/buildx_imagetools_create.md index 4e74b03a6fcd..157f99ab7b12 100644 --- a/docs/reference/buildx_imagetools_create.md +++ b/docs/reference/buildx_imagetools_create.md @@ -17,6 +17,7 @@ Create a new image based on source images | `-D`, `--debug` | `bool` | | Enable debug logging | | [`--dry-run`](#dry-run) | `bool` | | Show final image instead of pushing | | [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file | +| `-p`, `--platform` | `stringArray` | | Filter specified platforms of target image | | `--prefer-index` | `bool` | `true` | When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy | | `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`, `rawjson`). Use plain to show container output | | [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image | diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 91afce35642f..b266286e7c76 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -28,7 +28,7 @@ type Source struct { Ref reference.Named } -func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, error) { +func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) { eg, ctx := errgroup.WithContext(ctx) dts := make([][]byte, len(srcs)) @@ -73,7 +73,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes } if err := eg.Wait(); err != nil { - return nil, ocispecs.Descriptor{}, err + return nil, ocispecs.Descriptor{}, nil, err } // on single source, return original bytes @@ -83,21 +83,25 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes // of preferIndex since if set to true then the source is already in the preferred format, and if false // it doesn't matter since we're not going to split it into separate manifests case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex: - return dts[0], srcs[0].Desc, nil + srcMap := map[digest.Digest]*Source{ + srcs[0].Desc.Digest: srcs[0], + } + return dts[0], srcs[0].Desc, srcMap, nil default: if !preferIndex { - return dts[0], srcs[0].Desc, nil + return dts[0], srcs[0].Desc, nil, nil } } } - m := map[digest.Digest]int{} - newDescs := make([]ocispecs.Descriptor, 0, len(srcs)) + indexes := map[digest.Digest]int{} + sources := map[digest.Digest]*Source{} + descs := make([]ocispecs.Descriptor, 0, len(srcs)) - addDesc := func(d ocispecs.Descriptor) { - idx, ok := m[d.Digest] + addDesc := func(d ocispecs.Descriptor, src *Source) { + idx, ok := indexes[d.Digest] if ok { - old := newDescs[idx] + old := descs[idx] if old.MediaType == "" { old.MediaType = d.MediaType } @@ -108,11 +112,12 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes old.Annotations = map[string]string{} } maps.Copy(old.Annotations, d.Annotations) - newDescs[idx] = old + descs[idx] = old } else { - m[d.Digest] = len(newDescs) - newDescs = append(newDescs, d) + indexes[d.Digest] = len(descs) + descs = append(descs, d) } + sources[d.Digest] = src } for i, src := range srcs { @@ -120,25 +125,25 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex: var mfst ocispecs.Index if err := json.Unmarshal(dts[i], &mfst); err != nil { - return nil, ocispecs.Descriptor{}, errors.WithStack(err) + return nil, ocispecs.Descriptor{}, nil, errors.WithStack(err) } for _, d := range mfst.Manifests { - addDesc(d) + addDesc(d, src) } default: - addDesc(src.Desc) + addDesc(src.Desc, src) } } dockerMfsts := 0 - for _, desc := range newDescs { + for _, desc := range descs { if strings.HasPrefix(desc.MediaType, "application/vnd.docker.") { dockerMfsts++ } } var mt string - if dockerMfsts == len(newDescs) { + if dockerMfsts == len(descs) { // all manifests are Docker types, use Docker manifest list mt = images.MediaTypeDockerSchema2ManifestList } else { @@ -154,18 +159,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes case exptypes.AnnotationIndex: indexAnnotation[k.Key] = v case exptypes.AnnotationManifestDescriptor: - for i := range newDescs { - if newDescs[i].Annotations == nil { - newDescs[i].Annotations = map[string]string{} + for i := range descs { + if descs[i].Annotations == nil { + descs[i].Annotations = map[string]string{} } - if k.Platform == nil || k.PlatformString() == platforms.Format(*newDescs[i].Platform) { - newDescs[i].Annotations[k.Key] = v + if k.Platform == nil || k.PlatformString() == platforms.Format(*descs[i].Platform) { + descs[i].Annotations[k.Key] = v } } case exptypes.AnnotationManifest, "": - return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are not supported yet", k.Type) + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are not supported yet", k.Type) case exptypes.AnnotationIndexDescriptor: - return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are invalid while creating an image", k.Type) + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are invalid while creating an image", k.Type) } } } @@ -175,18 +180,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes Versioned: specs.Versioned{ SchemaVersion: 2, }, - Manifests: newDescs, + Manifests: descs, Annotations: indexAnnotation, }, "", " ") if err != nil { - return nil, ocispecs.Descriptor{}, errors.Wrap(err, "failed to marshal index") + return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index") } return idxBytes, ocispecs.Descriptor{ MediaType: mt, Size: int64(len(idxBytes)), Digest: digest.FromBytes(idxBytes), - }, nil + }, sources, nil } func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispecs.Descriptor, dt []byte) error { diff --git a/vendor/github.com/moby/buildkit/util/attestation/types.go b/vendor/github.com/moby/buildkit/util/attestation/types.go new file mode 100644 index 000000000000..accccd307e24 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/attestation/types.go @@ -0,0 +1,9 @@ +package attestation + +const ( + DockerAnnotationReferenceType = "vnd.docker.reference.type" + DockerAnnotationReferenceDigest = "vnd.docker.reference.digest" + DockerAnnotationReferenceDescription = "vnd.docker.reference.description" + + DockerAnnotationReferenceTypeDefault = "attestation-manifest" +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 8a40f6fdcf6a..a8c54a1445bb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -504,6 +504,7 @@ github.com/moby/buildkit/util/apicaps github.com/moby/buildkit/util/apicaps/pb github.com/moby/buildkit/util/appcontext github.com/moby/buildkit/util/appdefaults +github.com/moby/buildkit/util/attestation github.com/moby/buildkit/util/bklog github.com/moby/buildkit/util/contentutil github.com/moby/buildkit/util/disk