diff --git a/api/oci/extensions/repositories/artifactset/artifactset.go b/api/oci/extensions/repositories/artifactset/artifactset.go index b14f019130..1edc16b062 100644 --- a/api/oci/extensions/repositories/artifactset/artifactset.go +++ b/api/oci/extensions/repositories/artifactset/artifactset.go @@ -1,6 +1,8 @@ package artifactset import ( + "maps" + "slices" "strings" "github.com/mandelsoft/goutils/errors" @@ -203,27 +205,106 @@ func (a *namespaceContainer) AddTags(digest digest.Digest, tags ...string) error a.base.Lock() defer a.base.Unlock() - idx := a.GetIndex() - for i, e := range idx.Manifests { - if e.Digest == digest { - if e.Annotations == nil { - e.Annotations = map[string]string{} - idx.Manifests[i].Annotations = e.Annotations + index := a.GetIndex() + + manifests := index.Manifests + + artifacts := decodeIndexManifests(manifests) + + art := artifacts[digest] + if art == nil { + return errors.ErrUnknown(cpi.KIND_OCIARTIFACT, digest.String()) + } + + for _, tag := range tags { + art.addTag(tag) + } + + isOCI := a.base. + FileSystemBlobAccess. + Access(). + GetInfo(). + GetDescriptorFileName() == OCIArtifactSetDescriptorFileName + + index.Manifests = encodeIndexManifests(artifacts, isOCI) + + return nil +} + +type descriptorWithTags struct { + cpi.Descriptor + Tags map[string]struct{} +} + +func (d *descriptorWithTags) addTag(tag string) { + if tag := strings.TrimSpace(tag); tag != "" { + d.Tags[tag] = struct{}{} + } +} + +func decodeIndexManifests(manifests []cpi.Descriptor) map[digest.Digest]*descriptorWithTags { + out := map[digest.Digest]*descriptorWithTags{} + for _, m := range manifests { + if out[m.Digest] == nil { + out[m.Digest] = &descriptorWithTags{ + Descriptor: normalizeDecodedDescriptorWithoutTags(m), + Tags: map[string]struct{}{}, } - cur := RetrieveTags(e.Annotations) - if cur != "" { - cur = strings.Join(append([]string{cur}, tags...), ",") - } else { - cur = strings.Join(tags, ",") + } + annotated := out[m.Digest] + + // OCM multi-tag annotation + tagsFromAnnotations := strings.Split(RetrieveTags(m.Annotations), ",") + for _, tag := range tagsFromAnnotations { + annotated.addTag(tag) + } + + // OCI single-tag annotation + if tag, ok := m.Annotations[OCITAG_ANNOTATION]; ok { + annotated.addTag(tag) + } + } + + return out +} + +func normalizeDecodedDescriptorWithoutTags(d cpi.Descriptor) cpi.Descriptor { + d.Annotations = maps.Clone(d.Annotations) + delete(d.Annotations, TAGS_ANNOTATION) + delete(d.Annotations, OCITAG_ANNOTATION) + if len(d.Annotations) == 0 { + d.Annotations = nil + } + return d +} + +func encodeIndexManifests( + descriptors map[digest.Digest]*descriptorWithTags, + oci bool, +) []cpi.Descriptor { + var manifests []cpi.Descriptor + for _, desc := range descriptors { + tags := slices.Sorted(maps.Keys(desc.Tags)) + joined := strings.Join(tags, ",") + if !oci { + d := desc.Descriptor + d.Annotations = map[string]string{ + TAGS_ANNOTATION: joined, } - e.Annotations[TAGS_ANNOTATION] = cur - if a.base.FileSystemBlobAccess.Access().GetInfo().GetDescriptorFileName() == OCIArtifactSetDescriptorFileName { - e.Annotations[OCITAG_ANNOTATION] = tags[0] + manifests = append(manifests, d) + continue + } + for _, t := range tags { + d := desc.Descriptor + d.Annotations = map[string]string{ + TAGS_ANNOTATION: joined, + OCITAG_ANNOTATION: t, } - return nil + manifests = append(manifests, d) } } - return errors.ErrUnknown(cpi.KIND_OCIARTIFACT, digest.String()) + + return manifests } //////////////////////////////////////////////////////////////////////////////// @@ -437,14 +518,24 @@ func (a *namespaceContainer) AddPlatformArtifact(artifact cpi.Artifact, platform return nil, err } - idx.Manifests = append(idx.Manifests, cpi.Descriptor{ - MediaType: blob.MimeType(), + desc := cpi.Descriptor{ Digest: blob.Digest(), Size: blob.Size(), URLs: nil, Annotations: nil, Platform: platform, - }) + } + + isOCI := a.base.FileSystemBlobAccess. + Access(). + GetInfo(). + GetDescriptorFileName() == OCIArtifactSetDescriptorFileName + + if isOCI { + desc.MediaType = blob.MimeType() + } + + idx.Manifests = append(idx.Manifests, desc) return blob, nil } diff --git a/api/oci/extensions/repositories/artifactset/format.go b/api/oci/extensions/repositories/artifactset/format.go index be2186d0a7..af1edd754b 100644 --- a/api/oci/extensions/repositories/artifactset/format.go +++ b/api/oci/extensions/repositories/artifactset/format.go @@ -244,12 +244,13 @@ func OpenFromDataAccess(acc accessobj.AccessMode, mediatype string, data blobacc } defer reader.Close() o.SetReader(reader) - fmt := accessio.FormatTar - - if mime.IsGZip(mediatype) { - fmt = accessio.FormatTGZ + if o.GetFileFormat() == nil { + fmt := accessio.FormatTar + if mime.IsGZip(mediatype) { + fmt = accessio.FormatTGZ + } + o.SetFileFormat(fmt) } - o.SetFileFormat(fmt) return Open(acc&accessobj.ACC_READONLY, "", 0, o) } diff --git a/api/oci/extensions/repositories/artifactset/utils_synthesis.go b/api/oci/extensions/repositories/artifactset/utils_synthesis.go index 74165c1154..43047f589c 100644 --- a/api/oci/extensions/repositories/artifactset/utils_synthesis.go +++ b/api/oci/extensions/repositories/artifactset/utils_synthesis.go @@ -84,18 +84,24 @@ func SynthesizeArtifactBlobWithFilter(ns cpi.NamespaceAccess, ref string, filter return nil, err } } - return SynthesizeArtifactBlobForArtifact(art, ref, filter) + return SynthesizeArtifactBlobForArtifact(art, []string{ref}, filter) } -func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, ref string, filter ...filters.Filter) (ArtifactBlob, error) { +func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, refs []string, filter ...filters.Filter) (ArtifactBlob, error) { blob, err := art.Blob() if err != nil { return nil, err } - vers, err := ociutils.ParseVersion(ref) - if err != nil { - return nil, err + tags := make([]string, 0, len(refs)) + for _, ref := range refs { + vers, err := ociutils.ParseVersion(ref) + if err != nil { + return nil, err + } + if vers.IsTagged() { + tags = append(tags, vers.GetTag()) + } } return SythesizeArtifactSet(func(set *ArtifactSet) (string, error) { @@ -104,9 +110,8 @@ func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, ref string, filte return "", fmt.Errorf("failed to transfer artifact: %w", err) } - if ok := vers.IsTagged(); ok { - err = set.AddTags(*dig, vers.GetTag()) - if err != nil { + if len(tags) > 0 { + if err := set.AddTags(*dig, tags...); err != nil { return "", fmt.Errorf("failed to add tag: %w", err) } } diff --git a/api/oci/extensions/repositories/ctf/index/ctfindex.go b/api/oci/extensions/repositories/ctf/index/ctfindex.go index 666d476ad8..247687fa06 100644 --- a/api/oci/extensions/repositories/ctf/index/ctfindex.go +++ b/api/oci/extensions/repositories/ctf/index/ctfindex.go @@ -18,6 +18,7 @@ type ArtifactMeta struct { Repository string `json:"repository"` Tag string `json:"tag,omitempty"` Digest digest.Digest `json:"digest,omitempty"` + MediaType string `json:"mediaType,omitempty"` } func Decode(data []byte) (*ArtifactIndex, error) { diff --git a/api/oci/extensions/repositories/ctf/index/index.go b/api/oci/extensions/repositories/ctf/index/index.go index 6bc2f9ddbf..dcd1557905 100644 --- a/api/oci/extensions/repositories/ctf/index/index.go +++ b/api/oci/extensions/repositories/ctf/index/index.go @@ -241,6 +241,7 @@ func (r *RepositoryIndex) GetDescriptor() *ArtifactIndex { Repository: vers.Repository, Tag: vers.Tag, Digest: vers.Digest, + MediaType: vers.MediaType, } index.Index = append(index.Index, *d) } diff --git a/api/oci/extensions/repositories/ctf/namespace.go b/api/oci/extensions/repositories/ctf/namespace.go index ac57fd3cb7..472e3b730c 100644 --- a/api/oci/extensions/repositories/ctf/namespace.go +++ b/api/oci/extensions/repositories/ctf/namespace.go @@ -40,10 +40,6 @@ func (n *namespaceContainer) Close() error { return nil } -func (n *namespaceContainer) GetBlobDescriptor(digest digest.Digest) *cpi.Descriptor { - return nil -} - func (n *namespaceContainer) ListTags() ([]string, error) { return n.repo.getIndex().GetTags(n.impl.GetNamespace()), nil // return digests as tags, also } @@ -84,6 +80,7 @@ func (n *namespaceContainer) AddArtifact(artifact cpi.Artifact, tags ...string) Repository: n.impl.GetNamespace(), Tag: "", Digest: blob.Digest(), + MediaType: blob.MimeType(), }) return blob, n.AddTags(blob.Digest(), tags...) } diff --git a/api/ocm/extensions/accessmethods/ociartifact/method.go b/api/ocm/extensions/accessmethods/ociartifact/method.go index 38fef2b5ef..86e843e340 100644 --- a/api/ocm/extensions/accessmethods/ociartifact/method.go +++ b/api/ocm/extensions/accessmethods/ociartifact/method.go @@ -35,6 +35,9 @@ const ( const ( LegacyType = "ociRegistry" LegacyTypeV1 = LegacyType + runtime.VersionSeparator + "v1" + + LegacyType2 = "OCIImage" + LegacyType2V1 = LegacyType2 + runtime.VersionSeparator + "v1" ) func init() { @@ -43,6 +46,9 @@ func init() { accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType)) accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyTypeV1)) + + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2)) + accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2V1)) } func Is(spec accspeccpi.AccessSpec) bool { @@ -355,7 +361,7 @@ func (m *accessMethod) getBlob() (artifactset.ArtifactBlob, error) { } logger := Logger(WrapContextProvider(m.ctx)) logger.Info("synthesize artifact blob", "ref", m.reference) - m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, m.ref.VersionSpec()) + m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, []string{m.ref.VersionSpec()}) logger.Info("synthesize artifact blob done", "ref", m.reference, "error", logging.ErrorMessage(err)) if err != nil { m.err = err diff --git a/api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go b/api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go index c661c9f294..02e7c87ef3 100644 --- a/api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go +++ b/api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go @@ -2,6 +2,7 @@ package genericocireg import ( "bytes" + "fmt" "io" "os" "strings" @@ -13,6 +14,7 @@ import ( "ocm.software/ocm/api/oci" "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" "ocm.software/ocm/api/ocm/cpi/accspeccpi" "ocm.software/ocm/api/ocm/extensions/accessmethods/localblob" "ocm.software/ocm/api/utils/blobaccess/blobaccess" @@ -26,6 +28,7 @@ type localBlobAccessMethod struct { spec *localblob.AccessSpec namespace oci.NamespaceAccess artifact oci.ArtifactAccess + mimeType string } var _ accspeccpi.AccessMethodImpl = (*localBlobAccessMethod)(nil) @@ -40,6 +43,11 @@ func newLocalBlobAccessMethodImpl(a *localblob.AccessSpec, ns oci.NamespaceAcces namespace: ns, artifact: art, } + if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest { + // if we discover a localblob with an index or manifest media type, we can + // assume that we are dealing with a new style of artifact created by the new reference library. + m.mimeType = artifactset.MediaType(m.spec.MediaType) + } ref.BeforeCleanup(refmgmt.CleanupHandlerFunc(m.cache)) return m, nil } @@ -99,8 +107,35 @@ func (m *localBlobAccessMethod) getBlob() (blobaccess.DataAccess, error) { err error ) if len(refs) < 2 { - _, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference)) - if err != nil { + if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest { + // if we have a nested manifest or index, we can use the blob synthesis utility here to download + // the entire artifact set. + art, err := m.namespace.GetArtifact(m.spec.LocalReference) + if err != nil { + return nil, fmt.Errorf("failed to get artifact for local reference %q: %w", m.spec.LocalReference, err) + } + defer art.Close() + var artifactRefs []string + if m.spec.ReferenceName != "" { + // if we have a reference name, it consists of repository and tag + // so we can extract the tag to use it as target, instead of latest + refSpec, err := oci.ParseRef(m.spec.ReferenceName) + if err != nil { + return nil, fmt.Errorf("failed to parse reference name %q: %w", m.spec.ReferenceName, err) + } + if refSpec.GetTag() != "" { + artifactRefs = append(artifactRefs, refSpec.GetTag()) + } + } + localReferenceDigest := digest.Digest(m.spec.LocalReference) + artifactRefs = append(artifactRefs, fmt.Sprintf("%s-%s", localReferenceDigest.Algorithm(), localReferenceDigest.Encoded())) + artifactRefs = append(artifactRefs, "latest") + artblob, err := artifactset.SynthesizeArtifactBlobForArtifact(art, artifactRefs) + if err != nil { + return nil, fmt.Errorf("failed to synthesize artifact blob: %w", err) + } + data = artblob + } else if _, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference)); err != nil { return nil, err } } else { @@ -123,10 +158,13 @@ func (m *localBlobAccessMethod) Get() ([]byte, error) { } func (m *localBlobAccessMethod) MimeType() string { + if m.mimeType != "" { + return m.mimeType + } return m.spec.MediaType } -//////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////// type composedBlock struct { m *localBlobAccessMethod diff --git a/api/ocm/extensions/repositories/genericocireg/componentversion.go b/api/ocm/extensions/repositories/genericocireg/componentversion.go index 37553e1710..c225df70b4 100644 --- a/api/ocm/extensions/repositories/genericocireg/componentversion.go +++ b/api/ocm/extensions/repositories/genericocireg/componentversion.go @@ -46,38 +46,81 @@ func newComponentVersionAccess(mode accessobj.AccessMode, comp *componentAccessI return &repocpi.ComponentVersionAccessInfo{c, true, persistent}, nil } -//////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////// type ComponentVersionContainer struct { bridge repocpi.ComponentVersionAccessBridge - comp *componentAccessImpl - version string - access oci.ArtifactAccess - manifest oci.ManifestAccess - state accessobj.State + comp *componentAccessImpl + version string + indexArtifact oci.ArtifactAccess + manifestArtifact oci.ArtifactAccess + manifest oci.ManifestAccess + index oci.IndexAccess + state accessobj.State } var _ repocpi.ComponentVersionAccessImpl = (*ComponentVersionContainer)(nil) func newComponentVersionContainer(mode accessobj.AccessMode, comp *componentAccessImpl, version string, access oci.ArtifactAccess) (*ComponentVersionContainer, error) { - m := access.ManifestAccess() - if m == nil { - return nil, errors.ErrInvalid("artifact type") + var m oci.ManifestAccess + var i oci.IndexAccess + + var manifestArtifact, indexArtifact oci.ArtifactAccess + + if access.IsIndex() { + idx, err := access.Index() + if err != nil { + return nil, err + } + if len(idx.Manifests) < 1 { + return nil, fmt.Errorf("index has no manifests") + } + first := idx.Manifests[0] + firstArtifact, err := access.GetArtifact(first.Digest) + if err != nil { + return nil, err + } + if !firstArtifact.IsManifest() { + return nil, fmt.Errorf("first manifest in index is not a manifest") + } + + m = firstArtifact.ManifestAccess() + manifestArtifact = firstArtifact + indexArtifact = access + } else { + m = access.ManifestAccess() + if m == nil { + return nil, fmt.Errorf("artifact is neither manifest nor index") + } + + manifestArtifact = access } - state, err := NewState(mode, comp.name, version, m, compatattr.Get(comp.GetContext())) + + state, err := NewState(mode, comp.name, version, i, m, compatattr.Get(comp.GetContext())) if err != nil { - access.Close() + err = errors.Join(err, manifestArtifact.Close()) + if indexArtifact != nil { + err = errors.Join(err, indexArtifact.Close()) + } return nil, err } - return &ComponentVersionContainer{ - comp: comp, - version: version, - access: access, - manifest: m, - state: state, - }, nil + container := &ComponentVersionContainer{ + comp: comp, + version: version, + manifestArtifact: manifestArtifact, + indexArtifact: indexArtifact, + manifest: m, + state: state, + } + + if indexArtifact != nil { + // index based manifests are optional and only read based for next gen support + container.SetReadOnly() + } + + return container, nil } func (c *ComponentVersionContainer) SetBridge(impl repocpi.ComponentVersionAccessBridge) { @@ -89,11 +132,20 @@ func (c *ComponentVersionContainer) GetParentBridge() repocpi.ComponentAccessBri } func (c *ComponentVersionContainer) Close() error { - if c.manifest == nil { + if c.manifest == nil && c.manifestArtifact == nil { return accessio.ErrClosed } c.manifest = nil - return c.access.Close() + c.index = nil + + var err error + if c.indexArtifact != nil { + err = errors.Join(err, c.indexArtifact.Close()) + } + if c.manifestArtifact != nil { + err = errors.Join(err, c.manifestArtifact.Close()) + } + return err } func (c *ComponentVersionContainer) SetReadOnly() { @@ -140,9 +192,9 @@ func (c *ComponentVersionContainer) AccessMethod(a cpi.AccessSpec, cv refmgmt.Ex switch a.GetKind() { case localblob.Type: - return newLocalBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.access, cv) + return newLocalBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.manifestArtifact, cv) case localociblob.Type: - return newLocalOCIBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.access, cv) + return newLocalOCIBlobAccessMethod(accessSpec.(*localblob.AccessSpec), c.comp.namespace, c.manifestArtifact, cv) case relativeociref.Type: m, err := ociartifact.NewMethod(c.GetContext(), a, accessSpec.(*relativeociref.AccessSpec).Reference, c.comp.repo.ocirepo) if err == nil { diff --git a/api/ocm/extensions/repositories/genericocireg/info.go b/api/ocm/extensions/repositories/genericocireg/info.go index 73fdadf242..6048fc64c8 100644 --- a/api/ocm/extensions/repositories/genericocireg/info.go +++ b/api/ocm/extensions/repositories/genericocireg/info.go @@ -28,7 +28,7 @@ func (h handler) Info(m cpi.ManifestAccess, config []byte) interface{} { info := &ComponentVersionInfo{ Description: "component version", } - acc := NewStateAccess(m) + acc := NewStateAccess(nil, m) data, err := blobaccess.BlobData(acc.Get()) if err != nil { info.Error = "cannot read component descriptor: " + err.Error() @@ -46,7 +46,7 @@ func (h handler) Info(m cpi.ManifestAccess, config []byte) interface{} { func (h handler) Description(pr common.Printer, m cpi.ManifestAccess, config []byte) { pr.Printf("component version:\n") - acc := NewStateAccess(m) + acc := NewStateAccess(nil, m) data, err := blobaccess.BlobData(acc.Get()) if err != nil { pr.Printf(" cannot read component descriptor: %s\n", err.Error()) diff --git a/api/ocm/extensions/repositories/genericocireg/state.go b/api/ocm/extensions/repositories/genericocireg/state.go index 834a768016..8a6db97aa4 100644 --- a/api/ocm/extensions/repositories/genericocireg/state.go +++ b/api/ocm/extensions/repositories/genericocireg/state.go @@ -27,12 +27,13 @@ import ( ocmlog "ocm.software/ocm/api/utils/logging" ) -func NewState(mode accessobj.AccessMode, name, version string, access oci.ManifestAccess, compat ...bool) (accessobj.State, error) { - return accessobj.NewState(mode, NewStateAccess(access, compat...), NewStateHandler(name, version)) +func NewState(mode accessobj.AccessMode, name, version string, parentIndex oci.IndexAccess, access oci.ManifestAccess, compat ...bool) (accessobj.State, error) { + return accessobj.NewState(mode, NewStateAccess(parentIndex, access, compat...), NewStateHandler(name, version)) } // StateAccess handles the component descriptor persistence in an OCI Manifest. type StateAccess struct { + parent oci.IndexAccess access oci.ManifestAccess layerMedia string compat bool @@ -40,10 +41,11 @@ type StateAccess struct { var _ accessobj.StateAccess = (*StateAccess)(nil) -func NewStateAccess(access oci.ManifestAccess, compat ...bool) accessobj.StateAccess { +func NewStateAccess(parentIndex oci.IndexAccess, access oci.ManifestAccess, compat ...bool) accessobj.StateAccess { return &StateAccess{ compat: utils.Optional(compat...), access: access, + parent: parentIndex, } } @@ -226,7 +228,7 @@ type ComponentDescriptorConfig struct { ComponentDescriptorLayer *ociv1.Descriptor `json:"componentDescriptorLayer,omitempty"` } -//////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////// // StateHandler handles the encoding of a component descriptor. type StateHandler struct { diff --git a/cmds/ocm/commands/ocicmds/artifacts/transfer/cmd_test.go b/cmds/ocm/commands/ocicmds/artifacts/transfer/cmd_test.go index 0ca832ed7a..ad97bec678 100644 --- a/cmds/ocm/commands/ocicmds/artifacts/transfer/cmd_test.go +++ b/cmds/ocm/commands/ocicmds/artifacts/transfer/cmd_test.go @@ -53,7 +53,7 @@ var _ = Describe("Test Environment", func() { copying /tmp/ctf//mandelsoft/test:v1 to directory::` + OUT + `//mandelsoft/test:v1... copied 1 from 1 artifact(s) and 1 repositories `)) - Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}]}"))) + Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\"}]}"))) }) It("transfers a named artifact to changed repository", func() { @@ -77,7 +77,7 @@ copied 1 from 1 artifact(s) and 1 repositories copying /tmp/ctf//mandelsoft/test:v1 to directory::` + OUT + `//changed:v1... copied 1 from 1 artifact(s) and 1 repositories `)) - Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"changed\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}]}"))) + Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"changed\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\"}]}"))) }) It("transfers a named artifact to sub repository", func() { @@ -101,7 +101,7 @@ copied 1 from 1 artifact(s) and 1 repositories copying /tmp/ctf//mandelsoft/test:v1 to directory::` + OUT + `//sub/mandelsoft/test:v1... copied 1 from 1 artifact(s) and 1 repositories `)) - Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"sub/mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}]}"))) + Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"sub/mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\"}]}"))) }) It("transfers an unnamed artifact set", func() { @@ -123,6 +123,6 @@ copied 1 from 1 artifact(s) and 1 repositories copying ArtifactSet::/tmp/ctf//:v1 to directory::` + OUT + `//mandelsoft/test:v1... copied 1 from 1 artifact(s) and 1 repositories `)) - Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\"}]}"))) + Expect(env.ReadFile(OUT + "/" + ctf.ArtifactIndexFileName)).To(Equal([]byte("{\"schemaVersion\":1,\"artifacts\":[{\"repository\":\"mandelsoft/test\",\"tag\":\"v1\",\"digest\":\"sha256:2c3e2c59e0ac9c99864bf0a9f9727c09f21a66080f9f9b03b36a2dad3cce6ff9\",\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\"}]}"))) }) }) diff --git a/cmds/ocm/commands/ocicmds/artifacts/transfer/index_transfer_test.go b/cmds/ocm/commands/ocicmds/artifacts/transfer/index_transfer_test.go new file mode 100644 index 0000000000..ef054c2a53 --- /dev/null +++ b/cmds/ocm/commands/ocicmds/artifacts/transfer/index_transfer_test.go @@ -0,0 +1,320 @@ +package transfer_test + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/mandelsoft/vfs/pkg/vfs" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/registry" + "golang.org/x/crypto/bcrypt" + "ocm.software/ocm/api/utils/tarutils" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "ocm.software/ocm/cmds/ocm/testhelper" + + "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + "ocm.software/ocm/api/utils/accessobj" +) + +const distributionRegistryImage = "registry:3.0.0" + +type TestLogger struct{} + +func (t TestLogger) Printf(format string, v ...interface{}) { + GinkgoWriter.Printf(format+"\n", v...) +} + +func StartDockerContainerRegistry(t FullGinkgoTInterface, container, htpasswd string) string { + t.Helper() + // Start containerized registry + t.Logf("Launching test registry (%s)...", distributionRegistryImage) + registryContainer, err := registry.Run(context.Background(), distributionRegistryImage, + WithHtpasswd(htpasswd), + testcontainers.WithEnv(map[string]string{ + "REGISTRY_VALIDATION_DISABLED": "true", + "REGISTRY_LOG_LEVEL": "debug", + }), + testcontainers.WithLogger(&TestLogger{}), + testcontainers.WithName(container), + ) + r := require.New(t) + r.NoError(err) + t.Cleanup(func() { + r.NoError(testcontainers.TerminateContainer(registryContainer)) + }) + + t.Logf("Test registry started") + + registryAddress, err := registryContainer.HostAddress(context.Background()) + r.NoError(err) + + return registryAddress +} + +func WithHtpasswd(credentials string) testcontainers.CustomizeRequestOption { + return func(req *testcontainers.GenericContainerRequest) error { + tmpFile, err := os.CreateTemp("", "htpasswd") + if err != nil { + tmpFile, err = os.Create(".") + if err != nil { + return fmt.Errorf("cannot create the file in the temp dir or in the current dir: %w", err) + } + } + defer tmpFile.Close() + + _, err = tmpFile.WriteString(credentials) + if err != nil { + return fmt.Errorf("cannot write the credentials to the file: %w", err) + } + + return registry.WithHtpasswdFile(tmpFile.Name())(req) + } +} + +func GenerateHtpasswd(t FullGinkgoTInterface, username, password string) string { + t.Helper() + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + require.NoError(t, err) + return fmt.Sprintf("%s:%s", username, hashedPassword) +} + +var _ = Describe("Index Transfer", func() { + var env *TestEnv + var registryAddress string + user := "admin" + pass := "password" + + BeforeEach(func() { + env = NewTestEnv() + // Start Local Registry + registryAddress = StartDockerContainerRegistry(GinkgoT(), "ocm-index-transfer-test-registry-"+time.Now().Format("20060102150405"), GenerateHtpasswd(GinkgoT(), user, pass)) + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("transfers an OCI Index artifact as a local blob", func() { + ctx := context.Background() + r := require.New(GinkgoT()) + tempDir := env.FSTempDir() + + user := "admin" + pass := "password" + + var ( + componentName = "ocm.software/test-component" + componentVersion = "v1.0.0" + resourceName = "cli-image" + ) + + By("pulling the OCI index image from GHCR and pushing it to the local registry") + srcRef := "ghcr.io/open-component-model/cli:main" + dstRef := fmt.Sprintf("%s/ocm-cli:main", registryAddress) + + repoSrc, err := remote.NewRepository(srcRef) + r.NoError(err) + + desc, err := oras.Resolve(ctx, repoSrc, srcRef, oras.ResolveOptions{}) + r.NoError(err) + + repoSrc.Reference.Reference = desc.Digest.String() + srcRef = repoSrc.Reference.String() + + repoDst, err := remote.NewRepository(dstRef) + r.NoError(err) + + repoDst.Client = &auth.Client{ + Client: retry.DefaultClient, + Credential: auth.StaticCredential(registryAddress, auth.Credential{ + Username: user, + Password: pass, + }), + } + repoDst.PlainHTTP = true + + _, err = oras.Copy( + ctx, + repoSrc, repoSrc.Reference.Reference, + repoDst, repoDst.Reference.Reference, + oras.CopyOptions{}, + ) + r.NoError(err, "failed to copy CLI image to local registry") + + By("writing the component constructor definition") + constructorContent := fmt.Sprintf(` +components: +- name: %s + version: %s + provider: + name: ocm.software + resources: + - name: %s + version: %s + type: ociImage + relation: external + copyPolicy: byValue + access: + type: ociArtifact + imageReference: "%s" +`, + componentName, + componentVersion, + resourceName, + componentVersion, + "http://"+dstRef, + ) + + constructorPath := filepath.Join(tempDir, "constructor.yaml") + r.NoError(os.WriteFile(constructorPath, []byte(constructorContent), os.ModePerm)) + + By("writing OCM credential configuration for the local registry") + host, port, err := net.SplitHostPort(registryAddress) + r.NoError(err) + + ocmConfigContent := fmt.Sprintf(` +type: generic.config.ocm.software/v1 +configurations: +- type: credentials.config.ocm.software + consumers: + - identity: + type: OCIRepository + hostname: %q + port: %q + scheme: http + credentials: + - type: Credentials/v1 + properties: + username: %s + password: %s +`, host, port, user, pass) + + configPath := filepath.Join(tempDir, "ocmconfig.yaml") + r.NoError(os.WriteFile(configPath, []byte(strings.TrimSpace(ocmConfigContent)), os.ModePerm)) + + By("adding the component version to the registry using the OCM CLI") + addArgs := []string{ + "run", "--rm", + "--network", "host", + "-v", fmt.Sprintf("%s:/work", tempDir), + "ghcr.io/open-component-model/cli:main", + "add", "component-version", + "--repository", "http://" + registryAddress, + "--config", "/work/ocmconfig.yaml", + "--constructor", "/work/constructor.yaml", + "--loglevel", "debug", + } + + cmd := exec.Command("docker", addArgs...) + out, err := cmd.CombinedOutput() + r.NoError(err, string(out)) + + By("verifying the component version exists in the registry") + getArgs := []string{ + "run", "--rm", + "--network", "host", + "-v", fmt.Sprintf("%s:/work", tempDir), + "ghcr.io/open-component-model/cli:main", + "get", "component-version", + "--config", "/work/ocmconfig.yaml", + "-oyaml", + fmt.Sprintf("http://%s//%s:%s", registryAddress, componentName, componentVersion), + } + + cmd = exec.Command("docker", getArgs...) + out, err = cmd.CombinedOutput() + r.NoError(err, string(out)) + + credArgs := []string{ + "--cred", ":type=OCIRegistry", + "--cred", ":hostname=" + host, + "--cred", ":port=" + port, + "--cred", ":scheme=http", + "--cred", "username=" + user, + "--cred", "password=" + pass, + } + + By("ensuring the resource is accessible via the OCM CLI") + Expect(env.Execute(append(credArgs, + "get", "resource", + "--repo", "http://"+registryAddress, + "--lookup", "http://"+registryAddress, + "-ojson", + componentName+":"+componentVersion, resourceName, + )...)).To(Succeed()) + + By("downloading the resource to verify on-demand synthesis") + Expect(env.Execute(append(credArgs, + "download", "resource", + "--repo", "http://"+registryAddress, + "--lookup", "http://"+registryAddress, + componentName+":"+componentVersion, resourceName, + "--outfile", filepath.Join(tempDir, "downloaded-cli"), + )...)).To(Succeed()) + + By("transferring the component version to a CTF with localBlob synthesis") + targetCTF := filepath.Join(tempDir, "target-ctf") + Expect(env.Execute(append(credArgs, + "transfer", "componentversions", + componentName+":"+componentVersion, + targetCTF, + "--repo", "http://"+registryAddress, + "--lookup", "http://"+registryAddress, + "--copy-resources", + )...)).To(Succeed()) + + By("verifying the resource is stored as a localBlob in the CTF") + repo, err := ctf.Open(env, accessobj.ACC_READONLY, targetCTF, 0o700, ctf.FormatDirectory) + r.NoError(err) + defer repo.Close() + + cv, err := repo.LookupComponentVersion(componentName, componentVersion) + r.NoError(err) + defer cv.Close() + + var res cpi.ResourceAccess + for _, rsc := range cv.GetResources() { + if rsc.Meta().Name == resourceName { + res = rsc + break + } + } + r.NotNil(res) + + meth, err := res.AccessMethod() + r.NoError(err) + defer meth.Close() + + r.Equal("localBlob", meth.GetKind()) + + By("extracting and resolving the OCI layout from the localBlob") + reader, err := meth.Reader() + r.NoError(err) + defer reader.Close() + + tempfs, err := tarutils.ExtractTgzToTempFs(reader) + r.NoError(err) + + store, err := oci.NewFromFS(ctx, vfs.AsIoFS(tempfs)) + r.NoError(err) + + desc, err = store.Resolve(ctx, "latest") + r.NoError(err) + r.NotNil(desc) + }) +}) diff --git a/go.mod b/go.mod index 49265b5467..ebe156d1e1 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( github.com/ulikunitz/xz v0.5.15 github.com/xeipuuv/gojsonschema v1.2.0 go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/net v0.48.0 @@ -532,7 +533,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp/typeparams v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/mod v0.30.0 // indirect golang.org/x/sync v0.19.0 // indirect