feat: support index based reading of OCI artifacts

api/oci/extensions/repositories/artifactset/artifactset.go

@@ -1,6 +1,8 @@
 package artifactset
 
 import (
+	"maps"
+	"slices"
 	"strings"
 
 	"github.com/mandelsoft/goutils/errors"
@@ -203,27 +205,106 @@ func (a *namespaceContainer) AddTags(digest digest.Digest, tags ...string) error
 	a.base.Lock()
 	defer a.base.Unlock()
 
-	idx := a.GetIndex()
-	for i, e := range idx.Manifests {
-		if e.Digest == digest {
-			if e.Annotations == nil {
-				e.Annotations = map[string]string{}
-				idx.Manifests[i].Annotations = e.Annotations
+	index := a.GetIndex()
+
+	manifests := index.Manifests
+
+	artifacts := decodeIndexManifests(manifests)
+
+	art := artifacts[digest]
+	if art == nil {
+		return errors.ErrUnknown(cpi.KIND_OCIARTIFACT, digest.String())
+	}
+
+	for _, tag := range tags {
+		art.addTag(tag)
+	}
+
+	isOCI := a.base.
+		FileSystemBlobAccess.
+		Access().
+		GetInfo().
+		GetDescriptorFileName() == OCIArtifactSetDescriptorFileName
+
+	index.Manifests = encodeIndexManifests(artifacts, isOCI)
+
+	return nil
+}
+
+type descriptorWithTags struct {
+	cpi.Descriptor
+	Tags map[string]struct{}
+}
+
+func (d *descriptorWithTags) addTag(tag string) {
+	if tag := strings.TrimSpace(tag); tag != "" {
+		d.Tags[tag] = struct{}{}
+	}
+}
+
+func decodeIndexManifests(manifests []cpi.Descriptor) map[digest.Digest]*descriptorWithTags {
+	out := map[digest.Digest]*descriptorWithTags{}
+	for _, m := range manifests {
+		if out[m.Digest] == nil {
+			out[m.Digest] = &descriptorWithTags{
+				Descriptor: normalizeDecodedDescriptorWithoutTags(m),
+				Tags:       map[string]struct{}{},
 			}
-			cur := RetrieveTags(e.Annotations)
-			if cur != "" {
-				cur = strings.Join(append([]string{cur}, tags...), ",")
-			} else {
-				cur = strings.Join(tags, ",")
+		}
+		annotated := out[m.Digest]
+
+		// OCM multi-tag annotation
+		tagsFromAnnotations := strings.Split(RetrieveTags(m.Annotations), ",")
+		for _, tag := range tagsFromAnnotations {
+			annotated.addTag(tag)
+		}
+
+		// OCI single-tag annotation
+		if tag, ok := m.Annotations[OCITAG_ANNOTATION]; ok {
+			annotated.addTag(tag)
+		}
+	}
+
+	return out
+}
+
+func normalizeDecodedDescriptorWithoutTags(d cpi.Descriptor) cpi.Descriptor {
+	d.Annotations = maps.Clone(d.Annotations)
+	delete(d.Annotations, TAGS_ANNOTATION)
+	delete(d.Annotations, OCITAG_ANNOTATION)
+	if len(d.Annotations) == 0 {
+		d.Annotations = nil
+	}
+	return d
+}
+
+func encodeIndexManifests(
+	descriptors map[digest.Digest]*descriptorWithTags,
+	oci bool,
+) []cpi.Descriptor {
+	var manifests []cpi.Descriptor
+	for _, desc := range descriptors {
+		tags := slices.Sorted(maps.Keys(desc.Tags))
+		joined := strings.Join(tags, ",")
+		if !oci {
+			d := desc.Descriptor
+			d.Annotations = map[string]string{
+				TAGS_ANNOTATION: joined,
 			}
-			e.Annotations[TAGS_ANNOTATION] = cur
-			if a.base.FileSystemBlobAccess.Access().GetInfo().GetDescriptorFileName() == OCIArtifactSetDescriptorFileName {
-				e.Annotations[OCITAG_ANNOTATION] = tags[0]
+			manifests = append(manifests, d)
+			continue
+		}
+		for _, t := range tags {
+			d := desc.Descriptor
+			d.Annotations = map[string]string{
+				TAGS_ANNOTATION:   joined,
+				OCITAG_ANNOTATION: t,
 			}
-			return nil
+			manifests = append(manifests, d)
 		}
 	}
-	return errors.ErrUnknown(cpi.KIND_OCIARTIFACT, digest.String())
+
+	return manifests
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -437,14 +518,24 @@ func (a *namespaceContainer) AddPlatformArtifact(artifact cpi.Artifact, platform
 		return nil, err
 	}
 
-	idx.Manifests = append(idx.Manifests, cpi.Descriptor{
-		MediaType:   blob.MimeType(),
+	desc := cpi.Descriptor{
 		Digest:      blob.Digest(),
 		Size:        blob.Size(),
 		URLs:        nil,
 		Annotations: nil,
 		Platform:    platform,
-	})
+	}
+
+	isOCI := a.base.FileSystemBlobAccess.
+		Access().
+		GetInfo().
+		GetDescriptorFileName() == OCIArtifactSetDescriptorFileName
+
+	if isOCI {
+		desc.MediaType = blob.MimeType()
+	}
+
+	idx.Manifests = append(idx.Manifests, desc)
 	return blob, nil
 }
 

api/oci/extensions/repositories/artifactset/format.go

@@ -225,12 +225,13 @@ func OpenFromDataAccess(acc accessobj.AccessMode, mediatype string, data blobacc
 	}
 	defer reader.Close()
 	o.SetReader(reader)
-	fmt := accessio.FormatTar
-
-	if mime.IsGZip(mediatype) {
-		fmt = accessio.FormatTGZ
+	if o.GetFileFormat() == nil {
+		fmt := accessio.FormatTar
+		if mime.IsGZip(mediatype) {
+			fmt = accessio.FormatTGZ
+		}
+		o.SetFileFormat(fmt)
 	}
-	o.SetFileFormat(fmt)
 	return Open(acc&accessobj.ACC_READONLY, "", 0, o)
 }
 

api/oci/extensions/repositories/artifactset/utils_synthesis.go

@@ -84,18 +84,24 @@ func SynthesizeArtifactBlobWithFilter(ns cpi.NamespaceAccess, ref string, filter
 			return nil, err
 		}
 	}
-	return SynthesizeArtifactBlobForArtifact(art, ref, filter)
+	return SynthesizeArtifactBlobForArtifact(art, []string{ref}, filter)
 }
 
-func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, ref string, filter ...filters.Filter) (ArtifactBlob, error) {
+func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, refs []string, filter ...filters.Filter) (ArtifactBlob, error) {
 	blob, err := art.Blob()
 	if err != nil {
 		return nil, err
 	}
 
-	vers, err := ociutils.ParseVersion(ref)
-	if err != nil {
-		return nil, err
+	tags := make([]string, 0, len(refs))
+	for _, ref := range refs {
+		vers, err := ociutils.ParseVersion(ref)
+		if err != nil {
+			return nil, err
+		}
+		if vers.IsTagged() {
+			tags = append(tags, vers.GetTag())
+		}
 	}
 
 	return SythesizeArtifactSet(func(set *ArtifactSet) (string, error) {
@@ -104,9 +110,8 @@ func SynthesizeArtifactBlobForArtifact(art cpi.ArtifactAccess, ref string, filte
 			return "", fmt.Errorf("failed to transfer artifact: %w", err)
 		}
 
-		if ok := vers.IsTagged(); ok {
-			err = set.AddTags(*dig, vers.GetTag())
-			if err != nil {
+		if len(tags) > 0 {
+			if err := set.AddTags(*dig, tags...); err != nil {
 				return "", fmt.Errorf("failed to add tag: %w", err)
 			}
 		}

api/oci/extensions/repositories/ctf/index/ctfindex.go

@@ -18,6 +18,7 @@ type ArtifactMeta struct {
 	Repository string        `json:"repository"`
 	Tag        string        `json:"tag,omitempty"`
 	Digest     digest.Digest `json:"digest,omitempty"`
+	MediaType  string        `json:"mediaType,omitempty"`
 }
 
 func Decode(data []byte) (*ArtifactIndex, error) {

api/oci/extensions/repositories/ctf/index/index.go

@@ -241,6 +241,7 @@ func (r *RepositoryIndex) GetDescriptor() *ArtifactIndex {
 					Repository: vers.Repository,
 					Tag:        vers.Tag,
 					Digest:     vers.Digest,
+					MediaType:  vers.MediaType,
 				}
 				index.Index = append(index.Index, *d)
 			}

api/oci/extensions/repositories/ctf/namespace.go

@@ -40,10 +40,6 @@ func (n *namespaceContainer) Close() error {
 	return nil
 }
 
-func (n *namespaceContainer) GetBlobDescriptor(digest digest.Digest) *cpi.Descriptor {
-	return nil
-}
-
 func (n *namespaceContainer) ListTags() ([]string, error) {
 	return n.repo.getIndex().GetTags(n.impl.GetNamespace()), nil // return digests as tags, also
 }
@@ -84,6 +80,7 @@ func (n *namespaceContainer) AddArtifact(artifact cpi.Artifact, tags ...string)
 		Repository: n.impl.GetNamespace(),
 		Tag:        "",
 		Digest:     blob.Digest(),
+		MediaType:  blob.MimeType(),
 	})
 	return blob, n.AddTags(blob.Digest(), tags...)
 }

api/ocm/extensions/accessmethods/ociartifact/method.go

@@ -35,6 +35,9 @@ const (
 const (
 	LegacyType   = "ociRegistry"
 	LegacyTypeV1 = LegacyType + runtime.VersionSeparator + "v1"
+
+	LegacyType2   = "OCIImage"
+	LegacyType2V1 = LegacyType2 + runtime.VersionSeparator + "v1"
 )
 
 func init() {
@@ -43,6 +46,9 @@ func init() {
 
 	accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType))
 	accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyTypeV1))
+
+	accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2))
+	accspeccpi.RegisterAccessType(accspeccpi.NewAccessSpecType[*AccessSpec](LegacyType2V1))
 }
 
 func Is(spec accspeccpi.AccessSpec) bool {
@@ -355,7 +361,7 @@ func (m *accessMethod) getBlob() (artifactset.ArtifactBlob, error) {
 	}
 	logger := Logger(WrapContextProvider(m.ctx))
 	logger.Info("synthesize artifact blob", "ref", m.reference)
-	m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, m.ref.VersionSpec())
+	m.blob, err = artifactset.SynthesizeArtifactBlobForArtifact(m.art, []string{m.ref.VersionSpec()})
 	logger.Info("synthesize artifact blob done", "ref", m.reference, "error", logging.ErrorMessage(err))
 	if err != nil {
 		m.err = err

api/ocm/extensions/repositories/genericocireg/accessmethod_localblob.go

@@ -2,6 +2,7 @@ package genericocireg
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"strings"
@@ -13,6 +14,7 @@ import (
 
 	"ocm.software/ocm/api/oci"
 	"ocm.software/ocm/api/oci/artdesc"
+	"ocm.software/ocm/api/oci/extensions/repositories/artifactset"
 	"ocm.software/ocm/api/ocm/cpi/accspeccpi"
 	"ocm.software/ocm/api/ocm/extensions/accessmethods/localblob"
 	"ocm.software/ocm/api/utils/blobaccess/blobaccess"
@@ -26,6 +28,7 @@ type localBlobAccessMethod struct {
 	spec      *localblob.AccessSpec
 	namespace oci.NamespaceAccess
 	artifact  oci.ArtifactAccess
+	mimeType  string
 }
 
 var _ accspeccpi.AccessMethodImpl = (*localBlobAccessMethod)(nil)
@@ -40,6 +43,11 @@ func newLocalBlobAccessMethodImpl(a *localblob.AccessSpec, ns oci.NamespaceAcces
 		namespace: ns,
 		artifact:  art,
 	}
+	if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest {
+		// if we discover a localblob with an index or manifest media type, we can
+		// assume that we are dealing with a new style of artifact created by the new reference library.
+		m.mimeType = artifactset.MediaType(m.spec.MediaType)
+	}
 	ref.BeforeCleanup(refmgmt.CleanupHandlerFunc(m.cache))
 	return m, nil
 }
@@ -99,8 +107,35 @@ func (m *localBlobAccessMethod) getBlob() (blobaccess.DataAccess, error) {
 		err  error
 	)
 	if len(refs) < 2 {
-		_, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference))
-		if err != nil {
+		if m.spec.MediaType == artdesc.MediaTypeImageIndex || m.spec.MediaType == artdesc.MediaTypeImageManifest {
+			// if we have a nested manifest or index, we can use the blob synthesis utility here to download
+			// the entire artifact set.
+			art, err := m.namespace.GetArtifact(m.spec.LocalReference)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get artifact for local reference %q: %w", m.spec.LocalReference, err)
+			}
+			defer art.Close()
+			var artifactRefs []string
+			if m.spec.ReferenceName != "" {
+				// if we have a reference name, it consists of repository and tag
+				// so we can extract the tag to use it as target, instead of latest
+				refSpec, err := oci.ParseRef(m.spec.ReferenceName)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse reference name %q: %w", m.spec.ReferenceName, err)
+				}
+				if refSpec.GetTag() != "" {
+					artifactRefs = append(artifactRefs, refSpec.GetTag())
+				}
+			}
+			localReferenceDigest := digest.Digest(m.spec.LocalReference)
+			artifactRefs = append(artifactRefs, fmt.Sprintf("%s-%s", localReferenceDigest.Algorithm(), localReferenceDigest.Encoded()))
+			artifactRefs = append(artifactRefs, "latest")
+			artblob, err := artifactset.SynthesizeArtifactBlobForArtifact(art, artifactRefs)
+			if err != nil {
+				return nil, fmt.Errorf("failed to synthesize artifact blob: %w", err)
+			}
+			data = artblob
+		} else if _, data, err = m.namespace.GetBlobData(digest.Digest(m.spec.LocalReference)); err != nil {
 			return nil, err
 		}
 	} else {
@@ -123,10 +158,13 @@ func (m *localBlobAccessMethod) Get() ([]byte, error) {
 }
 
 func (m *localBlobAccessMethod) MimeType() string {
+	if m.mimeType != "" {
+		return m.mimeType
+	}
 	return m.spec.MediaType
 }
 
-////////////////////////////////////////////////////////////////////////////////
+// //////////////////////////////////////////////////////////////////////////////
 
 type composedBlock struct {
 	m    *localBlobAccessMethod