diff --git a/api/oci/extensions/repositories/artifactset/artifactset_test.go b/api/oci/extensions/repositories/artifactset/artifactset_test.go index 9b55623838..0deacc4e2b 100644 --- a/api/oci/extensions/repositories/artifactset/artifactset_test.go +++ b/api/oci/extensions/repositories/artifactset/artifactset_test.go @@ -81,10 +81,26 @@ var _ = Describe("artifact management", func() { for _, fi := range infos { blobs = append(blobs, fi.Name()) } - Expect(blobs).To(ContainElements( - "sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", - "sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50")) + if format == artifactset.FORMAT_OCI { + // OCI format uses nested directory structure: blobs/sha256/DIGEST + Expect(blobs).To(ContainElement("sha256")) + subInfos, err := vfs.ReadDir(tempfs, "test/"+artifactset.BlobsDirectoryName+"/sha256") + Expect(err).To(Succeed()) + subBlobs := []string{} + for _, fi := range subInfos { + subBlobs = append(subBlobs, fi.Name()) + } + Expect(subBlobs).To(ContainElements( + "3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50")) + } else { + // OCM format uses flat structure: blobs/sha256.DIGEST + Expect(blobs).To(ContainElements( + "sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50")) + } }) TestForAllFormats("instantiate tgz artifact", func(format string) { @@ -108,6 +124,7 @@ var _ = Describe("artifact management", func() { tr := tar.NewReader(zip) files := []string{} + dirs := []string{} for { header, err := tr.Next() if err != nil { @@ -119,19 +136,33 @@ var _ = Describe("artifact management", func() { switch header.Typeflag { case tar.TypeDir: - Expect(header.Name).To(Equal(artifactset.BlobsDirectoryName)) + dirs = append(dirs, header.Name) case tar.TypeReg: files = append(files, header.Name) } } + + // Check directories + Expect(dirs).To(ContainElement(artifactset.BlobsDirectoryName)) + if format == artifactset.FORMAT_OCI { + Expect(dirs).To(ContainElement("blobs/sha256")) + } + + // Check files based on format elems := []interface{}{ artifactset.DescriptorFileName(format), - "blobs/sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", - "blobs/sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "blobs/sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50", } if format == artifactset.FORMAT_OCI { elems = append(elems, artifactset.OCILayouFileName) + elems = append(elems, + "blobs/sha256/3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "blobs/sha256/810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50") + } else { + elems = append(elems, + "blobs/sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "blobs/sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "blobs/sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50") } Expect(files).To(ContainElements(elems)) }) @@ -160,6 +191,7 @@ var _ = Describe("artifact management", func() { tr := tar.NewReader(zip) files := []string{} + dirs := []string{} for { header, err := tr.Next() if err != nil { @@ -171,19 +203,33 @@ var _ = Describe("artifact management", func() { switch header.Typeflag { case tar.TypeDir: - Expect(header.Name).To(Equal(artifactset.BlobsDirectoryName)) + dirs = append(dirs, header.Name) case tar.TypeReg: files = append(files, header.Name) } } + + // Check directories + Expect(dirs).To(ContainElement(artifactset.BlobsDirectoryName)) + if format == artifactset.FORMAT_OCI { + Expect(dirs).To(ContainElement("blobs/sha256")) + } + + // Check files based on format elems := []interface{}{ artifactset.DescriptorFileName(format), - "blobs/sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", - "blobs/sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "blobs/sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50", } if format == artifactset.FORMAT_OCI { elems = append(elems, artifactset.OCILayouFileName) + elems = append(elems, + "blobs/sha256/3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "blobs/sha256/810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50") + } else { + elems = append(elems, + "blobs/sha256.3d05e105e350edf5be64fe356f4906dd3f9bf442a279e4142db9879bba8e677a", + "blobs/sha256.44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "blobs/sha256.810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50") } Expect(files).To(ContainElements(elems)) }) diff --git a/api/oci/extensions/repositories/artifactset/ctf_roundtrip_test.go b/api/oci/extensions/repositories/artifactset/ctf_roundtrip_test.go new file mode 100644 index 0000000000..f8aafd44cf --- /dev/null +++ b/api/oci/extensions/repositories/artifactset/ctf_roundtrip_test.go @@ -0,0 +1,193 @@ +//go:build integration + +package artifactset_test + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mandelsoft/vfs/pkg/osfs" + "oras.land/oras-go/v2/content/oci" + + envhelper "ocm.software/ocm/api/helper/env" + . "ocm.software/ocm/cmds/ocm/testhelper" +) + +const ( + componentName = "example.com/hello" + componentVersion = "1.0.0" + resourceName = "hello-image" + resourceVersion = "1.0.0" + imageReference = "hello-world:linux" +) + +// This test verifies the CTF-based workflow with --oci-layout flag: +// 1. Create a CTF archive with an OCI image resource +// 2. Transfer CTF to new CTF with --copy-resources +// 3. Verify components and resources in target CTF +// 4. Download resource with --oci-layout flag: +// - Creates OCI Image Layout directory (index.json, oci-layout, blobs/sha256/...) +// - Verifies layout structure is OCI-compliant +// - Resolves artifact by resource version using ORAS +// 5. Download resource without --oci-layout: +// - Creates OCM artifact set format (not OCI-compliant) +// - Verifies layout structure check fails +var _ = Describe("CTF to CTF-with-resource to OCI roundtrip", Ordered, func() { + var ( + tempDir string + sourceCTF string + targetCTF string + resourcesOciDir string + resourcesOcmDir string + env *TestEnv + ) + + BeforeAll(func() { + + var err error + tempDir, err = os.MkdirTemp("", "ocm-ctf-oci-layout-*") + Expect(err).To(Succeed()) + + env = NewTestEnv(envhelper.FileSystem(osfs.New())) + }) + + AfterAll(func() { + if env != nil { + env.Cleanup() + } + }) + + It("creates CTF using stable OCM release", func() { + sourceCTF = filepath.Join(tempDir, "ctf-source") + constructorFile := filepath.Join(tempDir, "component-constructor.yaml") + constructorContent := `components: + - name: ` + componentName + ` + version: ` + componentVersion + ` + provider: + name: example.com + resources: + - name: ` + resourceName + ` + type: ociImage + version: ` + resourceVersion + ` + relation: external + access: + type: ociArtifact + imageReference: ` + imageReference + ` +` + err := os.WriteFile(constructorFile, []byte(constructorContent), 0644) + Expect(err).To(Succeed(), "MUST create constructor file") + + // Create CTF directory + err = os.MkdirAll(sourceCTF, 0755) + Expect(err).To(Succeed(), "MUST create CTF directory") + + // Use the current OCM version to create the CTF + GinkgoWriter.Printf("Creating CTF using current OCM version\n") + + buf := bytes.NewBuffer(nil) + err = env.CatchOutput(buf).Execute( + "add", "componentversions", + "--create", + "--file", sourceCTF, + constructorFile, + ) + GinkgoWriter.Printf("OCM output: %s\n", buf.String()) + Expect(err).To(Succeed(), "OCM MUST create CTF: %s", buf.String()) + }) + + It("transfers CTF to new CTF with --copy-resources", func() { + targetCTF = filepath.Join(tempDir, "ctf-target") + buf := bytes.NewBuffer(nil) + GinkgoWriter.Printf(" #### transfer componentversions " + sourceCTF + " " + targetCTF + "--copy-resources") + Expect(env.CatchOutput(buf).Execute( + "transfer", "componentversions", sourceCTF, targetCTF, "--copy-resources")).To(Succeed()) + GinkgoWriter.Printf("Transfer output: %s\n", buf.String()) + }) + + It("verifies components and resources in target CTF", func() { + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute("get", "componentversions", targetCTF)).To(Succeed()) + GinkgoWriter.Printf("Components: %s\n", buf.String()) + Expect(buf.String()).To(ContainSubstring(componentName)) + + // List resources + buf.Reset() + Expect(env.CatchOutput(buf).Execute( + "get", "resources", + targetCTF+"//"+componentName+":"+componentVersion, + )).To(Succeed()) + GinkgoWriter.Printf("Resources: %s\n", buf.String()) + Expect(buf.String()).To(ContainSubstring(resourceName)) + }) + + It("downloads resource from target CTF with --oci-layout", func() { + resourcesOciDir = filepath.Join(tempDir, "resource-oci-layout") + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute( + "download", "resources", + "--oci-layout", + "-O", resourcesOciDir, + targetCTF+"//"+componentName+":"+componentVersion, + resourceName, + )).To(Succeed()) + Expect(verifyOCILayoutStructure(resourcesOciDir)).To(Succeed()) + + store, err := oci.New(resourcesOciDir) + Expect(err).To(Succeed(), "ORAS failed to open OCI layout: %w", err) + + srcDesc, err := store.Resolve(context.Background(), resourceVersion) + Expect(err).To(Succeed(), "resource MUST be OCI compliant") + GinkgoWriter.Printf("Successfully verified OCI layout with ORAS: digest=%s\n", srcDesc.Digest) + + }) + + It("downloads resource from target CTF without --oci-layout", func() { + resourcesOcmDir = filepath.Join(tempDir, "resource-ocm-layout") + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute( + "download", "resources", + "-O", resourcesOcmDir, + targetCTF+"//"+componentName+":"+componentVersion, + resourceName, + )).To(Succeed()) + Expect(verifyOCILayoutStructure(resourcesOcmDir)).ToNot(Succeed()) + GinkgoWriter.Printf("Resource download output: %s\n", buf.String()) + }) +}) + +// verifyOCILayoutStructure checks that the OCI layout has the expected structure. +// Returns an error if any required file or directory is missing. +func verifyOCILayoutStructure(ociDir string) error { + // Check oci-layout file exists + ociLayoutPath := filepath.Join(ociDir, "oci-layout") + if _, err := os.Stat(ociLayoutPath); err != nil { + return fmt.Errorf("oci-layout file MUST exist: %w", err) + } + + // Check index.json exists + indexPath := filepath.Join(ociDir, "index.json") + if _, err := os.Stat(indexPath); err != nil { + return fmt.Errorf("index.json MUST exist: %w", err) + } + + // Check blobs directory exists + blobsDir := filepath.Join(ociDir, "blobs") + info, err := os.Stat(blobsDir) + if err != nil { + return fmt.Errorf("blobs directory MUST exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("blobs MUST be a directory, got file") + } + + GinkgoWriter.Printf("OCI layout structure verified: oci-layout, index.json, blobs/\n") + return nil +} diff --git a/api/oci/extensions/repositories/artifactset/format.go b/api/oci/extensions/repositories/artifactset/format.go index fb752d95a3..70032ae74c 100644 --- a/api/oci/extensions/repositories/artifactset/format.go +++ b/api/oci/extensions/repositories/artifactset/format.go @@ -1,6 +1,8 @@ package artifactset import ( + "path/filepath" + "strings" "sync" "github.com/mandelsoft/goutils/errors" @@ -42,6 +44,7 @@ func DescriptorFileName(format string) string { type accessObjectInfo struct { accessobj.DefaultAccessObjectInfo + ociFormat bool } var _ accessobj.AccessObjectInfo = (*accessObjectInfo)(nil) @@ -61,7 +64,7 @@ func validateDescriptor(data []byte) error { func NewAccessObjectInfo(fmts ...string) accessobj.AccessObjectInfo { a := &accessObjectInfo{ - baseInfo, + DefaultAccessObjectInfo: baseInfo, } oci := IsOCIDefaultFormat() if len(fmts) > 0 { @@ -84,6 +87,19 @@ func NewAccessObjectInfo(fmts ...string) accessobj.AccessObjectInfo { func (a *accessObjectInfo) setOCI() { a.DescriptorFileName = OCIArtifactSetDescriptorFileName a.AdditionalFiles = []string{OCILayouFileName} + a.ociFormat = true +} + +// SubPath returns the path for a blob. For OCI format, converts "sha256.DIGEST" +// to "blobs/sha256/DIGEST" per OCI Image Layout Specification. +func (a *accessObjectInfo) SubPath(name string) string { + if a.ociFormat { + // Convert sha256.DIGEST to sha256/DIGEST for OCI compliance + if algo, dig, ok := strings.Cut(name, "."); ok { + return filepath.Join(a.ElementDirectoryName, algo, dig) + } + } + return filepath.Join(a.ElementDirectoryName, name) } func (a *accessObjectInfo) setOCM() { diff --git a/api/ocm/extensions/download/handlers/init.go b/api/ocm/extensions/download/handlers/init.go index 6f8614fa44..e36de1939d 100644 --- a/api/ocm/extensions/download/handlers/init.go +++ b/api/ocm/extensions/download/handlers/init.go @@ -6,5 +6,6 @@ import ( _ "ocm.software/ocm/api/ocm/extensions/download/handlers/dirtree" _ "ocm.software/ocm/api/ocm/extensions/download/handlers/executable" _ "ocm.software/ocm/api/ocm/extensions/download/handlers/helm" + _ "ocm.software/ocm/api/ocm/extensions/download/handlers/ocilayout" _ "ocm.software/ocm/api/ocm/extensions/download/handlers/ocirepo" ) diff --git a/api/ocm/extensions/download/handlers/ocilayout/handler.go b/api/ocm/extensions/download/handlers/ocilayout/handler.go new file mode 100644 index 0000000000..51d2dfd66f --- /dev/null +++ b/api/ocm/extensions/download/handlers/ocilayout/handler.go @@ -0,0 +1,125 @@ +package ocilayout + +import ( + "strings" + + "github.com/mandelsoft/goutils/errors" + "github.com/mandelsoft/goutils/finalizer" + "github.com/mandelsoft/vfs/pkg/vfs" + + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" + "ocm.software/ocm/api/ocm/cpi" + "ocm.software/ocm/api/ocm/extensions/download" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" + "ocm.software/ocm/api/utils/logging" + common "ocm.software/ocm/api/utils/misc" +) + +const PRIORITY = 200 + +type Handler struct{} + +func New() download.Handler { + return &Handler{} +} + +func (h *Handler) Download(p common.Printer, racc cpi.ResourceAccess, path string, fs vfs.FileSystem) (ok bool, _ string, err error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagation(&err) + + // Step 1: Get access method to read resource content + m, err := racc.AccessMethod() + if err != nil { + return false, "", err + } + finalize.Close(m) + + // Step 2: Check MIME type - only handle OCI artifacts (tar/tar+gzip) + if !isOCIArtifact(m.MimeType()) { + logging.Logger().Debug("skipping non-OCI artifact", "mime", m.MimeType()) + return false, "", nil + } + + if path == "" { + path = racc.Meta().GetName() + } + + // Step 3: Open resource blob as artifact set (contains OCI image) + src, err := artifactset.OpenFromDataAccess(accessobj.ACC_READONLY, m.MimeType(), m) + if err != nil { + return true, "", errors.Wrapf(err, "open artifact set") + } + finalize.Close(src) + + // Step 4: Get the main artifact from the set + art, err := src.GetArtifact(src.GetMain().String()) + if err != nil { + return true, "", errors.Wrapf(err, "get artifact") + } + finalize.Close(art) + + // Step 5: Create target directory with OCI format (index.json + oci-layout) + target, err := artifactset.Create(accessobj.ACC_CREATE, path, 0o755, + accessio.PathFileSystem(fs), + accessobj.FormatDirectory, + artifactset.StructureFormat(artifactset.FORMAT_OCI), + ) + if err != nil { + return true, "", errors.Wrapf(err, "create OCI layout") + } + + // Step 6: Transfer all manifests and blobs to target with hybrid tagging: + // - Original tags from source (e.g., "latest", "linux") + // - Resource version (e.g., "1.0.0") + tags := collectTags(src, racc.Meta().GetVersion()) + if err := artifactset.TransferArtifact(art, target, tags...); err != nil { + err = errors.Join(err, target.Close()) + return true, "", errors.Wrapf(err, "transfer artifact") + } + + if err := target.Close(); err != nil { + return true, "", errors.Wrapf(err, "close target") + } + + p.Printf("%s: downloaded to OCI layout\n", path) + return true, path, nil +} + +func isOCIArtifact(mime string) bool { + return artdesc.IsOCIMediaType(mime) && + (strings.HasSuffix(mime, "+tar") || strings.HasSuffix(mime, "+tar+gzip")) +} + +// collectTags returns a deduplicated list of tags combining: +// - Resource version FIRST (becomes org.opencontainers.image.ref.name for ORAS resolution) +// - Original tags from the source artifact set (preserves mutable refs like "latest") +func collectTags(src *artifactset.ArtifactSet, version string) []string { + seen := make(map[string]struct{}) + var tags []string + + // Add resource version first - it becomes the primary tag (org.opencontainers.image.ref.name) + if version != "" { + seen[version] = struct{}{} + tags = append(tags, version) + } + + // Add original tags from source index annotations + mainDigest := src.GetMain() + for _, m := range src.GetIndex().Manifests { + if m.Digest == mainDigest && m.Annotations != nil { + if tagStr := artifactset.RetrieveTags(m.Annotations); tagStr != "" { + for _, t := range strings.Split(tagStr, ",") { + t = strings.TrimSpace(t) + if _, ok := seen[t]; !ok && t != "" { + seen[t] = struct{}{} + tags = append(tags, t) + } + } + } + } + } + + return tags +} diff --git a/api/utils/accessobj/filesystemaccess.go b/api/utils/accessobj/filesystemaccess.go index a15f37b05b..ac9ee77b73 100644 --- a/api/utils/accessobj/filesystemaccess.go +++ b/api/utils/accessobj/filesystemaccess.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sync" "github.com/mandelsoft/vfs/pkg/vfs" @@ -140,6 +141,14 @@ func (a *FileSystemBlobAccess) AddBlob(blob blobaccess.BlobAccess) error { } defer r.Close() + + // Create parent directory if path uses nested structure (e.g., blobs/sha256/DIGEST vs blobs/sha256.DIGEST) + if dir := filepath.Dir(path); dir != a.base.GetInfo().GetElementDirectoryName() { + if err := a.base.GetFileSystem().MkdirAll(dir, a.base.GetMode()|0o111); err != nil { + return fmt.Errorf("unable to create directory for '%s': %w", path, err) + } + } + w, err := a.base.GetFileSystem().OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, a.base.GetMode()&0o666) if err != nil { return fmt.Errorf("unable to open file '%s': %w", path, err) diff --git a/api/utils/accessobj/format-tar.go b/api/utils/accessobj/format-tar.go index 72d8532116..a6c48972c4 100644 --- a/api/utils/accessobj/format-tar.go +++ b/api/utils/accessobj/format-tar.go @@ -67,9 +67,28 @@ func (h *TarHandler) Write(obj *AccessObject, path string, opts accessio.Options defer writer.Close() + // Check if OCI layout (has subdirectories in element dir) + if h.hasNestedDirs(obj) { + return h.writeToStreamOCI(obj, writer, opts) + } return h.WriteToStream(obj, writer, opts) } +// hasNestedDirs checks if the element directory contains subdirectories (OCI layout). +func (h *TarHandler) hasNestedDirs(obj *AccessObject) bool { + elemDir := obj.info.GetElementDirectoryName() + entries, err := vfs.ReadDir(obj.fs, elemDir) + if err != nil { + return false + } + for _, e := range entries { + if e.IsDir() { + return true + } + } + return false +} + func (h TarHandler) WriteToStream(obj *AccessObject, writer io.Writer, opts accessio.Options) error { if h.compression != nil { w, err := h.compression.Compressor(writer, nil, nil) @@ -194,6 +213,130 @@ func (h TarHandler) WriteToStream(obj *AccessObject, writer io.Writer, opts acce return tw.Close() } +// writeToStreamOCI writes the access object to a tar stream using OCI layout format. +// This handles nested directory structures (e.g., blobs/sha256/DIGEST). +func (h TarHandler) writeToStreamOCI(obj *AccessObject, writer io.Writer, opts accessio.Options) error { + if h.compression != nil { + w, err := h.compression.Compressor(writer, nil, nil) + if err != nil { + return fmt.Errorf("unable to compress writer: %w", err) + } + defer w.Close() + + writer = w + } + + // write descriptor + _, err := obj.Update() + if err != nil { + return fmt.Errorf("unable to update access object: %w", err) + } + + data, err := obj.state.GetBlob() + if err != nil { + return fmt.Errorf("unable to write to get state blob: %w", err) + } + defer data.Close() + + tw := tar.NewWriter(writer) + cdHeader := &tar.Header{ + Name: obj.info.GetDescriptorFileName(), + Size: data.Size(), + Mode: FileMode, + ModTime: ModTime, + } + + if err := tw.WriteHeader(cdHeader); err != nil { + return fmt.Errorf("unable to write descriptor header: %w", err) + } + + r, err := data.Reader() + if err != nil { + return fmt.Errorf("unable to get reader: %w", err) + } + defer r.Close() + + if _, err := io.Copy(tw, r); err != nil { + return fmt.Errorf("unable to write descriptor content: %w", err) + } + + // Copy additional files + for _, f := range obj.info.GetAdditionalFiles(obj.fs) { + ok, err := vfs.IsFile(obj.fs, f) + if err != nil { + return errors.Wrapf(err, "cannot check for file %q", f) + } + if ok { + fi, err := obj.fs.Stat(f) + if err != nil { + return errors.Wrapf(err, "cannot stat file %q", f) + } + header := &tar.Header{ + Name: f, + Size: fi.Size(), + Mode: FileMode, + ModTime: ModTime, + } + if err := tw.WriteHeader(header); err != nil { + return errors.Wrapf(err, "unable to write descriptor header") + } + + r, err := obj.fs.Open(f) + if err != nil { + return errors.Wrapf(err, "unable to get reader") + } + if _, err := io.Copy(tw, r); err != nil { + r.Close() + return errors.Wrapf(err, "unable to write file %s", f) + } + r.Close() + } + } + + // add all element content (nested structure for OCI layout) + elemDir := obj.info.GetElementDirectoryName() + err = vfs.Walk(obj.fs, elemDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + header := &tar.Header{ + Name: path, + Mode: FileMode, + ModTime: ModTime, + } + + if info.IsDir() { + header.Typeflag = tar.TypeDir + header.Mode = DirMode + } else { + header.Size = info.Size() + } + + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("unable to write header for %s: %w", path, err) + } + + if !info.IsDir() { + content, err := obj.fs.Open(path) + if err != nil { + return fmt.Errorf("unable to open %s: %w", path, err) + } + _, err = io.Copy(tw, content) + content.Close() + if err != nil { + return fmt.Errorf("unable to write content for %s: %w", path, err) + } + } + return nil + }) + if err != nil && !os.IsNotExist(err) { + return err + } + + return tw.Close() +} + func (h *TarHandler) NewFromReader(info AccessObjectInfo, acc AccessMode, in io.Reader, opts accessio.Options, closer Closer) (*AccessObject, error) { if h.compression != nil { reader, err := h.compression.Decompressor(in) diff --git a/cmds/ocm/commands/ocicmds/artifacts/download/cmd.go b/cmds/ocm/commands/ocicmds/artifacts/download/cmd.go index 621ced55fc..06127d7a87 100644 --- a/cmds/ocm/commands/ocicmds/artifacts/download/cmd.go +++ b/cmds/ocm/commands/ocicmds/artifacts/download/cmd.go @@ -54,9 +54,13 @@ func (o *Command) ForName(name string) *cobra.Command { Short: "download oci artifacts", Long: ` Download artifacts from an OCI registry. The result is stored in -artifact set format, without the repository part +artifact set format, without the repository part. The files are named according to the artifact repository name. + +By default, blobs are stored in OCM artifact set format (blobs/.). +Use --oci-layout to store blobs in OCI Image Layout format (blobs//) +for compatibility with tools that expect the OCI Image Layout Specification. `, } } @@ -242,7 +246,18 @@ func (d *download) Save(o *artifacthdlr.Object, f string) error { digest := blob.Digest() format := formatoption.From(d.opts) - set, err := artifactset.Create(accessobj.ACC_CREATE, f, format.Mode(), format.Format, accessio.PathFileSystem(dest.PathFilesystem)) + + createOpts := []accessio.Option{ + format.Format, + accessio.PathFileSystem(dest.PathFilesystem), + } + // When --oci-layout is specified, use FORMAT_OCI to store blobs at + // blobs// per OCI Image Layout Specification. + if opts.OCILayout { + createOpts = append(createOpts, artifactset.StructureFormat(artifactset.FORMAT_OCI)) + } + + set, err := artifactset.Create(accessobj.ACC_CREATE, f, format.Mode(), createOpts...) if err != nil { return err } diff --git a/cmds/ocm/commands/ocicmds/artifacts/download/option.go b/cmds/ocm/commands/ocicmds/artifacts/download/option.go index 174e60727f..40a5d697bc 100644 --- a/cmds/ocm/commands/ocicmds/artifacts/download/option.go +++ b/cmds/ocm/commands/ocicmds/artifacts/download/option.go @@ -17,13 +17,15 @@ func New() *Option { } type Option struct { - Layers []int - DirTree bool + Layers []int + DirTree bool + OCILayout bool } func (o *Option) AddFlags(fs *pflag.FlagSet) { fs.IntSliceVarP(&o.Layers, "layers", "", nil, "extract dedicated layers") fs.BoolVarP(&o.DirTree, "dirtree", "", false, "extract as effective filesystem content") + fs.BoolVarP(&o.OCILayout, "oci-layout", "", false, "download as OCI Image Layout (blobs in blobs//)") } func (o *Option) Usage() string { @@ -31,6 +33,8 @@ func (o *Option) Usage() string { With option --layers it is possible to request the download of dedicated layers, only. Option --dirtree expects the artifact to be a layered filesystem (for example OCI Image) and provided the effective -filesystem content. +filesystem content. Option --oci-layout downloads the artifact +in OCI Image Layout format with blobs stored at blobs/<algorithm>/<encoded> +according to the OCI Image Layout Specification. ` } diff --git a/cmds/ocm/commands/ocmcmds/resources/download/action.go b/cmds/ocm/commands/ocmcmds/resources/download/action.go index 8fada4b0b1..69c47813e3 100644 --- a/cmds/ocm/commands/ocmcmds/resources/download/action.go +++ b/cmds/ocm/commands/ocmcmds/resources/download/action.go @@ -12,6 +12,7 @@ import ( "ocm.software/ocm/api/ocm" v1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" "ocm.software/ocm/api/ocm/extensions/download" + ocilayouthdlr "ocm.software/ocm/api/ocm/extensions/download/handlers/ocilayout" "ocm.software/ocm/api/ocm/tools/signing" "ocm.software/ocm/api/utils/blobaccess" common2 "ocm.software/ocm/api/utils/misc" @@ -33,7 +34,14 @@ type Action struct { } func NewAction(ctx ocm.ContextProvider, opts *output.Options) *Action { - return &Action{downloaders: download.For(ctx), opts: opts} + downloaders := download.For(ctx).Copy() + + local := From(opts) + if local.OCILayout { + downloaders.Register(ocilayouthdlr.New(), download.ForArtifactType(download.ALL), download.WithPrio(ocilayouthdlr.PRIORITY)) + } + + return &Action{downloaders: downloaders, opts: opts} } func (d *Action) AddOptions(opts ...options.Options) { diff --git a/cmds/ocm/commands/ocmcmds/resources/download/options.go b/cmds/ocm/commands/ocmcmds/resources/download/options.go index 4942477e1d..1714bf3b30 100644 --- a/cmds/ocm/commands/ocmcmds/resources/download/options.go +++ b/cmds/ocm/commands/ocmcmds/resources/download/options.go @@ -21,6 +21,7 @@ type Option struct { SilentOption bool UseHandlers bool Verify bool + OCILayout bool } func (o *Option) SetUseHandlers(ok ...bool) *Option { @@ -33,6 +34,7 @@ func (o *Option) AddFlags(fs *pflag.FlagSet) { fs.BoolVarP(&o.UseHandlers, "download-handlers", "d", false, "use download handler if possible") } fs.BoolVarP(&o.Verify, "verify", "", false, "verify downloads") + fs.BoolVarP(&o.OCILayout, "oci-layout", "", false, "download OCI artifacts in OCI Image Layout format (blobs//)") } func (o *Option) Usage() string {