Add support for retrieving data from OCI registries

Several existing registries with OCI support can be detected and the URL will recieve the oci:// protocol. Alternatively, the oci:// protocol can be added to the URL for other (e.g. private) registries. The deis/ORAS library is used to fetch the OCI artifacts from the storage.

Signed-off-by: Lennard Eijsackers <[email protected]>

detect_oci.go

@@ -0,0 +1,77 @@
+package getter
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var matchRegistries = []*regexp.Regexp{
+	regexp.MustCompile("azurecr.io"),
+	regexp.MustCompile("gcr.io"),
+	regexp.MustCompile("registry.gitlab.com"),
+	regexp.MustCompile("[0-9]{12}.dkr.ecr.[a-z0-9-]*.amazonaws.com"),
+}
+
+// OCIDetector implements Detector to detect OCI registry URLs and turn
+// them into URLs that the OCI getter can understand.
+type OCIDetector struct{}
+
+// Detect will detect if the source is an OCI registry
+func (d *OCIDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	if containsOCIRegistry(src) || containsLocalRegistry(src) {
+		url, err := d.detectHTTP(src)
+		if err != nil {
+			return "", false, fmt.Errorf("detect http: %w", err)
+		}
+
+		return url, true, nil
+	}
+
+	return "", false, nil
+}
+
+func containsOCIRegistry(src string) bool {
+	for _, matchRegistry := range matchRegistries {
+		if matchRegistry.MatchString(src) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func containsLocalRegistry(src string) bool {
+	return strings.Contains(src, "127.0.0.1:5000") || strings.Contains(src, "localhost:5000")
+}
+
+func (d *OCIDetector) detectHTTP(src string) (string, error) {
+	parts := strings.Split(src, "/")
+	if len(parts) < 2 {
+		return "", fmt.Errorf(
+			"URL is not a valid Azure registry URL")
+	}
+
+	return "oci://" + getRepositoryFromURL(src), nil
+}
+
+func getRepositoryFromURL(url string) string {
+	if repositoryContainsTag(url) {
+		return url
+	}
+
+	return url + ":latest"
+}
+
+func repositoryContainsTag(repository string) bool {
+	path := strings.Split(repository, "/")
+	return pathContainsTag(path[len(path)-1])
+}
+
+func pathContainsTag(path string) bool {
+	return strings.Contains(path, ":")
+}

detect_oci_test.go

@@ -0,0 +1,68 @@
+package getter
+
+import "testing"
+
+func TestOCIDetector_Detect(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			"should detect azurecr",
+			"user.azurecr.io/policies:tag",
+			"oci://user.azurecr.io/policies:tag",
+		},
+		{
+			"should detect gcr",
+			"gcr.io/conftest/policies:tag",
+			"oci://gcr.io/conftest/policies:tag",
+		},
+		{
+			"should detect ecr",
+			"123456789012.dkr.ecr.us-east-1.amazonaws.com/conftest/policies:tag",
+			"oci://123456789012.dkr.ecr.us-east-1.amazonaws.com/conftest/policies:tag",
+		},
+		{
+			"should detect gitlab",
+			"registry.gitlab.com/conftest/policies:tag",
+			"oci://registry.gitlab.com/conftest/policies:tag",
+		},
+		{
+			"should add latest tag",
+			"user.azurecr.io/policies",
+			"oci://user.azurecr.io/policies:latest",
+		},
+		{
+			"should detect 127.0.0.1:5000 as most likely being an OCI registry",
+			"127.0.0.1:5000/policies:tag",
+			"oci://127.0.0.1:5000/policies:tag",
+		},
+		{
+			"should detect 127.0.0.1:5000 as most likely being an OCI registry and tag it properly if no tag is supplied",
+			"127.0.0.1:5000/policies",
+			"oci://127.0.0.1:5000/policies:latest",
+		},
+		{
+			"should detect localhost:5000 as most likely being an OCI registry and tag it properly if no tag is supplied",
+			"localhost:5000/policies",
+			"oci://localhost:5000/policies:latest",
+		},
+	}
+	pwd := "/pwd"
+	d := &OCIDetector{}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, ok, err := d.Detect(tt.input, pwd)
+			if err != nil {
+				t.Fatalf("OCIDetector.Detect() error = %v", err)
+			}
+			if !ok {
+				t.Fatal("OCIDetector.Detect() not ok, should have detected")
+			}
+			if out != tt.expected {
+				t.Errorf("OCIDetector.Detect() output = %v, want %v", out, tt.expected)
+			}
+		})
+	}
+}

get_oci.go

@@ -0,0 +1,62 @@
+package getter
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+
+	auth "github.com/deislabs/oras/pkg/auth/docker"
+	"github.com/deislabs/oras/pkg/content"
+	"github.com/deislabs/oras/pkg/oras"
+)
+
+// OCIGetter is responsible for handling OCI repositories
+type OCIGetter struct {
+	getter
+}
+
+// ClientMode returns the client mode directory
+func (g *OCIGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
+// Get gets the repository as the specified url
+func (g *OCIGetter) Get(path string, u *url.URL) error {
+	ctx := g.Context()
+
+	if !pathContainsTag(u.Path) {
+		u.Path = u.Path + ":latest"
+	}
+
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("make policy directory: %w", err)
+	}
+
+	cli, err := auth.NewClient()
+	if err != nil {
+		return fmt.Errorf("new auth client: %w", err)
+	}
+
+	resolver, err := cli.Resolver(ctx, http.DefaultClient, false)
+	if err != nil {
+		return fmt.Errorf("new resolver: %w", err)
+	}
+
+	fileStore := content.NewFileStore(path)
+	defer fileStore.Close()
+
+	repository := u.Host + u.Path
+	_, _, err = oras.Pull(ctx, resolver, repository, fileStore)
+	if err != nil {
+		return fmt.Errorf("pulling policy: %w", err)
+	}
+
+	return nil
+}
+
+// GetFile is currently a NOOP
+func (g *OCIGetter) GetFile(dst string, u *url.URL) error {
+	return nil
+}

go.mod

@@ -6,17 +6,17 @@ require (
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
 	github.com/cheggaaa/pb v1.0.27
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deislabs/oras v0.8.1
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.0
 	github.com/hashicorp/go-safetemp v1.0.0
 	github.com/hashicorp/go-version v1.1.0
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.4 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/mitchellh/go-homedir v1.0.0
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-testing-interface v1.0.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/testify v1.2.2 // indirect
 	github.com/ulikunitz/xz v0.5.5
 	google.golang.org/api v0.9.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect