Add support for retrieving data from OCI registries

detect_oci.go

@@ -0,0 +1,77 @@
+package getter
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var matchRegistries = []*regexp.Regexp{
+	regexp.MustCompile("azurecr.io"),
+	regexp.MustCompile("gcr.io"),
+	regexp.MustCompile("registry.gitlab.com"),
+	regexp.MustCompile("[0-9]{12}.dkr.ecr.[a-z0-9-]*.amazonaws.com"),
+}
+
+// OCIDetector implements Detector to detect OCI registry URLs and turn
+// them into URLs that the OCI getter can understand.
+type OCIDetector struct{}
+
+// Detect will detect if the source is an OCI registry
+func (d *OCIDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	if containsOCIRegistry(src) || containsLocalRegistry(src) {
+		url, err := d.detectHTTP(src)
+		if err != nil {
+			return "", false, fmt.Errorf("detect http: %w", err)
+		}
+
+		return url, true, nil
+	}
+
+	return "", false, nil
+}
+
+func containsOCIRegistry(src string) bool {
+	for _, matchRegistry := range matchRegistries {
+		if matchRegistry.MatchString(src) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func containsLocalRegistry(src string) bool {
+	return strings.Contains(src, "127.0.0.1:5000") || strings.Contains(src, "localhost:5000")
+}
+
+func (d *OCIDetector) detectHTTP(src string) (string, error) {
+	parts := strings.Split(src, "/")
+	if len(parts) < 2 {
+		return "", fmt.Errorf(
+			"URL is not a valid Azure registry URL")
+	}
+
+	return "oci://" + getRepositoryFromURL(src), nil
+}
+
+func getRepositoryFromURL(url string) string {
+	if repositoryContainsTag(url) {
+		return url
+	}
+
+	return url + ":latest"
+}
+
+func repositoryContainsTag(repository string) bool {
+	path := strings.Split(repository, "/")
+	return pathContainsTag(path[len(path)-1])
+}
+
+func pathContainsTag(path string) bool {
+	return strings.Contains(path, ":")
+}

detect_oci_test.go

@@ -0,0 +1,68 @@
+package getter
+
+import "testing"
+
+func TestOCIDetector_Detect(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			"should detect azurecr",
+			"user.azurecr.io/policies:tag",
+			"oci://user.azurecr.io/policies:tag",
+		},
+		{
+			"should detect gcr",
+			"gcr.io/conftest/policies:tag",
+			"oci://gcr.io/conftest/policies:tag",
+		},
+		{
+			"should detect ecr",
+			"123456789012.dkr.ecr.us-east-1.amazonaws.com/conftest/policies:tag",
+			"oci://123456789012.dkr.ecr.us-east-1.amazonaws.com/conftest/policies:tag",
+		},
+		{
+			"should detect gitlab",
+			"registry.gitlab.com/conftest/policies:tag",
+			"oci://registry.gitlab.com/conftest/policies:tag",
+		},
+		{
+			"should add latest tag",
+			"user.azurecr.io/policies",
+			"oci://user.azurecr.io/policies:latest",
+		},
+		{
+			"should detect 127.0.0.1:5000 as most likely being an OCI registry",
+			"127.0.0.1:5000/policies:tag",
+			"oci://127.0.0.1:5000/policies:tag",
+		},
+		{
+			"should detect 127.0.0.1:5000 as most likely being an OCI registry and tag it properly if no tag is supplied",
+			"127.0.0.1:5000/policies",
+			"oci://127.0.0.1:5000/policies:latest",
+		},
+		{
+			"should detect localhost:5000 as most likely being an OCI registry and tag it properly if no tag is supplied",
+			"localhost:5000/policies",
+			"oci://localhost:5000/policies:latest",
+		},
+	}
+	pwd := "/pwd"
+	d := &OCIDetector{}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			out, ok, err := d.Detect(tt.input, pwd)
+			if err != nil {
+				t.Fatalf("OCIDetector.Detect() error = %v", err)
+			}
+			if !ok {
+				t.Fatal("OCIDetector.Detect() not ok, should have detected")
+			}
+			if out != tt.expected {
+				t.Errorf("OCIDetector.Detect() output = %v, want %v", out, tt.expected)
+			}
+		})
+	}
+}

get_oci.go

@@ -0,0 +1,74 @@
+package getter
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/content/file"
+	"oras.land/oras-go/v2/registry/remote"
+)
+
+// OCIGetter is responsible for handling OCI repositories
+type OCIGetter struct {
+	getter
+}
+
+// ClientMode returns the client mode directory
+func (g *OCIGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
+// Get gets the repository as the specified url
+func (g *OCIGetter) Get(path string, u *url.URL) error {
+	ctx := g.Context()
+
+	src, err := g.getRepository(u)
+	if err != nil {
+		return err
+	}
+
+	reference := src.Reference.Reference
+
+	if reference == "" {
+		reference = "latest"
+	}
+
+	err = os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("make directory for OCI storage: %w", err)
+	}
+
+	dst, err := file.New(path)
+	if err != nil {
+		return fmt.Errorf("cannot create file destination OCIGetter: %w", err)
+	}
+	defer dst.Close()
+
+	_, err = oras.Copy(ctx, src, reference, dst, reference, oras.DefaultCopyOptions)
+	if err != nil {
+		return fmt.Errorf("unable to copy OCI artifact: %w", err)
+	}
+
+	return nil
+}
+
+func (g *OCIGetter) getRepository(u *url.URL) (*remote.Repository, error) {
+	repository, err := remote.NewRepository(getReferenceFromURL(u))
+	if err != nil {
+		return nil, fmt.Errorf("invalid OCI URL: %w", err)
+	}
+
+	return repository, nil
+}
+
+func getReferenceFromURL(u *url.URL) (string) {
+	return path.Join(u.Host, u.Path)
+}
+
+// GetFile is currently a NOOP
+func (g *OCIGetter) GetFile(dst string, u *url.URL) error {
+	return nil
+}

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/aws/aws-sdk-go v1.44.122
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
 	github.com/cheggaaa/pb v1.0.27
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2
@@ -23,6 +22,7 @@ require (
 	google.golang.org/api v0.100.0
 	google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
+	oras.land/oras-go/v2 v2.2.0
 )
 
 go 1.13

go.sum

@@ -351,20 +351,29 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
 github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0-rc.3 h1:GT9Xon8YrLxz6N7sErbN81V8J4lOQKGUZQmI3ioviqU=
+github.com/opencontainers/image-spec v1.1.0-rc.3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -510,6 +519,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -875,13 +886,16 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
+oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=