Add support for retrieving data from OCI registries

Several existing registries with OCI support can be detected and the URL will recieve the oci:// protocol. Alternatively, the oci:// protocol can be added to the URL for other (e.g. private) registries. The deis/ORAS library is used to fetch the OCI artifacts from the storage.

Signed-off-by: Lennard Eijsackers <[email protected]>

.github/workflows/release.yml

@@ -0,0 +1,76 @@
+### This builds, packages, signs, performs AV and malware scanning, and
+### creates a new GitHub release for the newest version of go-getter. 
+### The GitHub release step performs the actions outlined in 
+### release.goreleaser.yml. A release is triggered when a new tag 
+### is pushed in the format vX.X.X
+
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15'
+      - name: Install hc-codesign
+        id: codesign
+        run: |
+          docker login docker.pkg.github.com -u docker -p $GITHUB_TOKEN && \
+          docker pull docker.pkg.github.com/hashicorp/hc-codesign/hc-codesign:$VERSION && \
+          echo "::set-output name=image::docker.pkg.github.com/hashicorp/hc-codesign/hc-codesign:$VERSION"
+        env:
+          VERSION: v0
+          GITHUB_TOKEN: ${{ secrets.CODESIGN_GITHUB_TOKEN }}
+      - name: Install wget & clamAV antivirus scanner
+        run : |
+           sudo apt-get -qq install -y ca-certificates wget clamav
+           wget --version
+      - name: Install maldet malware scanner
+        run: |
+          wget --no-verbose -O maldet-$VERSION.tar.gz https://github.com/rfxn/linux-malware-detect/archive/$VERSION.tar.gz
+          sha256sum -c - <<< "$SHA256SUM  maldet-$VERSION.tar.gz"
+          sudo mkdir -p maldet-$VERSION
+          sudo tar -xzf maldet-$VERSION.tar.gz --strip-components=1 -C maldet-$VERSION
+          cd maldet-$VERSION
+          sudo ./install.sh
+          sudo maldet -u
+        env: 
+          VERSION: 1.6.4
+          SHA256SUM: 3ad66eebd443d32dd6c811dcf2d264b78678c75ed1d40c15434180d4453e60d2
+      - name: Import PGP key for archive signing
+        run: echo -e $PGP_KEY | base64 -di | gpg --import --batch
+        env:
+          GPG_TTY: $(tty)
+          PGP_KEY: ${{ secrets.PGP_SIGNING_KEY }}
+      - name: GitHub Release
+        uses: goreleaser/goreleaser-action@v1
+        with:
+          version: latest
+          args: release --skip-validate --timeout "60m"
+        env:
+          PGP_KEY_ID: ${{ secrets.PGP_KEY_ID }}
+          CODESIGN_IMAGE: ${{ steps.codesign.outputs.image }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
+          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
+          CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
+      - name: Run clamAV antivirus scanner
+        run: sudo clamscan /home/runner/work/$REPO/$REPO/dist/
+        env:
+          REPO: ${{ github.event.repository.name }}
+      - name: Run maldet malware scanner
+        run: sudo maldet -a /home/runner/work/$REPO/$REPO/dist/
+        env:
+          REPO: ${{ github.event.repository.name }}
+

.goreleaser.yml

@@ -0,0 +1,49 @@
+before:
+  hooks:
+    - go test ./...
+
+builds:
+  - id: signable
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    targets:
+      - darwin_amd64
+      - windows_386
+      - windows_amd64
+    hooks:
+      post: |
+        docker run
+          -e ARTIFACTORY_TOKEN={{ .Env.ARTIFACTORY_TOKEN }}
+          -e ARTIFACTORY_USER={{ .Env.ARTIFACTORY_USER }}
+          -e CIRCLE_TOKEN={{ .Env.CIRCLE_TOKEN }}
+          -v {{ dir .Path }}:/workdir
+          {{ .Env.CODESIGN_IMAGE }}
+          sign -product-name={{ .ProjectName }} {{ .Name }}
+    flags:
+      - -trimpath
+    ldflags:
+      - -X main.GitCommit={{ .Commit }}
+  - mod_timestamp: '{{ .CommitTimestamp }}'
+    targets:
+      - linux_386
+      - linux_amd64
+    flags:
+      - -trimpath
+    ldflags:
+      - -X main.GitCommit={{ .Commit }}
+
+archives:
+  - format: zip
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    files: 
+      - none*
+
+checksum:
+  name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS'
+  algorithm: sha256
+
+signs:
+  - args: ["-u", "{{ .Env.PGP_KEY_ID }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
+    artifacts: checksum
+
+changelog:
+  skip: true

README.md

@@ -80,6 +80,8 @@ is built-in by default:
     file URLs.
   * GitHub URLs, such as "github.com/mitchellh/vagrant" are automatically
     changed to Git protocol over HTTP.
+  * GitLab URLs, such as "gitlab.com/inkscape/inkscape" are automatically 
+    changed to Git protocol over HTTP.
   * BitBucket URLs, such as "bitbucket.org/mitchellh/vagrant" are automatically
     changed to a Git or mercurial protocol using the BitBucket API.
 
@@ -316,6 +318,7 @@ are also supported. If the query parameters are present, these take priority.
   * `aws_access_key_id` - AWS access key.
   * `aws_access_key_secret` - AWS access key secret.
   * `aws_access_token` - AWS access token if this is being used.
+  * `aws_profile` - Use this profile from local ~/.aws/ config. Takes priority over the other three.
 
 #### Using IAM Instance Profiles with S3
 

checksum.go

@@ -259,7 +259,10 @@ func (c *Client) ChecksumFromFile(checksumFile string, src *url.URL) (*FileCheck
 				return nil, fmt.Errorf(
 					"Error reading checksum file: %s", err)
 			}
-			break
+			if line == "" {
+				break
+			}
+			// parse the line, if we hit EOF, but the line is not empty
 		}
 		checksum, err := parseChecksumLine(line)
 		if err != nil || checksum == nil {

client.go

@@ -39,6 +39,10 @@ type Client struct {
 	// for documentation.
 	Mode ClientMode
 
+	// Umask is used to mask file permissions when storing local files or decompressing
+	// an archive
+	Umask os.FileMode
+
 	// Detectors is the list of detectors that are tried on the source.
 	// If this is nil, then the default Detectors will be used.
 	Detectors []Detector
@@ -66,6 +70,20 @@ type Client struct {
 	Options []ClientOption
 }
 
+// umask returns the effective umask for the Client, defaulting to the process umask
+func (c *Client) umask() os.FileMode {
+	if c == nil {
+		return 0
+	}
+	return c.Umask
+}
+
+// mode returns file mode umasked by the Client umask
+func (c *Client) mode(mode os.FileMode) os.FileMode {
+	m := mode & ^c.umask()
+	return m
+}
+
 // Get downloads the configured source to the destination.
 func (c *Client) Get() error {
 	if err := c.Configure(c.Options...); err != nil {
@@ -233,7 +251,7 @@ func (c *Client) Get() error {
 		if decompressor != nil {
 			// We have a decompressor, so decompress the current destination
 			// into the final destination with the proper mode.
-			err := decompressor.Decompress(decompressDst, dst, decompressDir)
+			err := decompressor.Decompress(decompressDst, dst, decompressDir, c.umask())
 			if err != nil {
 				return err
 			}
@@ -281,7 +299,7 @@ func (c *Client) Get() error {
 		if err := os.RemoveAll(realDst); err != nil {
 			return err
 		}
-		if err := os.MkdirAll(realDst, 0755); err != nil {
+		if err := os.MkdirAll(realDst, c.mode(0755)); err != nil {
 			return err
 		}
 
@@ -291,7 +309,7 @@ func (c *Client) Get() error {
 			return err
 		}
 
-		return copyDir(c.Ctx, realDst, subDir, false)
+		return copyDir(c.Ctx, realDst, subDir, false, c.umask())
 	}
 
 	return nil

copy_dir.go

@@ -7,11 +7,16 @@ import (
 	"strings"
 )
 
+// mode returns the file mode masked by the umask
+func mode(mode, umask os.FileMode) os.FileMode {
+	return mode & ^umask
+}
+
 // copyDir copies the src directory contents into dst. Both directories
 // should already exist.
 //
 // If ignoreDot is set to true, then dot-prefixed files/folders are ignored.
-func copyDir(ctx context.Context, dst string, src string, ignoreDot bool) error {
+func copyDir(ctx context.Context, dst string, src string, ignoreDot bool, umask os.FileMode) error {
 	src, err := filepath.EvalSymlinks(src)
 	if err != nil {
 		return err
@@ -46,32 +51,16 @@ func copyDir(ctx context.Context, dst string, src string, ignoreDot bool) error
 				return nil
 			}
 
-			if err := os.MkdirAll(dstPath, 0755); err != nil {
+			if err := os.MkdirAll(dstPath, mode(0755, umask)); err != nil {
 				return err
 			}
 
 			return nil
 		}
 
 		// If we have a file, copy the contents.
-		srcF, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer srcF.Close()
-
-		dstF, err := os.Create(dstPath)
-		if err != nil {
-			return err
-		}
-		defer dstF.Close()
-
-		if _, err := Copy(ctx, dstF, srcF); err != nil {
-			return err
-		}
-
-		// Chmod it
-		return os.Chmod(dstPath, info.Mode())
+		_, err = copyFile(ctx, dstPath, path, info.Mode(), umask)
+		return err
 	}
 
 	return filepath.Walk(src, walkFn)

decompress.go

@@ -1,6 +1,7 @@
 package getter
 
 import (
+	"os"
 	"strings"
 )
 
@@ -14,7 +15,7 @@ type Decompressor interface {
 	// Decompress should decompress src to dst. dir specifies whether dst
 	// is a directory or single file. src is guaranteed to be a single file
 	// that exists. dst is not guaranteed to exist already.
-	Decompress(dst, src string, dir bool) error
+	Decompress(dst, src string, dir bool, umask os.FileMode) error
 }
 
 // Decompressors is the mapping of extension to the Decompressor implementation

decompress_bzip2.go

@@ -3,7 +3,6 @@ package getter
 import (
 	"compress/bzip2"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 )
@@ -12,14 +11,14 @@ import (
 // decompress bz2 files.
 type Bzip2Decompressor struct{}
 
-func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool) error {
+func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// Directory isn't supported at all
 	if dir {
 		return fmt.Errorf("bzip2-compressed files can only unarchive to a single file")
 	}
 
 	// If we're going into a directory we should make that first
-	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(dst), mode(0755, umask)); err != nil {
 		return err
 	}
 
@@ -34,12 +33,5 @@ func (d *Bzip2Decompressor) Decompress(dst, src string, dir bool) error {
 	bzipR := bzip2.NewReader(f)
 
 	// Copy it out
-	dstF, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer dstF.Close()
-
-	_, err = io.Copy(dstF, bzipR)
-	return err
+	return copyReader(dst, bzipR, 0622, umask)
 }

decompress_gzip.go

@@ -3,7 +3,6 @@ package getter
 import (
 	"compress/gzip"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 )
@@ -12,14 +11,14 @@ import (
 // decompress gzip files.
 type GzipDecompressor struct{}
 
-func (d *GzipDecompressor) Decompress(dst, src string, dir bool) error {
+func (d *GzipDecompressor) Decompress(dst, src string, dir bool, umask os.FileMode) error {
 	// Directory isn't supported at all
 	if dir {
 		return fmt.Errorf("gzip-compressed files can only unarchive to a single file")
 	}
 
 	// If we're going into a directory we should make that first
-	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(dst), mode(0755, umask)); err != nil {
 		return err
 	}
 
@@ -38,12 +37,5 @@ func (d *GzipDecompressor) Decompress(dst, src string, dir bool) error {
 	defer gzipR.Close()
 
 	// Copy it out
-	dstF, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer dstF.Close()
-
-	_, err = io.Copy(dstF, gzipR)
-	return err
+	return copyReader(dst, gzipR, 0622, umask)
 }