Implement vetu login credentials validation and introduce vetu {pull,push} --insecure command-line argument (#27)

* vetu login: validate credentials and support --no-validate

* Chunker: fix NPE when Close()'ing a chunker with no Write()s made on it

* Chunker: always emit at least one chunk on Close()

This may happen when no Write()s were made.

* Work around pierrec/lz4#212

This happens when attempting to compress a disk that is empty (0 bytes).

* Hide progress bar when the size of disk to be pulled is zero (0 bytes)

* vetu {pull,push}: support --insecure

* dockerhosts: fix comments

Author: edigaryev

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/distribution/reference v0.5.0
 	github.com/docker/cli v24.0.6+incompatible
 	github.com/dustin/go-humanize v1.0.1
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.4.0
 	github.com/gosuri/uitable v0.0.4
 	github.com/hashicorp/go-version v1.6.0
 	github.com/insomniacslk/dhcp v0.0.0-20230908212754-65c27093e38a
@@ -16,15 +16,15 @@ require (
 	github.com/otiai10/copy v1.12.0
 	github.com/pierrec/lz4/v4 v4.1.18
 	github.com/projectcalico/libcalico-go v1.7.3
-	github.com/regclient/regclient v0.5.2
+	github.com/regclient/regclient v0.5.4
 	github.com/samber/lo v1.38.1
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/seancfoley/ipaddress-go v1.5.4
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
-	golang.org/x/sys v0.12.0
-	golang.org/x/term v0.12.0
+	golang.org/x/sys v0.14.0
+	golang.org/x/term v0.14.0
 	gvisor.dev/gvisor v0.0.0-20230926030033-4af66e670562
 	inet.af/tcpproxy v0.0.0-20221017015627-91f861402626
 )

go.sum

@@ -1,7 +1,7 @@
 github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU=
 github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
 github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -24,8 +24,8 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=
 github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
@@ -77,8 +77,8 @@ github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74
 github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
 github.com/projectcalico/libcalico-go v1.7.3 h1:qcbxAhsq/5zqZqpHE24VqMHfmoBVdXZV0Kf82+5rbqU=
 github.com/projectcalico/libcalico-go v1.7.3/go.mod h1:0b/n/rPzNXjhn4ywFcEJuQdA/5olt9UxFIATz57xkbc=
-github.com/regclient/regclient v0.5.2 h1:8iLMsMIbI0/5iNwWUOHn463g3NSy9beLSpmFx/VfXAQ=
-github.com/regclient/regclient v0.5.2/go.mod h1:2iTgEKbcEEa2tJr3gWs0s1nkRm4V72XbPVDxmF4biX4=
+github.com/regclient/regclient v0.5.4 h1:oJLY/M18jo+Lzg+6KH29ZjzZ8XX5uIybAOTcAqP8hwU=
+github.com/regclient/regclient v0.5.4/go.mod h1:604ymXFhwbmWjyfGFp3uF91gfCXIcjWBxJhBg94s9cM=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -93,8 +93,8 @@ github.com/seancfoley/ipaddress-go v1.5.4 h1:ZdjewWC1J2y5ruQjWHwK6rA1tInWB6mz1ft
 github.com/seancfoley/ipaddress-go v1.5.4/go.mod h1:fpvVPC+Jso+YEhNcNiww8HQmBgKP8T4T6BTp1SLxxIo=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -122,11 +122,11 @@ golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/chunker/chunker.go

@@ -20,7 +20,8 @@ type Chunker struct {
 	initializeWriter InitializeWriterFunc
 
 	// State
-	chunks chan *Chunk
+	chunks  chan *Chunk
+	emitted bool
 
 	// Per-chunk state
 	buf              *bytes.Buffer
@@ -38,7 +39,7 @@ type Chunk struct {
 	UncompressedDigest digest.Digest
 }
 
-func NewChunker(chunkSize int, initializeWriter InitializeWriterFunc) *Chunker {
+func NewChunker(chunkSize int, initializeWriter InitializeWriterFunc) (*Chunker, error) {
 	chunker := &Chunker{
 		// Settings
 		chunkSize:        chunkSize,
@@ -48,9 +49,11 @@ func NewChunker(chunkSize int, initializeWriter InitializeWriterFunc) *Chunker {
 		chunks: make(chan *Chunk),
 	}
 
-	chunker.resetPerChunkState()
+	if err := chunker.resetPerChunkState(); err != nil {
+		return nil, err
+	}
 
-	return chunker
+	return chunker, nil
 }
 
 func (chunker *Chunker) Write(b []byte) (int, error) {
@@ -73,18 +76,11 @@ func (chunker *Chunker) Write(b []byte) (int, error) {
 			UncompressedSize:   chunker.uncompressedSize,
 			UncompressedDigest: digest.NewDigest(digest.SHA256, chunker.uncompressedHash),
 		}
+		chunker.emitted = true
 
-		chunker.resetPerChunkState()
-	}
-
-	// chunker.writer is nil resetPerChunkState(), so initialize it before using it
-	if chunker.writer == nil {
-		writer, err := chunker.initializeWriter(chunker.buf)
-		if err != nil {
+		if err := chunker.resetPerChunkState(); err != nil {
 			return 0, err
 		}
-
-		chunker.writer = writer
 	}
 
 	// Update uncompressed chunk size
@@ -114,8 +110,9 @@ func (chunker *Chunker) Close() error {
 		return err
 	}
 
-	// Only emit a chunk if we have some data available
-	if chunker.buf.Len() != 0 {
+	// Only emit a last chunk if we have some data available
+	// or there were no chunks emitted before
+	if chunker.buf.Len() != 0 || !chunker.emitted {
 		chunker.chunks <- &Chunk{
 			Data:               chunker.buf.Bytes(),
 			UncompressedSize:   chunker.uncompressedSize,
@@ -125,14 +122,19 @@ func (chunker *Chunker) Close() error {
 
 	close(chunker.chunks)
 
-	chunker.resetPerChunkState()
-
-	return nil
+	return chunker.resetPerChunkState()
 }
 
-func (chunker *Chunker) resetPerChunkState() {
+func (chunker *Chunker) resetPerChunkState() error {
 	chunker.buf = &bytes.Buffer{}
 	chunker.uncompressedSize = 0
 	chunker.uncompressedHash = sha256.New()
-	chunker.writer = nil
+
+	writer, err := chunker.initializeWriter(chunker.buf)
+	if err != nil {
+		return err
+	}
+	chunker.writer = writer
+
+	return nil
 }

internal/chunker/chunker_test.go

@@ -27,9 +27,10 @@ func TestSimple(t *testing.T) {
 		})
 	}
 
-	chunker := chunkerpkg.NewChunker(chunkSize, func(w io.Writer) (io.WriteCloser, error) {
+	chunker, err := chunkerpkg.NewChunker(chunkSize, func(w io.Writer) (io.WriteCloser, error) {
 		return WriteNopCloser(w), nil
 	})
+	require.NoError(t, err)
 
 	go func() {
 		defer chunker.Close()
@@ -51,6 +52,36 @@ func TestSimple(t *testing.T) {
 	require.Equal(t, expectedChunks, actualChunks)
 }
 
+func TestNoWrites(t *testing.T) {
+	const chunkSize = 1 * 1024 * 1024
+
+	// Create a chunker and close it right away without doing any Write()s
+	chunker, err := chunkerpkg.NewChunker(chunkSize, func(w io.Writer) (io.WriteCloser, error) {
+		return WriteNopCloser(w), nil
+	})
+	require.NoError(t, err)
+	go func() {
+		if err := chunker.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	// Ensure that exactly one empty chunk is emitted as a result of the above
+	var actualChunks []*chunkerpkg.Chunk
+
+	for chunk := range chunker.Chunks() {
+		actualChunks = append(actualChunks, chunk)
+	}
+
+	require.Equal(t, []*chunkerpkg.Chunk{
+		{
+			Data:               nil,
+			UncompressedSize:   0,
+			UncompressedDigest: digest.FromBytes([]byte{}),
+		},
+	}, actualChunks)
+}
+
 type writeNopCloser struct {
 	io.Writer
 }

internal/command/login/login.go

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	dockerconfig "github.com/docker/cli/cli/config"
 	"github.com/docker/cli/cli/config/types"
+	"github.com/regclient/regclient"
+	"github.com/regclient/regclient/config"
+	"github.com/regclient/regclient/types/ref"
 	"github.com/spf13/cobra"
 	"golang.org/x/term"
 	"io"
@@ -15,6 +18,7 @@ import (
 
 var username string
 var passwordStdin bool
+var noValidate bool
 
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -30,6 +34,8 @@ func NewCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&passwordStdin, "password-stdin", false,
 		"receive the password from the standard input instead of asking it interactively "+
 			"(requires --username)")
+	cmd.Flags().BoolVar(&noValidate, "no-validate", false,
+		"skip validation of the OCI registry's credentials before logging-in")
 
 	return cmd
 }
@@ -43,6 +49,28 @@ func runLogin(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if !noValidate {
+		// Create an OCI registry client that only has the provided credentials
+		client := regclient.New(regclient.WithConfigHost(config.Host{
+			Name:     registry,
+			Hostname: registry,
+			User:     username,
+			Pass:     password,
+		}))
+
+		// Create a reference that only has the Registry field set
+		reference, err := ref.NewHost(registry)
+		if err != nil {
+			return err
+		}
+
+		// Validate credentials
+		_, err = client.Ping(cmd.Context(), reference)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Store credentials
 	configFile, err := dockerconfig.Load("")
 	if err != nil {

internal/command/pull/pull.go

@@ -3,6 +3,7 @@ package pull
 import (
 	"errors"
 	"fmt"
+	"github.com/cirruslabs/vetu/internal/dockerhosts"
 	"github.com/cirruslabs/vetu/internal/name/remotename"
 	"github.com/cirruslabs/vetu/internal/oci"
 	"github.com/cirruslabs/vetu/internal/storage/remote"
@@ -13,6 +14,7 @@ import (
 )
 
 var concurrency uint8
+var insecure bool
 
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -24,6 +26,8 @@ func NewCommand() *cobra.Command {
 
 	cmd.Flags().Uint8Var(&concurrency, "concurrency", 4,
 		"network concurrency to use when pulling a remote VM from the OCI-compatible registry")
+	cmd.Flags().BoolVar(&insecure, "insecure", false,
+		"connect to the OCI registry via insecure HTTP protocol")
 
 	return cmd
 }
@@ -43,21 +47,27 @@ func runPull(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// Convert remoteName to ref.Ref that is used in github.com/regclient/regclient
+	reference, err := ref.New(remoteName.String())
+	if err != nil {
+		return err
+	}
+
 	// Initialize a temporary directory to which we'll first pull the VM image
 	vmDir, err := temporary.Create()
 	if err != nil {
 		return err
 	}
 
-	// Initialize OCI registry client and convert remote name to a reference
-	client := regclient.New(regclient.WithDockerCreds())
-
-	// Convert remoteName to ref.Ref that is used in github.com/regclient/regclient
-	reference, err := ref.New(remoteName.String())
+	// Load hosts from the Docker configuration file
+	hosts, err := dockerhosts.Load(reference, insecure)
 	if err != nil {
 		return err
 	}
 
+	// Initialize OCI registry client
+	client := regclient.New(regclient.WithConfigHost(hosts...))
+
 	// Resolve the reference to a manifest
 	fmt.Printf("pulling %s...\n", reference.CommonName())
 

internal/command/push/push.go

@@ -1,6 +1,7 @@
 package push
 
 import (
+	"github.com/cirruslabs/vetu/internal/dockerhosts"
 	"github.com/cirruslabs/vetu/internal/name/localname"
 	"github.com/cirruslabs/vetu/internal/name/remotename"
 	"github.com/cirruslabs/vetu/internal/oci"
@@ -12,6 +13,7 @@ import (
 )
 
 var populateCache bool
+var insecure bool
 
 func NewCommand() *cobra.Command {
 	cmd := &cobra.Command{
@@ -23,6 +25,8 @@ func NewCommand() *cobra.Command {
 
 	cmd.Flags().BoolVar(&populateCache, "populate-cache", false, "cache pushed image locally, "+
 		"increases disk usage, but saves time if you're going to pull the pushed images shortly thereafter")
+	cmd.Flags().BoolVar(&insecure, "insecure", false,
+		"connect to the OCI registry via insecure HTTP protocol")
 
 	return cmd
 }
@@ -48,15 +52,21 @@ func runPush(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Initialize OCI registry client
-	client := regclient.New(regclient.WithDockerCreds())
-
-	// Convert remoteName to ref.Ref that is used in github.com/regclient/regclient
+	// Convert dstRemoteName to ref.Ref that is used in github.com/regclient/regclient
 	reference, err := ref.New(dstRemoteName.String())
 	if err != nil {
 		return err
 	}
 
+	// Load hosts from the Docker configuration file
+	hosts, err := dockerhosts.Load(reference, insecure)
+	if err != nil {
+		return err
+	}
+
+	// Initialize OCI registry client
+	client := regclient.New(regclient.WithConfigHost(hosts...))
+
 	// Push the VM image
 	digest, err := oci.PushVMDirectory(cmd.Context(), client, vmDir, reference)
 	if err != nil {