diff --git a/README.md b/README.md index 1f5bea9..b0b47f0 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,14 @@ The location of a custom configuration-file and logging are specified via the co | `--help,` | `-h` | show help | | `--version` | `-v` | prints version-information and exists | - ### Structured Logging -Vault Raft Snapshot Agent uses go's [slog package](https://pkg.go.dev/log/slog) to provide structured logging capabilities. -Log format `text` uses [TextHandler](https://pkg.go.dev/log/slog#TextHandler), `json` uses [JSONHandler](https://pkg.go.dev/log/slog#JSONHandler). -If no log format or `default` is specified the default log format is used which outputs the timestamp followed by the message followed by additional key=value-pairs if any are present. +Vault Raft Snapshot Agent uses go's [slog package](https://pkg.go.dev/log/slog) to provide structured logging +capabilities. +Log format `text` uses [TextHandler](https://pkg.go.dev/log/slog#TextHandler), `json` +uses [JSONHandler](https://pkg.go.dev/log/slog#JSONHandler). +If no log format or `default` is specified the default log format is used which outputs the timestamp followed by the +message followed by additional key=value-pairs if any are present. ## Environment variables @@ -486,6 +488,7 @@ to `timestampFormat` and `nameSuffix`, e.g. the defaults would generate CEST (GMT + 2h). These options can be overridden for a specific storage: + ``` snapshots: frequency: 1h @@ -499,6 +502,7 @@ snapshots: timestampFormat: 2006-01-02 #... ``` + In this example the agent would take and store a snapshot to the local-storage every hour, retaining 24 snapshots and store a daily snapshot on aws remote storage, retaining the last 365 snapshots with a appropriate shorter timestamp. @@ -513,9 +517,13 @@ When using multiple remote storages, increase the timeout allowed via `snapahots Each option can be specified exactly once; it is currently not possible to e.g. upload to multiple aws regions by specifying multiple `aws`-storage-options. -#### AWS S3 Upload +#### AWS S3 Storage + +Uploads snapshots to an [AWS S3 storage](https://aws.amazon.com/s3/) bucket. This storage uses +the [AWS Go SDK](https://pkg.go.dev/github.com/aws/aws-sdk-go/service/s3). Use this storage for S3 services that use an AWS S3-API compatible addressing-scheme (e.g. `https://-`). For other S3 implementations, try the [generic s3 storage](#genericminio-s3-storage). ##### Minimal Configuration + ``` snapshots: storage @@ -524,6 +532,7 @@ snapshots: ``` ##### Configuration Options + | Key | Type | Required/*Default* | Description | | ------------------------- | ------------------------------------------------ | ----------------------------- | ----------------------------------------------------------------------------------------------------------------- | | `bucket` | String | **required** | bucket to store snapshots in | @@ -540,7 +549,10 @@ Any common [snapshot configuration option](#snapshot-configuration) overrides th #### Azure Storage +Uploads snapshots to an [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) container. + ##### Minimal Configuration + ``` snapshots: storages: @@ -549,6 +561,7 @@ snapshots: ``` ##### Configuration Options + | Key | Type | Required/*Default* | Description | | ------------- | ------------------------------------------------ | ----------------------------- | ---------------------------------------------------------------------------- | | `container` | String | **required** | the name of the blob container to write to | @@ -559,7 +572,11 @@ snapshots: Any common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration. #### Google Cloud Storage + +Uploads snapshots into a [Google Cloud storage](https://cloud.google.com/storage) bucket. + ##### Minimal Configuration + ``` snapshots: storages: @@ -568,6 +585,7 @@ snapshots: ``` ##### Configuration Options + | Key | Type | Required/*Default* | Description | | -------- | ------ | ------------------ | ----------------------------------------------------------------------------------------- | | `bucket` | String | **required** | the Google Storage Bucket to write to. Auth is expected to be default machine credentials | @@ -575,14 +593,18 @@ snapshots: Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration. #### Local Storage + ##### Minimal Configuration + ``` snapshots: storages: local: path: ``` + ##### Configuration Options + | Key | Type | Required/*Default* | Description | | ------ | ------ | ------------------ | --------------------------------------------------------------------------------------------------------------- | | `path` | String | **required** | fully qualified path, not including file name, for where the snapshot should be written. i.e. `/raft/snapshots` | @@ -590,7 +612,11 @@ snapshots: Any common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration. #### Openstack Swift Storage + +Uploads snapshots to a [Openstack Swift Object Storage](https://www.openstack.org/software/releases/zed/components/swift) container. + ##### Minimal Configuration + ``` snapshots: storages: @@ -612,6 +638,35 @@ snapshots: Any common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration. +#### Generic/MinIO S3 Storage + +Uploads snapshots to any S3-compatible server. This storage uses +the [MinIO Go Client SDK](https://github.com/minio/minio-go). If your self-hosted S3-server does not support the default adressing-scheme of [AWS S3](#aws-s3-storage), then this storage might still work. + +##### Minimal Configuration + +``` +snapshots: + storage + s3: + endpoint: + bucket: +``` + +##### Configuration Options + +| Key | Type | Required/*Default* | Description | +| -------------- | ------------------------------------------------ | ---------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `endpoint` | String | **required** | S3 compatible storage endpoint (ex: my-storage.example.com) | +| `bucket` | String | **required** | bucket to store snapshots in | +| `accessKeyId` | [Secret](#secrets-and-external-property-sources) | *env://S3_ACCESS_KEY_ID* | specifies the access key | +| `accessKey` | [Secret](#secrets-and-external-property-sources) | *env://S3_SECRET_ACCESS_KEY* | specifies the secret access key; **must resolve to non-empty value if accessKeyId resolves to a non-empty value** | +| `sessionToken` | [Secret](#secrets-and-external-property-sources) | *env://S3_SESSION_TOKEN* | specifies the session token | +| `region` | [Secret](#secrets-and-external-property-sources) | | S3 region if it is required | +| `insecure` | Boolean | *false* | whether to connect using https (false) or not | + +Any common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration. + ## License - Source code is licensed under MIT diff --git a/go.mod b/go.mod index 3516d69..c7c98a1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/urfave/cli/v2 v2.25.7 ) -// AWS-Uploader +// AWS-Storage require ( github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.37 @@ -22,18 +22,22 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 ) -// Azure-Uploader +// Azure-Storage require github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 -// GCP-Uploader +// GCP-Storage require ( cloud.google.com/go/storage v1.31.0 google.golang.org/api v0.140.0 ) -// Swift-Uploader +// Swift-Storage require github.com/ncw/swift/v2 v2.0.2 +// S3-Storage +require github.com/minio/minio-go/v7 v7.0.63 // indirect + + // Vault require ( github.com/hashicorp/vault/api v1.10.0 @@ -83,6 +87,7 @@ require ( github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect @@ -108,16 +113,25 @@ require ( github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 800ff05..14f0fed 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -230,6 +232,7 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -328,9 +331,16 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -361,6 +371,12 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -368,6 +384,11 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= @@ -386,11 +407,15 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= @@ -406,6 +431,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -622,6 +648,7 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/agent/snapshot-agent-config_test.go b/internal/agent/snapshot-agent-config_test.go index be2c889..8980b94 100644 --- a/internal/agent/snapshot-agent-config_test.go +++ b/internal/agent/snapshot-agent-config_test.go @@ -119,6 +119,15 @@ func TestReadCompleteConfig(t *testing.T) { Region: "test-region", TenantId: "test-tenant", }, + S3: storage.S3StorageConfig{ + Endpoint: "test-s3-endpoint", + Bucket: "test-s3-bucket", + AccessKeyId: "test-s3-key", + AccessKey: "test-s3-secret", + SessionToken: "test-s3-token", + Region: "test-s3-region", + Insecure: true, + }, }, }, } @@ -207,6 +216,12 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { Region: secret.FromEnv("SWIFT_REGION"), Empty: true, }, + S3: storage.S3StorageConfig{ + AccessKeyId: secret.FromEnv("S3_ACCESS_KEY_ID"), + AccessKey: secret.FromEnv("S3_SECRET_ACCESS_KEY"), + SessionToken: secret.FromEnv("S3_SESSION_TOKEN"), + Empty: true, + }, }, }, } diff --git a/internal/agent/snapshot-agent.go b/internal/agent/snapshot-agent.go index bb13be8..b63f4da 100644 --- a/internal/agent/snapshot-agent.go +++ b/internal/agent/snapshot-agent.go @@ -50,7 +50,7 @@ type snapshotAgentVaultAPI interface { type snapshotManager interface { ScheduleSnapshot(ctx context.Context, lastSnapshot time.Time, defaults storage.StorageConfigDefaults) time.Time - UploadSnapshot(ctx context.Context, snapshot io.ReadSeeker, timestamp time.Time, defaults storage.StorageConfigDefaults) time.Time + UploadSnapshot(ctx context.Context, snapshot io.ReadSeeker, snapshotSize int64, timestamp time.Time, defaults storage.StorageConfigDefaults) time.Time } func (c SnapshotAgentConfig) HasStorages() bool { @@ -163,7 +163,7 @@ func (a *SnapshotAgent) TakeSnapshot(ctx context.Context) *time.Ticker { return a.snapshotTicker } - nextSnapshot = a.manager.UploadSnapshot(ctx, snapshot, a.lastSnapshotTime, a.storageConfigDefaults) + nextSnapshot = a.manager.UploadSnapshot(ctx, snapshot, 0, a.lastSnapshotTime, a.storageConfigDefaults) return a.updateTicker(nextSnapshot) } diff --git a/internal/agent/snapshot-agent_test.go b/internal/agent/snapshot-agent_test.go index 2c556df..ea1bf28 100644 --- a/internal/agent/snapshot-agent_test.go +++ b/internal/agent/snapshot-agent_test.go @@ -335,7 +335,7 @@ func (stub storageControllerStub) DeleteObsoleteSnapshots(_ context.Context, _ s return 0, nil } -func (stub storageControllerStub) UploadSnapshot(_ context.Context, snapshot io.Reader, timestamp time.Time, defaults storage.StorageConfigDefaults) (bool, time.Time, error) { +func (stub storageControllerStub) UploadSnapshot(_ context.Context, snapshot io.Reader, _ int64, timestamp time.Time, defaults storage.StorageConfigDefaults) (bool, time.Time, error) { stub.factory.snapshotTimestamp = timestamp stub.factory.defaults = defaults if stub.factory.uploadFails { diff --git a/internal/agent/storage/aws.go b/internal/agent/storage/aws.go index ef9db73..500b968 100644 --- a/internal/agent/storage/aws.go +++ b/internal/agent/storage/aws.go @@ -111,11 +111,12 @@ func (conf AWSStorageConfig) createClient(ctx context.Context) (*s3.Client, erro // nolint:unused // implements interface storage -func (s awsStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (s awsStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader, size int64) error { input := &s3.PutObjectInput{ - Bucket: &s.bucket, - Key: aws.String(s.keyPrefix + name), - Body: data, + Bucket: &s.bucket, + Key: aws.String(s.keyPrefix + name), + Body: data, + ContentLength: size, } if s.sse { diff --git a/internal/agent/storage/azure.go b/internal/agent/storage/azure.go index 702512d..8a19c81 100644 --- a/internal/agent/storage/azure.go +++ b/internal/agent/storage/azure.go @@ -64,7 +64,7 @@ func createAzBlobClient(config AzureStorageConfig) (*azblob.Client, error) { // nolint:unused // implements interface storage -func (s azureStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (s azureStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader, _ int64) error { uploadOptions := &azblob.UploadStreamOptions{ BlockSize: 4 * 1024 * 1024, Concurrency: 16, diff --git a/internal/agent/storage/config.go b/internal/agent/storage/config.go index 42c85a7..44c11b3 100644 --- a/internal/agent/storage/config.go +++ b/internal/agent/storage/config.go @@ -11,6 +11,7 @@ type StoragesConfig struct { GCP GCPStorageConfig `default:"{\"Empty\": true}"` Local LocalStorageConfig `default:"{\"Empty\": true}"` Swift SwiftStorageConfig `default:"{\"Empty\": true}"` + S3 S3StorageConfig `default:"{\"Empty\": true}"` } // StorageConfigDefaults specified the default values of storageConfig for all factories diff --git a/internal/agent/storage/controller.go b/internal/agent/storage/controller.go index 5bc5b72..28d89b7 100644 --- a/internal/agent/storage/controller.go +++ b/internal/agent/storage/controller.go @@ -21,7 +21,7 @@ type storageControllerImpl[S any] struct { // storage defines the interface used by storageControllerImpl to access a storage-location type storage[S any] interface { - uploadSnapshot(ctx context.Context, name string, data io.Reader) error + uploadSnapshot(ctx context.Context, name string, data io.Reader, size int64) error deleteSnapshot(ctx context.Context, snapshot S) error listSnapshots(ctx context.Context, prefix string, suffix string) ([]S, error) getLastModifiedTime(snapshot S) time.Time @@ -55,7 +55,7 @@ func (u *storageControllerImpl[S]) ScheduleSnapshot(ctx context.Context, lastSna return u.lastUpload.Add(u.config.frequencyOrDefault(defaults)), nil } -func (u *storageControllerImpl[S]) UploadSnapshot(ctx context.Context, snapshot io.Reader, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) { +func (u *storageControllerImpl[S]) UploadSnapshot(ctx context.Context, snapshot io.Reader, snapshotSize int64, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) { frequency := u.config.frequencyOrDefault(defaults) if timestamp.Before(u.lastUpload.Add(frequency)) { @@ -73,7 +73,7 @@ func (u *storageControllerImpl[S]) UploadSnapshot(ctx context.Context, snapshot ts := timestamp.Format(u.config.timestampFormatOrDefault(defaults)) snapshotName := strings.Join([]string{prefix, ts, suffix}, "") - if err := u.storage.uploadSnapshot(ctx, snapshotName, snapshot); err != nil { + if err := u.storage.uploadSnapshot(ctx, snapshotName, snapshot, snapshotSize); err != nil { return false, nextSnapshot, err } diff --git a/internal/agent/storage/controller_test.go b/internal/agent/storage/controller_test.go index 1d5109d..f2d71fe 100644 --- a/internal/agent/storage/controller_test.go +++ b/internal/agent/storage/controller_test.go @@ -124,12 +124,7 @@ func TestUploadSnapshotUploadsToStorage(t *testing.T) { data := "test" timestamp := time.Now() start := time.Now() - uploaded, nextSnapshot, err := controller.UploadSnapshot( - ctx, - strings.NewReader(data), - timestamp, - StorageConfigDefaults{}, - ) + uploaded, nextSnapshot, err := controller.UploadSnapshot(ctx, strings.NewReader(data), 0, timestamp, StorageConfigDefaults{}) assert.NoError(t, err, "uploadSnapshot failed unexpectedly") assert.True(t, uploaded) @@ -158,12 +153,7 @@ func TestUploadSnapshotHandlesStorageFailure(t *testing.T) { ctx := context.Background() timestamp := time.Now() - uploaded, nextSnapshot, err := controller.UploadSnapshot( - ctx, - strings.NewReader("test"), - timestamp, - StorageConfigDefaults{}, - ) + uploaded, nextSnapshot, err := controller.UploadSnapshot(ctx, strings.NewReader("test"), 0, timestamp, StorageConfigDefaults{}) assert.False(t, uploaded) assert.Error(t, err, "uploadSnapshot should return error if storage fails") @@ -296,12 +286,7 @@ func TestUploadSnapshotSkipsUploadBeforeScheduledTime(t *testing.T) { storage: storage, } - uploaded, nextSnapshot, err := controller.UploadSnapshot( - context.Background(), - strings.NewReader("test"), - controller.lastUpload.Add(time.Second), - StorageConfigDefaults{}, - ) + uploaded, nextSnapshot, err := controller.UploadSnapshot(context.Background(), strings.NewReader("test"), 0, controller.lastUpload.Add(time.Second), StorageConfigDefaults{}) assert.NoError(t, err, "uploadSnapshot failed unexpectedly") assert.False(t, uploaded) @@ -328,7 +313,7 @@ type storageStub struct { // nolint:unused // implements interface storage -func (stub *storageStub) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (stub *storageStub) uploadSnapshot(ctx context.Context, name string, data io.Reader, _ int64) error { stub.uploadContext = ctx stub.uploadName = name upload, err := io.ReadAll(data) diff --git a/internal/agent/storage/gcp.go b/internal/agent/storage/gcp.go index a0d5f4b..085fc37 100644 --- a/internal/agent/storage/gcp.go +++ b/internal/agent/storage/gcp.go @@ -39,7 +39,7 @@ func (conf GCPStorageConfig) CreateController(ctx context.Context) (StorageContr // nolint:unused // implements interface storage -func (u gcpStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (u gcpStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader, _ int64) error { obj := u.bucket.Object(name) w := obj.NewWriter(ctx) diff --git a/internal/agent/storage/local.go b/internal/agent/storage/local.go index 37fb884..3a04c17 100644 --- a/internal/agent/storage/local.go +++ b/internal/agent/storage/local.go @@ -33,7 +33,7 @@ func (conf LocalStorageConfig) CreateController(context.Context) (StorageControl ), nil } -func (u localStorageImpl) uploadSnapshot(_ context.Context, name string, data io.Reader) error { +func (u localStorageImpl) uploadSnapshot(_ context.Context, name string, data io.Reader, _ int64) error { fileName := fmt.Sprintf("%s/%s", u.path, name) file, err := os.Create(fileName) diff --git a/internal/agent/storage/local_test.go b/internal/agent/storage/local_test.go index 1c691b0..6c24791 100644 --- a/internal/agent/storage/local_test.go +++ b/internal/agent/storage/local_test.go @@ -15,7 +15,7 @@ import ( func TestLocalUploadSnapshotFailsIfFileCannotBeCreated(t *testing.T) { impl := localStorageImpl{"./does/not/exist"} - err := impl.uploadSnapshot(context.Background(), "test", &bytes.Buffer{}) + err := impl.uploadSnapshot(context.Background(), "test", &bytes.Buffer{}, 0) assert.Error(t, err, "uploadSnapshot() should fail if file could not be created!") } @@ -24,7 +24,7 @@ func TestLocalUploadeSnapshotCreatesFile(t *testing.T) { impl := localStorageImpl{t.TempDir()} snapshotData := []byte("test") - err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData)) + err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData), 0) assert.NoError(t, err, "uploadSnapshot() failed unexpectedly!") @@ -42,7 +42,7 @@ func TestLocalDeleteSnapshot(t *testing.T) { _ = os.RemoveAll(filepath.Dir(impl.path)) }() - err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData)) + err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData), 0) assert.NoError(t, err, "uploadSnapshot() failed unexpectedly!") info, err := os.Stat(fmt.Sprintf("%s/test.snap", impl.path)) diff --git a/internal/agent/storage/manager.go b/internal/agent/storage/manager.go index 071902f..419f2f3 100644 --- a/internal/agent/storage/manager.go +++ b/internal/agent/storage/manager.go @@ -33,7 +33,7 @@ type StorageController interface { // corresponds with its scheduled upload-date. // For the case that the storageConfig of the controller does not specify one of its fields, // StorageConfigDefaults is passed. - UploadSnapshot(ctx context.Context, snapshot io.Reader, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) + UploadSnapshot(ctx context.Context, snapshot io.Reader, snapshotSize int64, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) DeleteObsoleteSnapshots(ctx context.Context, defaults StorageConfigDefaults) (int, error) } @@ -57,6 +57,9 @@ func CreateManager(storageConfig StoragesConfig) *Manager { if !storageConfig.Swift.Empty { manager.AddStorageFactory(storageConfig.Swift) } + if !storageConfig.S3.Empty { + manager.AddStorageFactory(storageConfig.S3) + } return manager } @@ -95,7 +98,7 @@ func (m *Manager) ScheduleSnapshot(ctx context.Context, lastSnapshotTime time.Ti // and returns the time the next snapshot should be taken. // Whether the snapshot is actually uploaded to a storage is controlled by the StorageController based // on the upload-frequency in its StoragesConfig -func (m *Manager) UploadSnapshot(ctx context.Context, snapshot io.ReadSeeker, timestamp time.Time, defaults StorageConfigDefaults) time.Time { +func (m *Manager) UploadSnapshot(ctx context.Context, snapshot io.ReadSeeker, snapshotSize int64, timestamp time.Time, defaults StorageConfigDefaults) time.Time { var ( nextSnapshot time.Time errs error @@ -112,7 +115,7 @@ func (m *Manager) UploadSnapshot(ctx context.Context, snapshot io.ReadSeeker, ti logging.Warn("Could not create storage-controller", "destination", factory.Destination(), "error", err) errs = multierr.Append(errs, err) } else { - uploaded, candidate, err := controller.UploadSnapshot(ctx, snapshot, timestamp, defaults) + uploaded, candidate, err := controller.UploadSnapshot(ctx, snapshot, snapshotSize, timestamp, defaults) if nextSnapshot.IsZero() || candidate.Before(nextSnapshot) { nextSnapshot = candidate } diff --git a/internal/agent/storage/manager_test.go b/internal/agent/storage/manager_test.go index e3547d2..cb990f1 100644 --- a/internal/agent/storage/manager_test.go +++ b/internal/agent/storage/manager_test.go @@ -51,7 +51,7 @@ func TestManagerUploadsToAllControllers(t *testing.T) { } data := "test" - nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), controller1.nextSnapshot, StorageConfigDefaults{}) + nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), 0, controller1.nextSnapshot, StorageConfigDefaults{}) assert.Equal(t, data, controller1.uploadData) assert.Equal(t, controller1.nextSnapshot, controller1.snapshotTimestamp) @@ -71,7 +71,7 @@ func TestManagerDeletesObsoleteSnapshotsWithAllControllers(t *testing.T) { } defaults := StorageConfigDefaults{Retain: 2} - _ = manager.UploadSnapshot(context.Background(), strings.NewReader("test"), controller1.nextSnapshot, defaults) + _ = manager.UploadSnapshot(context.Background(), strings.NewReader("test"), 0, controller1.nextSnapshot, defaults) assert.Equal(t, defaults, controller1.deleteDefaults) assert.Equal(t, defaults, controller2.deleteDefaults) @@ -92,7 +92,7 @@ func TestManagerIgnoresFactoryAndControllerFailure(t *testing.T) { data := "test" defaults := StorageConfigDefaults{} - nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), controller3.nextSnapshot, defaults) + nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), 0, controller3.nextSnapshot, defaults) assert.Equal(t, data, controller3.uploadData) assert.Equal(t, controller3.nextSnapshot, controller3.snapshotTimestamp) @@ -115,7 +115,7 @@ func TestManagerIgnoresSkippedControllers(t *testing.T) { } data := "test" - nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), controller2.nextSnapshot, StorageConfigDefaults{}) + nextSnapshot := manager.UploadSnapshot(context.Background(), strings.NewReader(data), 0, controller2.nextSnapshot, StorageConfigDefaults{}) assert.Equal(t, data, controller2.uploadData) assert.Equal(t, controller2.nextSnapshot, controller2.snapshotTimestamp) @@ -132,7 +132,7 @@ func TestManagerFailsIfSnapshotCannotBeReset(t *testing.T) { defaults := StorageConfigDefaults{Frequency: time.Second} timestamp := time.Now() - nextSnapshot := manager.UploadSnapshot(context.Background(), ReadSeekerStub{}, timestamp, defaults) + nextSnapshot := manager.UploadSnapshot(context.Background(), ReadSeekerStub{}, 0, timestamp, defaults) assert.Equal(t, timestamp.Add(defaults.Frequency), nextSnapshot) assert.Zero(t, controller.uploadData) @@ -172,7 +172,7 @@ func (stub *storageControllerStub) ScheduleSnapshot(context.Context, time.Time, return stub.nextSnapshot, nil } -func (stub *storageControllerStub) UploadSnapshot(_ context.Context, snapshot io.Reader, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) { +func (stub *storageControllerStub) UploadSnapshot(_ context.Context, snapshot io.Reader, _ int64, timestamp time.Time, defaults StorageConfigDefaults) (bool, time.Time, error) { stub.snapshotTimestamp = timestamp stub.uploadDefaults = defaults if stub.uploadFails { diff --git a/internal/agent/storage/s3.go b/internal/agent/storage/s3.go new file mode 100644 index 0000000..830d7d9 --- /dev/null +++ b/internal/agent/storage/s3.go @@ -0,0 +1,137 @@ +package storage + +import ( + "context" + "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/config/secret" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/logging" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "io" + "strings" + "time" +) + +type S3StorageConfig struct { + storageConfig `mapstructure:",squash"` + Endpoint string `validate:"required_if=Empty false"` + Bucket string `validate:"required_if=Empty false"` + AccessKeyId secret.Secret `default:"env://S3_ACCESS_KEY_ID"` + AccessKey secret.Secret `default:"env://S3_SECRET_ACCESS_KEY" validate:"required_with=AccessKeyId"` + SessionToken secret.Secret `default:"env://S3_SESSION_TOKEN"` + Region secret.Secret + Insecure bool + Empty bool +} + +type s3StorageImpl struct { + client *minio.Client + bucket string +} + +func (conf S3StorageConfig) Destination() string { + return fmt.Sprintf("s3 bucket %s at %s", conf.Bucket, conf.Endpoint) +} + +func (conf S3StorageConfig) CreateController(ctx context.Context) (StorageController, error) { + client, err := conf.createClient(ctx) + if err != nil { + return nil, err + } + + return newStorageController[minio.ObjectInfo]( + conf.storageConfig, + s3StorageImpl{ + client: client, + bucket: conf.Bucket, + }, + ), nil + +} + +func (conf S3StorageConfig) createClient(ctx context.Context) (*minio.Client, error) { + accessKeyId, err := conf.AccessKeyId.Resolve(false) + if err != nil { + return nil, err + } + + accessKey, err := conf.AccessKey.Resolve(accessKeyId != "") + if err != nil { + return nil, err + } + + sessionToken, err := conf.SessionToken.Resolve(false) + if err != nil { + return nil, err + } + + region, err := conf.Region.Resolve(false) + if err != nil { + return nil, err + } + + client, err := minio.New(conf.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyId, accessKey, sessionToken), + Secure: !conf.Insecure, + Region: region, + }) + if err != nil { + return nil, err + } + + err = client.MakeBucket(ctx, conf.Bucket, minio.MakeBucketOptions{Region: region}) + if err != nil { + exists, err := client.BucketExists(ctx, conf.Bucket) + if err != nil { + return nil, err + } + if !exists { + return nil, fmt.Errorf("bucket %s does not exist", conf.Bucket) + } + } + + logging.Debug("Successfully connected", "destination", conf.Destination()) + + return client, nil +} + +// nolint:unused +// implements interface storage +func (s s3StorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader, size int64) error { + _, err := s.client.PutObject(ctx, s.bucket, name, data, size, minio.PutObjectOptions{}) + if err != nil { + return err + } + + return nil +} + +// nolint:unused +// implements interface storage +func (s s3StorageImpl) deleteSnapshot(ctx context.Context, snapshot minio.ObjectInfo) error { + return s.client.RemoveObject(ctx, s.bucket, snapshot.Key, minio.RemoveObjectOptions{ForceDelete: true}) +} + +// nolint:unused +// implements interface storage +func (s s3StorageImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]minio.ObjectInfo, error) { + var result []minio.ObjectInfo + objectCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix}) + + for snapshot := range objectCh { + if snapshot.Err != nil { + return nil, snapshot.Err + } + if strings.HasSuffix(snapshot.Key, ext) { + result = append(result, snapshot) + } + } + + return result, nil +} + +// nolint:unused +// implements interface storage +func (s s3StorageImpl) getLastModifiedTime(snapshot minio.ObjectInfo) time.Time { + return snapshot.LastModified +} diff --git a/internal/agent/storage/swift.go b/internal/agent/storage/swift.go index e32faac..5582cdb 100644 --- a/internal/agent/storage/swift.go +++ b/internal/agent/storage/swift.go @@ -82,7 +82,7 @@ func createSwiftConnection(ctx context.Context, config SwiftStorageConfig) (*swi // nolint:unused // implements interface storage -func (u swiftStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (u swiftStorageImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader, _ int64) error { _, header, err := u.conn.Container(ctx, u.container) if err != nil { return err diff --git a/testdata/complete.yaml b/testdata/complete.yaml index c149466..7405c92 100644 --- a/testdata/complete.yaml +++ b/testdata/complete.yaml @@ -69,4 +69,13 @@ snapshots: domain: https://user.com region: test-region tenantId: test-tenant + s3: + endpoint: test-s3-endpoint + bucket: test-s3-bucket + accessKeyId: test-s3-key + accessKey: test-s3-secret + sessionToken: test-s3-token + region: test-s3-region + insecure: true +