From cc2c8fb92b6eb7a0ce318d269ff5c4d2bcd5f712 Mon Sep 17 00:00:00 2001 From: Poonam Jadhav Date: Mon, 26 Aug 2024 11:10:57 -0400 Subject: [PATCH 1/9] NET-5912/service-defaults protocol validation (#21593) * fix: add validation for protocol field on service-defaults config entry * test: update test cases with correct protocol --- agent/config/builder.go | 4 +++- agent/config_endpoint_test.go | 4 ++-- agent/consul/config_endpoint.go | 4 +++- agent/consul/config_replication_test.go | 6 +++--- agent/structs/config_entry.go | 9 ++++++-- agent/structs/config_entry_test.go | 10 ++++++++- api/config_entry_test.go | 6 +++--- command/config/write/config_write_test.go | 25 +++++++++++++++++++++-- 8 files changed, 53 insertions(+), 15 deletions(-) diff --git a/agent/config/builder.go b/agent/config/builder.go index 64e9120fdec7..5c2eba5c554e 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -22,12 +22,13 @@ import ( "time" "github.com/armon/go-metrics/prometheus" + "golang.org/x/time/rate" + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-sockaddr/template" "github.com/hashicorp/memberlist" - "golang.org/x/time/rate" "github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/checks" @@ -774,6 +775,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) { if err != nil { return RuntimeConfig{}, fmt.Errorf("config_entries.bootstrap[%d]: %s", i, err) } + // Ensure Normalize is called before Validate for accurate validation if err := entry.Normalize(); err != nil { return RuntimeConfig{}, fmt.Errorf("config_entries.bootstrap[%d]: %s", i, err) } diff --git a/agent/config_endpoint_test.go b/agent/config_endpoint_test.go index 8697b55e5bf0..ec00b172d1e4 100644 --- a/agent/config_endpoint_test.go +++ b/agent/config_endpoint_test.go @@ -612,7 +612,7 @@ func TestConfig_Apply_CAS(t *testing.T) { { "Kind": "service-defaults", "Name": "foo", - "Protocol": "udp" + "Protocol": "http" } `)) req, _ = http.NewRequest("PUT", "/v1/config?cas=0", body) @@ -628,7 +628,7 @@ func TestConfig_Apply_CAS(t *testing.T) { { "Kind": "service-defaults", "Name": "foo", - "Protocol": "udp" + "Protocol": "http" } `)) req, _ = http.NewRequest("PUT", fmt.Sprintf("/v1/config?cas=%d", entry.GetRaftIndex().ModifyIndex), body) diff --git a/agent/consul/config_endpoint.go b/agent/consul/config_endpoint.go index a78859c35058..96906dac6824 100644 --- a/agent/consul/config_endpoint.go +++ b/agent/consul/config_endpoint.go @@ -10,10 +10,11 @@ import ( metrics "github.com/armon/go-metrics" "github.com/armon/go-metrics/prometheus" + hashstructure_v2 "github.com/mitchellh/hashstructure/v2" + "github.com/hashicorp/go-bexpr" "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" - hashstructure_v2 "github.com/mitchellh/hashstructure/v2" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/configentry" @@ -85,6 +86,7 @@ func (c *ConfigEntry) Apply(args *structs.ConfigEntryRequest, reply *bool) error } // Normalize and validate the incoming config entry as if it came from a user. + // Ensure Normalize is called before Validate for accurate validation if err := args.Entry.Normalize(); err != nil { return err } diff --git a/agent/consul/config_replication_test.go b/agent/consul/config_replication_test.go index e2c4fbee8d8a..3117e046a463 100644 --- a/agent/consul/config_replication_test.go +++ b/agent/consul/config_replication_test.go @@ -6,11 +6,11 @@ package consul import ( "context" "fmt" - "github.com/oklog/ulid/v2" - "github.com/stretchr/testify/assert" "os" "testing" + "github.com/oklog/ulid/v2" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/hashicorp/consul/agent/structs" @@ -129,7 +129,7 @@ func TestReplication_ConfigEntries(t *testing.T) { Entry: &structs.ServiceConfigEntry{ Kind: structs.ServiceDefaults, Name: fmt.Sprintf("svc-%d", i), - Protocol: "udp", + Protocol: "tcp", }, } diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 5d419e083295..32b4e0c89de0 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -12,12 +12,11 @@ import ( "time" "github.com/miekg/dns" - - "github.com/hashicorp/go-multierror" "github.com/mitchellh/hashstructure" "github.com/mitchellh/mapstructure" "github.com/hashicorp/consul-net-rpc/go-msgpack/codec" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" @@ -269,6 +268,12 @@ func (e *ServiceConfigEntry) Validate() error { validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for balance_inbound_connections: %v", e.BalanceInboundConnections)) } + switch e.Protocol { + case "", "http", "http2", "grpc", "tcp": + default: + validationErr = multierror.Append(validationErr, fmt.Errorf("invalid value for protocol: %v", e.Protocol)) + } + // External endpoints are invalid with an existing service's upstream configuration if e.UpstreamConfig != nil && e.Destination != nil { validationErr = multierror.Append(validationErr, errors.New("UpstreamConfig and Destination are mutually exclusive for service defaults")) diff --git a/agent/structs/config_entry_test.go b/agent/structs/config_entry_test.go index e57e2c404134..4c092acc46e2 100644 --- a/agent/structs/config_entry_test.go +++ b/agent/structs/config_entry_test.go @@ -10,12 +10,12 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/hashicorp/hcl" "github.com/mitchellh/copystructure" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/hashicorp/consul-net-rpc/go-msgpack/codec" + "github.com/hashicorp/hcl" "github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/agent/cache" @@ -3225,6 +3225,14 @@ func TestServiceConfigEntry(t *testing.T) { }, validateErr: `Invalid MutualTLSMode "invalid-mtls-mode". Must be one of "", "strict", or "permissive".`, }, + "validate: invalid Protocol in service-defaults": { + entry: &ServiceConfigEntry{ + Kind: ServiceDefaults, + Name: "web", + Protocol: "blah", + }, + validateErr: `invalid value for protocol: blah`, + }, } testConfigEntryNormalizeAndValidate(t, cases) } diff --git a/api/config_entry_test.go b/api/config_entry_test.go index 2461c387b35f..d6fe8373c9c0 100644 --- a/api/config_entry_test.go +++ b/api/config_entry_test.go @@ -104,7 +104,7 @@ func TestAPI_ConfigEntries(t *testing.T) { service := &ServiceConfigEntry{ Kind: ServiceDefaults, Name: "foo", - Protocol: "udp", + Protocol: "http", MutualTLSMode: MutualTLSModeStrict, Meta: map[string]string{ "foo": "bar", @@ -124,7 +124,7 @@ func TestAPI_ConfigEntries(t *testing.T) { service2 := &ServiceConfigEntry{ Kind: ServiceDefaults, Name: "bar", - Protocol: "tcp", + Protocol: "http", Destination: dest, } @@ -176,7 +176,7 @@ func TestAPI_ConfigEntries(t *testing.T) { require.True(t, written) // update no cas - service.Protocol = "http" + service.Protocol = "tcp" _, wm, err = config_entries.Set(service, nil) require.NoError(t, err) diff --git a/command/config/write/config_write_test.go b/command/config/write/config_write_test.go index 15e7a4746532..ae782c082631 100644 --- a/command/config/write/config_write_test.go +++ b/command/config/write/config_write_test.go @@ -43,7 +43,7 @@ func TestConfigWrite(t *testing.T) { _, err := f.WriteString(` Kind = "service-defaults" Name = "web" - Protocol = "udp" + Protocol = "tcp" `) require.NoError(t, err) @@ -65,7 +65,7 @@ func TestConfigWrite(t *testing.T) { require.True(t, ok) require.Equal(t, api.ServiceDefaults, svc.Kind) require.Equal(t, "web", svc.Name) - require.Equal(t, "udp", svc.Protocol) + require.Equal(t, "tcp", svc.Protocol) }) t.Run("Stdin", func(t *testing.T) { @@ -170,6 +170,27 @@ kind = "proxy-defaults" `Config entry written: proxy-defaults/global`) require.Equal(t, 0, code) }) + + // Test that protocol field is first normalized and then validated + // before writing the config entry + t.Run("service defaults config entry mixed case in protocol field", func(t *testing.T) { + stdin := new(bytes.Buffer) + stdin.WriteString(` + Kind = "service-defaults" + Name = "web" + Protocol = "TcP" +`) + + ui := cli.NewMockUi() + c := New(ui) + c.testStdin = stdin + + code := c.Run([]string{"-http-addr=" + a.HTTPAddr(), "-"}) + require.Empty(t, ui.ErrorWriter.String()) + require.Contains(t, ui.OutputWriter.String(), + `Config entry written: service-defaults/web`) + require.Equal(t, 0, code) + }) } func TestConfigWrite_Warning(t *testing.T) { From 2a99624859ff572e0002df01606d47ca8e29e32c Mon Sep 17 00:00:00 2001 From: Michael Zalimeni Date: Mon, 26 Aug 2024 12:39:35 -0400 Subject: [PATCH 2/9] test: update pause Docker image in Envoy int tests (#21659) k8s.gcr.io has been migrated to registry.k8s.io for several years now, and the old registry is being shut down, causing image pull failures. Update to target the new registry when pulling the pause image used in Envoy integration tests. --- test/integration/connect/envoy/run-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/connect/envoy/run-tests.sh b/test/integration/connect/envoy/run-tests.sh index 46dcb9965fe2..1efebe7cf620 100755 --- a/test/integration/connect/envoy/run-tests.sh +++ b/test/integration/connect/envoy/run-tests.sh @@ -553,8 +553,7 @@ function suite_setup { docker run --sysctl net.ipv6.conf.all.disable_ipv6=1 -d --name envoy_workdir_1 \ $WORKDIR_SNIPPET \ --net=none \ - k8s.gcr.io/pause &>/dev/null - # TODO(rb): switch back to "${HASHICORP_DOCKER_PROXY}/google/pause" once that is cached + registry.k8s.io/pause &>/dev/null # pre-build the verify container echo "Rebuilding 'bats-verify' image..." From 9c02eff1cddbb276bf2a91b55283b3768d5e242b Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 26 Aug 2024 11:49:51 -0600 Subject: [PATCH 3/9] add module retractions (#21665) --- api/go.mod | 6 +++++- envoyextensions/go.mod | 2 ++ troubleshoot/go.mod | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/go.mod b/api/go.mod index 0a1f7fee6fcf..e0c7f19f1a12 100644 --- a/api/go.mod +++ b/api/go.mod @@ -7,7 +7,11 @@ replace ( github.com/hashicorp/consul/sdk => ../sdk ) -retract v1.28.0 // tag was mutated +retract ( + v1.28.0 // tag was mutated + v1.27.1 // tag was mutated + v1.21.2 // tag was mutated +) require ( github.com/google/go-cmp v0.5.9 diff --git a/envoyextensions/go.mod b/envoyextensions/go.mod index c9bb9cb6f00c..57b8c966365a 100644 --- a/envoyextensions/go.mod +++ b/envoyextensions/go.mod @@ -8,6 +8,8 @@ replace ( github.com/hashicorp/consul/sdk => ../sdk ) +retract v0.7.2 // tag was mutated + require ( github.com/envoyproxy/go-control-plane v0.12.0 github.com/google/go-cmp v0.5.9 diff --git a/troubleshoot/go.mod b/troubleshoot/go.mod index 9af318f62f0b..b6c579ee3a2f 100644 --- a/troubleshoot/go.mod +++ b/troubleshoot/go.mod @@ -14,6 +14,12 @@ exclude ( github.com/hashicorp/go-msgpack v1.1.6 // contains retractions but same as v1.1.5 ) +retract ( + v0.6.4 // tag was mutated + v0.6.2 // tag has incorrect line of deps + v0.6.1 // tag has incorrect line of deps +) + require ( github.com/envoyproxy/go-control-plane v0.12.0 github.com/envoyproxy/go-control-plane/xdsmatcher v0.0.0-20230524161521-aaaacbfbe53e From f187b92e3ade7545cace8355adf648b7c4468c39 Mon Sep 17 00:00:00 2001 From: John Murret Date: Mon, 26 Aug 2024 14:12:54 -0600 Subject: [PATCH 4/9] run integration tests on push in main and release/* (#21666) * run integration tests on push in main and release/* * Update .github/workflows/test-integrations.yml Co-authored-by: Michael Zalimeni --------- Co-authored-by: Michael Zalimeni --- .github/workflows/test-integrations.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 3d57c9099471..a6fa149bba35 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -13,6 +13,11 @@ on: - 'backport/docs/**' - 'backport/ui/**' - 'backport/mktg-**' + push: + branches: + # Push events on the main branch + - main + - release/** env: TEST_RESULTS_DIR: /tmp/test-results From ab794b59f84abf9d705263b8a46a74395047c41c Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 28 Aug 2024 09:39:12 -0600 Subject: [PATCH 5/9] update version, changelog, and submodules after 1.19.2, 1.18.4, 1.17.7 and 1.15.14 releases (#21676) * update changelog * Update CHANGELOG.md * remove duplicate 1.19.1 section * update version * update go.mod with most recent modules --- CHANGELOG.md | 46 ++++++++++++++++++++++++ api/go.mod | 2 +- envoyextensions/go.mod | 2 +- go.mod | 8 ++--- test-integ/go.mod | 4 +-- test/integration/consul-container/go.mod | 6 ++-- testing/deployer/go.mod | 4 +-- troubleshoot/go.mod | 4 +-- version/VERSION | 2 +- 9 files changed, 62 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acc8bbe298f9..6d3f07d4361c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,49 @@ +## 1.19.2 (August 26, 2024) + +SECURITY: + +* ui: Upgrade modules with d3-color as a dependency to address denial of service issue in d3-color < 3.1.0 [[GH-21588](https://github.com/hashicorp/consul/issues/21588)] + +IMPROVEMENTS: + +* Use Envoy's default for a route's validate_clusters option, which is false. This fixes a case where non-existent clusters could cause a route to no longer route to any of its backends, including existing ones. [[GH-21587](https://github.com/hashicorp/consul/issues/21587)] + +BUG FIXES: + +* api-gateway: **(Enterprise only)** ensure clusters are properly created for JWT providers with a remote URI for the JWKS endpoint [[GH-21604](https://github.com/hashicorp/consul/issues/21604)] + +## 1.18.4 Enterprise (August 26, 2024) + +Enterprise LTS: Consul Enterprise 1.18 is a Long-Term Support (LTS) release. + +SECURITY: +* ui: Upgrade modules with d3-color as a dependency to address denial of service issue in d3-color < 3.1.0 + +IMPROVEMENTS: + +* Use Envoy's default for a route's validate_clusters option, which is false. This fixes a case where non-existent clusters could cause a route to no longer route to any of its backends, including existing ones. [[GH-21587](https://github.com/hashicorp/consul/issues/21587)] + +## 1.17.7 Enterprise (August 26, 2024) + +SECURITY: +* ui: Upgrade modules with d3-color as a dependency to address denial of service issue in d3-color < 3.1.0 + +IMPROVEMENTS: + +* Use Envoy's default for a route's validate_clusters option, which is false. This fixes a case where non-existent clusters could cause a route to no longer route to any of its backends, including existing ones. [[GH-21587](https://github.com/hashicorp/consul/issues/21587)] + +## 1.15.14 Enterprise (August 26, 2024) + +Enterprise LTS: Consul Enterprise 1.15 is a Long-Term Support (LTS) release. + +SECURITY: + +* ui: Upgrade modules with d3-color as a dependency to address denial of service issue in d3-color < 3.1.0 [[GH-21588](https://github.com/hashicorp/consul/issues/21588)] + +IMPROVEMENTS: + +* Use Envoy's default for a route's validate_clusters option, which is false. This fixes a case where non-existent clusters could cause a route to no longer route to any of its backends, including existing ones. [[GH-21587](https://github.com/hashicorp/consul/issues/21587)] + ## 1.19.1 (July 11, 2024) SECURITY: diff --git a/api/go.mod b/api/go.mod index e0c7f19f1a12..e4de0fa6c17e 100644 --- a/api/go.mod +++ b/api/go.mod @@ -15,7 +15,7 @@ retract ( require ( github.com/google/go-cmp v0.5.9 - github.com/hashicorp/consul/proto-public v0.6.1 + github.com/hashicorp/consul/proto-public v0.6.2 github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-hclog v1.5.0 diff --git a/envoyextensions/go.mod b/envoyextensions/go.mod index 57b8c966365a..9970e5d71f65 100644 --- a/envoyextensions/go.mod +++ b/envoyextensions/go.mod @@ -13,7 +13,7 @@ retract v0.7.2 // tag was mutated require ( github.com/envoyproxy/go-control-plane v0.12.0 github.com/google/go-cmp v0.5.9 - github.com/hashicorp/consul/api v1.29.1 + github.com/hashicorp/consul/api v1.29.4 github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.mod b/go.mod index a6ebcbaabd58..1feee6383ceb 100644 --- a/go.mod +++ b/go.mod @@ -43,11 +43,11 @@ require ( github.com/hashi-derek/grpc-proxy v0.0.0-20231207191910-191266484d75 github.com/hashicorp/consul-awsauth v0.0.0-20220713182709-05ac1c5c2706 github.com/hashicorp/consul-net-rpc v0.0.0-20221205195236-156cfab66a69 - github.com/hashicorp/consul/api v1.29.1 - github.com/hashicorp/consul/envoyextensions v0.7.0 - github.com/hashicorp/consul/proto-public v0.6.1 + github.com/hashicorp/consul/api v1.29.4 + github.com/hashicorp/consul/envoyextensions v0.7.3 + github.com/hashicorp/consul/proto-public v0.6.2 github.com/hashicorp/consul/sdk v0.16.1 - github.com/hashicorp/consul/troubleshoot v0.6.1 + github.com/hashicorp/consul/troubleshoot v0.7.1 github.com/hashicorp/go-bexpr v0.1.2 github.com/hashicorp/go-checkpoint v0.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 diff --git a/test-integ/go.mod b/test-integ/go.mod index 072dda44acf0..7a91a5a20e5c 100644 --- a/test-integ/go.mod +++ b/test-integ/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.5 require ( github.com/google/go-cmp v0.5.9 - github.com/hashicorp/consul/api v1.29.1 - github.com/hashicorp/consul/proto-public v0.6.1 + github.com/hashicorp/consul/api v1.29.4 + github.com/hashicorp/consul/proto-public v0.6.2 github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/consul/test/integration/consul-container v0.0.0-20230628201853-bdf4fad7c5a5 github.com/hashicorp/consul/testing/deployer v0.0.0-20230811171106-4a0afb5d1373 diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod index d1e077db52a4..e7803fa6e99a 100644 --- a/test/integration/consul-container/go.mod +++ b/test/integration/consul-container/go.mod @@ -12,9 +12,9 @@ require ( github.com/evanphx/json-patch v4.12.0+incompatible github.com/go-jose/go-jose/v3 v3.0.3 github.com/hashicorp/consul v1.16.1 - github.com/hashicorp/consul/api v1.29.1 - github.com/hashicorp/consul/envoyextensions v0.7.0 - github.com/hashicorp/consul/proto-public v0.6.1 + github.com/hashicorp/consul/api v1.29.4 + github.com/hashicorp/consul/envoyextensions v0.7.3 + github.com/hashicorp/consul/proto-public v0.6.2 github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/consul/testing/deployer v0.0.0-20230811171106-4a0afb5d1373 github.com/hashicorp/go-cleanhttp v0.5.2 diff --git a/testing/deployer/go.mod b/testing/deployer/go.mod index 0dd3855e3b9e..e1bd6711a33a 100644 --- a/testing/deployer/go.mod +++ b/testing/deployer/go.mod @@ -6,8 +6,8 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/google/go-cmp v0.5.9 github.com/hashicorp/consul-server-connection-manager v0.1.4 - github.com/hashicorp/consul/api v1.26.1 - github.com/hashicorp/consul/proto-public v0.6.1 + github.com/hashicorp/consul/api v1.29.4 + github.com/hashicorp/consul/proto-public v0.6.2 github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-hclog v1.5.0 diff --git a/troubleshoot/go.mod b/troubleshoot/go.mod index b6c579ee3a2f..753fd9e09955 100644 --- a/troubleshoot/go.mod +++ b/troubleshoot/go.mod @@ -23,8 +23,8 @@ retract ( require ( github.com/envoyproxy/go-control-plane v0.12.0 github.com/envoyproxy/go-control-plane/xdsmatcher v0.0.0-20230524161521-aaaacbfbe53e - github.com/hashicorp/consul/api v1.29.1 - github.com/hashicorp/consul/envoyextensions v0.7.0 + github.com/hashicorp/consul/api v1.29.4 + github.com/hashicorp/consul/envoyextensions v0.7.3 github.com/hashicorp/consul/sdk v0.16.1 github.com/stretchr/testify v1.8.4 google.golang.org/protobuf v1.33.0 diff --git a/version/VERSION b/version/VERSION index 734375f897d0..ece450e8fe9b 100644 --- a/version/VERSION +++ b/version/VERSION @@ -1 +1 @@ -1.20.0-dev +1.19.3-dev From d12f9cf4d1617d3128e28a3199cc8de3ccd8475f Mon Sep 17 00:00:00 2001 From: Jorge Marey <6938602+jorgemarey@users.noreply.github.com> Date: Thu, 29 Aug 2024 18:51:44 +0200 Subject: [PATCH 6/9] Set replication metric to 0 when losing leadership (#20665) * Set replication metric to 0 when losing leadership * Fix replication metrics on replication.go also --------- Co-authored-by: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> --- agent/consul/leader.go | 6 ++++++ agent/consul/replication.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/agent/consul/leader.go b/agent/consul/leader.go index 53312c7fe53c..f8340d2b32c5 100644 --- a/agent/consul/leader.go +++ b/agent/consul/leader.go @@ -771,6 +771,12 @@ func (s *Server) runACLReplicator( index, exit, err := replicateFunc(ctx, logger, lastRemoteIndex) if exit { + metrics.SetGauge([]string{"leader", "replication", metricName, "status"}, + 0, + ) + metrics.SetGauge([]string{"leader", "replication", metricName, "index"}, + 0, + ) return nil } diff --git a/agent/consul/replication.go b/agent/consul/replication.go index 08b8811129be..ebed1377e1ff 100644 --- a/agent/consul/replication.go +++ b/agent/consul/replication.go @@ -153,6 +153,12 @@ func (r *Replicator) Run(ctx context.Context) error { // Perform a single round of replication index, exit, err := r.delegate.Replicate(ctx, atomic.LoadUint64(&r.lastRemoteIndex), r.logger) if exit { + metrics.SetGauge([]string{"leader", "replication", r.delegate.MetricName(), "status"}, + 0, + ) + metrics.SetGauge([]string{"leader", "replication", r.delegate.MetricName(), "index"}, + 0, + ) return nil } if err != nil { From c1d0fc938a7573df620d805f020da45358c6a2c9 Mon Sep 17 00:00:00 2001 From: Aimee Ukasick Date: Thu, 29 Aug 2024 11:57:32 -0500 Subject: [PATCH 7/9] Docs CE-709: Remove circular links (#21685) Docs CE-70: Remove circular links Remove links to tutorials that no longer exist and redirect back to the ACL overview page. --- website/content/docs/security/acl/index.mdx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/website/content/docs/security/acl/index.mdx b/website/content/docs/security/acl/index.mdx index e29ac995e5cf..f8b8b97ec85c 100644 --- a/website/content/docs/security/acl/index.mdx +++ b/website/content/docs/security/acl/index.mdx @@ -9,15 +9,6 @@ description: >- This topic describes core concepts associated with the optional access control list (ACL) system shipped with Consul. ACLs authenticate requests and authorize access to resources. They also control access to the Consul UI, API, and CLI, as well as secure service-to-service and agent-to-agent communication. -Refer to the following tutorials for step-by-step instructions on how to get started using ACLs: - -- [Bootstrap and Explore ACLs] -- [Secure Consul with ACLs] -- [Troubleshoot the ACL System](/consul/tutorials/security/access-control-troubleshoot) - -[bootstrap and explore acls]: /consul/tutorials/security/access-control-setup-production?utm_source=docs -[secure consul with acls]: /consul/tutorials/security/access-control-setup-production - Refer to the [ACL API reference](/consul/api-docs/acl) and [ACL CLI reference](/consul/commands/acl) for additional usage information. ## Workflow overview From 64683180f3a72dacb0b8ae01503acbb4d97cb504 Mon Sep 17 00:00:00 2001 From: Deniz Onur Duzgun <59659739+dduzgun-security@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:04:51 -0400 Subject: [PATCH 8/9] security(deps): bump aws-sdk-go to v1.55.5 (#21684) * security(deps): bump aws-sdk-go to v1.55.5 * add changelog * edit changelog --- .changelog/21684.txt | 6 ++++++ go.mod | 2 +- go.sum | 8 ++------ 3 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 .changelog/21684.txt diff --git a/.changelog/21684.txt b/.changelog/21684.txt new file mode 100644 index 000000000000..3702737cb09a --- /dev/null +++ b/.changelog/21684.txt @@ -0,0 +1,6 @@ +```release-note:security +Upgrade to support aws/aws-sdk-go `v1.55.5 or higher`. This resolves CVEs +[CVE-2020-8911](https://nvd.nist.gov/vuln/detail/cve-2020-8911) and +[CVE-2020-8912](https://nvd.nist.gov/vuln/detail/cve-2020-8912). +``` + diff --git a/go.mod b/go.mod index 1feee6383ceb..649a591606cc 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/go-metrics v0.4.1 github.com/armon/go-radix v1.0.0 - github.com/aws/aws-sdk-go v1.44.289 + github.com/aws/aws-sdk-go v1.55.5 github.com/coreos/go-oidc/v3 v3.9.0 github.com/deckarep/golang-set/v2 v2.3.1 github.com/docker/go-connections v0.4.0 diff --git a/go.sum b/go.sum index 86dbf3e57d07..89ac14fdd68e 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.289 h1:5CVEjiHFvdiVlKPBzv0rjG4zH/21W/onT18R5AH/qx0= -github.com/aws/aws-sdk-go v1.44.289/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA= github.com/benbjohnson/immutable v0.4.0/go.mod h1:iAr8OjJGLnLmVUr9MZ/rz4PWUy6Ouc2JLYuMArmvAJM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -1013,7 +1013,6 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= @@ -1125,7 +1124,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1134,7 +1132,6 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -1150,7 +1147,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= From 188af1ccb0f366464fc86438fefe67ceb2e5d8b3 Mon Sep 17 00:00:00 2001 From: Michael Zalimeni Date: Fri, 30 Aug 2024 16:25:27 -0400 Subject: [PATCH 9/9] test: fix Envoy int tests and add container logs (#21674) Correctly set the the version of Consul built by the `dev-build` job, which is then copied into the Consul dev image used in integration tests. This was causing failures starting sidecar proxies via `consul connect envoy` due to a mismatch between the (incorrect) Consul binary's supported Envoy versions and the (correct) Envoy version under test. Also add debug log uploads to each int test so we can more easily diagnose this sort of failure in the future, as it was entirely hidden in test output. --- .../nightly-test-integrations-1.15.x.yml | 21 +++++++++++++++++++ .../nightly-test-integrations-1.17.x.yml | 18 ++++++++++++++++ .../nightly-test-integrations-1.18.x.yml | 20 +++++++++++++++++- .../nightly-test-integrations-1.19.x.yml | 20 +++++++++++++++++- .github/workflows/test-integrations.yml | 18 ++++++++++++++++ 5 files changed, 95 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-test-integrations-1.15.x.yml b/.github/workflows/nightly-test-integrations-1.15.x.yml index 8ac92f282f34..abb24520efde 100644 --- a/.github/workflows/nightly-test-integrations-1.15.x.yml +++ b/.github/workflows/nightly-test-integrations-1.15.x.yml @@ -146,11 +146,15 @@ jobs: path: ./bin - name: restore mode+x run: chmod +x ./bin/consul + - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + - name: Docker build run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + - name: Envoy Integration Tests + id: envoy-integration-tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose @@ -171,6 +175,23 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + # See https://github.com/orgs/community/discussions/8945#discussioncomment-9897011 + # and overall topic discussion for why this is necessary. + - name: Generate artifact ID + id: generate-artifact-id + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + run: | + ARTIFACT_ID=$(uuidgen) + echo "Artifact ID: $ARTIFACT_ID (search this in job summary for download link)" + echo "artifact_id=$ARTIFACT_ID" >> "$GITHUB_ENV" + + - name: Upload failure logs + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: envoy-${{ matrix.envoy-version }}-logs-${{ env.artifact_id }} + path: test/integration/connect/envoy/workdir/logs/ + # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} diff --git a/.github/workflows/nightly-test-integrations-1.17.x.yml b/.github/workflows/nightly-test-integrations-1.17.x.yml index df952d8234fa..471cdb163f04 100644 --- a/.github/workflows/nightly-test-integrations-1.17.x.yml +++ b/.github/workflows/nightly-test-integrations-1.17.x.yml @@ -154,6 +154,7 @@ jobs: run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests + id: envoy-integration-tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose @@ -174,6 +175,23 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + # See https://github.com/orgs/community/discussions/8945#discussioncomment-9897011 + # and overall topic discussion for why this is necessary. + - name: Generate artifact ID + id: generate-artifact-id + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + run: | + ARTIFACT_ID=$(uuidgen) + echo "Artifact ID: $ARTIFACT_ID (search this in job summary for download link)" + echo "artifact_id=$ARTIFACT_ID" >> "$GITHUB_ENV" + + - name: Upload failure logs + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: envoy-${{ matrix.envoy-version }}-logs-${{ env.artifact_id }} + path: test/integration/connect/envoy/workdir/logs/ + # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} diff --git a/.github/workflows/nightly-test-integrations-1.18.x.yml b/.github/workflows/nightly-test-integrations-1.18.x.yml index 59df23bb814d..2d358cda69ed 100644 --- a/.github/workflows/nightly-test-integrations-1.18.x.yml +++ b/.github/workflows/nightly-test-integrations-1.18.x.yml @@ -68,7 +68,7 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-large }} repository-name: ${{ github.repository }} uploaded-binary-name: 'consul-bin' - branch-name: "release/1.17.x" + branch-name: "release/1.18.x" go-version: ${{ needs.get-go-version.outputs.go-version }} secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} @@ -154,6 +154,7 @@ jobs: run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests + id: envoy-integration-tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose @@ -174,6 +175,23 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + # See https://github.com/orgs/community/discussions/8945#discussioncomment-9897011 + # and overall topic discussion for why this is necessary. + - name: Generate artifact ID + id: generate-artifact-id + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + run: | + ARTIFACT_ID=$(uuidgen) + echo "Artifact ID: $ARTIFACT_ID (search this in job summary for download link)" + echo "artifact_id=$ARTIFACT_ID" >> "$GITHUB_ENV" + + - name: Upload failure logs + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: envoy-${{ matrix.envoy-version }}-logs-${{ env.artifact_id }} + path: test/integration/connect/envoy/workdir/logs/ + # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} diff --git a/.github/workflows/nightly-test-integrations-1.19.x.yml b/.github/workflows/nightly-test-integrations-1.19.x.yml index 452d174e1eca..327907d184d3 100644 --- a/.github/workflows/nightly-test-integrations-1.19.x.yml +++ b/.github/workflows/nightly-test-integrations-1.19.x.yml @@ -59,7 +59,7 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-large }} repository-name: ${{ github.repository }} uploaded-binary-name: 'consul-bin' - branch-name: "release/1.17.x" + branch-name: "release/1.19.x" go-version: ${{ needs.get-go-version.outputs.go-version }} secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} @@ -145,6 +145,7 @@ jobs: run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests + id: envoy-integration-tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose @@ -165,6 +166,23 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + # See https://github.com/orgs/community/discussions/8945#discussioncomment-9897011 + # and overall topic discussion for why this is necessary. + - name: Generate artifact ID + id: generate-artifact-id + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + run: | + ARTIFACT_ID=$(uuidgen) + echo "Artifact ID: $ARTIFACT_ID (search this in job summary for download link)" + echo "artifact_id=$ARTIFACT_ID" >> "$GITHUB_ENV" + + - name: Upload failure logs + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: envoy-${{ matrix.envoy-version }}-logs-${{ env.artifact_id }} + path: test/integration/connect/envoy/workdir/logs/ + # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }} diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index a6fa149bba35..8b14ec8e4d58 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -340,6 +340,7 @@ jobs: run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin - name: Envoy Integration Tests + id: envoy-integration-tests env: GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml GOTESTSUM_FORMAT: standard-verbose @@ -360,6 +361,23 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + # See https://github.com/orgs/community/discussions/8945#discussioncomment-9897011 + # and overall topic discussion for why this is necessary. + - name: Generate artifact ID + id: generate-artifact-id + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + run: | + ARTIFACT_ID=$(uuidgen) + echo "Artifact ID: $ARTIFACT_ID (search this in job summary for download link)" + echo "artifact_id=$ARTIFACT_ID" >> "$GITHUB_ENV" + + - name: Upload failure logs + if: ${{ failure() && steps.envoy-integration-tests.conclusion == 'failure' }} + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: envoy-${{ matrix.envoy-version }}-logs-${{ env.artifact_id }} + path: test/integration/connect/envoy/workdir/logs/ + # NOTE: ENT specific step as we store secrets in Vault. - name: Authenticate to Vault if: ${{ !cancelled() && endsWith(github.repository, '-enterprise') }}