Skip to content

Commit

Permalink
Synchronize operator source code with kubebuilder scafolding project
Browse files Browse the repository at this point in the history
Clean the kustomization.
Remove unused kustomization `default-with-metrics`.
Remove `kube-rbac-proxy` container from kustomization and helm chart.
  • Loading branch information
RafalKorepta committed Jan 31, 2025
1 parent d811c41 commit 25e4fe2
Show file tree
Hide file tree
Showing 33 changed files with 665 additions and 462 deletions.
26 changes: 26 additions & 0 deletions acceptance/features/operator.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
Feature: Metrics endpoint has authentication and authorization

@skip:gke @skip:aks @skip:eks
Scenario: Reject request without TLS
Given operator is running
Then metrics endpoint should reject http request with status code "400"

@skip:gke @skip:aks @skip:eks
Scenario: Reject unauthenticated token
Given operator is running
Then metrics endpoint should reject authorization random token request with status code "500"

@skip:gke @skip:aks @skip:eks
Scenario: Accept request
Given operator is running
When I apply Kubernetes manifest:
"""
apiVersion: v1
kind: ServiceAccount
metadata:
name: testing
"""
And "testing" service account has bounded "redpanda-operator-metrics-reader" cluster role
Then metrics endpoint should accept https request with "testing" service account token


122 changes: 122 additions & 0 deletions acceptance/steps/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,30 @@ package steps

import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"slices"
"strconv"
"strings"
"time"

"github.com/cucumber/godog"
"github.com/prometheus/common/expfmt"
"github.com/redpanda-data/common-go/rpadmin"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/twmb/franz-go/pkg/kadm"
"github.com/twmb/franz-go/pkg/kgo"
"github.com/twmb/franz-go/pkg/sr"
"golang.org/x/exp/rand"
appsv1 "k8s.io/api/apps/v1"
authenticationv1 "k8s.io/api/authentication/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/utils/ptr"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -354,3 +363,116 @@ func checkStableResource(ctx context.Context, t framework.TestingT, o runtimecli
}, 30*time.Second, 1*time.Second, "Resource never stabilized")
t.Logf("Resource %q has been stable for 5 seconds", key.String())
}

type operatorClients struct {
client http.Client
operatorPodName string
namespace string
schema string
token string
expectedStatusCode int
}

func (c *operatorClients) ExpectRequestRejected(ctx context.Context) {
t := framework.T(ctx)

url := fmt.Sprintf("%s://%s.%s:8443/metrics", c.schema, c.operatorPodName, c.namespace)

t.Logf("Request %s to operator", url)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
require.NoError(t, err)

req.Header.Set("Authorization", "Bearer "+c.token)
resp, err := c.client.Do(req)
require.NoError(t, err)

require.Equal(t, c.expectedStatusCode, resp.StatusCode)

defer resp.Body.Close()
}

func (c *operatorClients) ExpectCorrectMetricsResponse(ctx context.Context) {
t := framework.T(ctx)

url := fmt.Sprintf("%s://%s.%s:8443/metrics", c.schema, c.operatorPodName, c.namespace)

t.Logf("Request %s to operator", url)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
require.NoError(t, err)

req.Header.Set("Authorization", "Bearer "+c.token)
resp, err := c.client.Do(req)
require.NoError(t, err)

require.Equal(t, http.StatusOK, resp.StatusCode)

defer resp.Body.Close()

var parser expfmt.TextParser
_, err = parser.TextToMetricFamilies(resp.Body)
require.NoError(t, err)
}

func clientsForOperator(ctx context.Context, includeTLS bool, serviceAccountName, expectedStatusCode string) *operatorClients {
t := framework.T(ctx)

var dep appsv1.Deployment
require.NoError(t, t.Get(ctx, t.ResourceKey("redpanda-operator"), &dep))

var podList corev1.PodList

require.NoError(t, t.List(ctx, &podList, &runtimeclient.ListOptions{
LabelSelector: labels.SelectorFromSet(dep.Spec.Selector.MatchLabels),
}))

require.Len(t, podList.Items, 1, "expected 1 pod, got %d", len(podList.Items))

var tlsCfg tls.Config
schema := "http"
if includeTLS {
tlsCfg = tls.Config{InsecureSkipVerify: includeTLS} // nolint:gosec
schema = "https"
}

token := randomString(20)
if serviceAccountName != "" {
cs, err := kubernetes.NewForConfig(t.RestConfig())
require.NoError(t, err)
tokenResponse, err := cs.CoreV1().ServiceAccounts(t.Namespace()).CreateToken(ctx, serviceAccountName, &authenticationv1.TokenRequest{}, metav1.CreateOptions{})
require.NoError(t, err)
token = tokenResponse.Status.Token
}

statusCode := http.StatusOK
if expectedStatusCode != "" {
var err error
statusCode, err = strconv.Atoi(expectedStatusCode)
require.NoError(t, err)
}

return &operatorClients{
expectedStatusCode: statusCode,
token: token,
schema: schema,
namespace: t.Namespace(),
operatorPodName: podList.Items[0].Name,
client: http.Client{Transport: &http.Transport{
TLSClientConfig: &tlsCfg,
DialContext: kube.NewPodDialer(t.RestConfig()).DialContext,
}},
}
}

func init() {
rand.Seed(uint64(time.Now().UnixNano()))
}

var letters = "0123456789abcdefghijklmnopqrstuvwxyz"

func randomString(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
78 changes: 78 additions & 0 deletions acceptance/steps/operator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2025 Redpanda Data, Inc.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.md
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0

package steps

import (
"context"

"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

framework "github.com/redpanda-data/redpanda-operator/harpoon"
)

func operatorIsRunning(ctx context.Context, t framework.TestingT) {
var dep appsv1.Deployment
require.NoError(t, t.Get(ctx, t.ResourceKey("redpanda-operator"), &dep))

// make sure the resource is stable
checkStableResource(ctx, t, &dep)

require.Equal(t, dep.Status.AvailableReplicas, int32(1))
require.Equal(t, dep.Status.Replicas, int32(1))
require.Equal(t, dep.Status.ReadyReplicas, int32(1))
require.Equal(t, dep.Status.UnavailableReplicas, int32(0))
}

func requestMetricsEndpointPlainHTTP(ctx context.Context, statusCode string) {
clientsForOperator(ctx, false, "", statusCode).ExpectRequestRejected(ctx)
}

func requestMetricsEndpointWithTLSAndRandomToken(ctx context.Context, statusCode string) {
clientsForOperator(ctx, true, "", statusCode).ExpectRequestRejected(ctx)
}

func acceptServiceAccountMetricsRequest(ctx context.Context, serviceAccountName string) {
clientsForOperator(ctx, true, serviceAccountName, "").ExpectCorrectMetricsResponse(ctx)
}

func createClusterRoleBinding(ctx context.Context, serviceAccountName, clusterRoleName string) {
t := framework.T(ctx)

require.NoError(t, t.Create(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Namespace: t.Namespace(),
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "ClusterRole",
Name: clusterRoleName,
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: serviceAccountName,
Namespace: t.Namespace(),
},
},
}))

t.Cleanup(func(ctx context.Context) {
require.NoError(t, t.Delete(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Namespace: t.Namespace(),
},
}))
})
}
7 changes: 7 additions & 0 deletions acceptance/steps/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,11 @@ func init() {
framework.RegisterStep(`^"([^"]*)" should exist and be able to authenticate to the "([^"]*)" cluster$`, shouldExistAndBeAbleToAuthenticateToTheCluster)
framework.RegisterStep(`^"([^"]*)" should be able to authenticate to the "([^"]*)" cluster with password "([^"]*)" and mechanism "([^"]*)"$`, shouldBeAbleToAuthenticateToTheClusterWithPasswordAndMechanism)
framework.RegisterStep(`^there should be ACLs in the cluster "([^"]*)" for user "([^"]*)"$`, thereShouldBeACLsInTheClusterForUser)

// Operator scenario steps
framework.RegisterStep(`^operator is running$`, operatorIsRunning)
framework.RegisterStep(`^metrics endpoint should reject http request with status code "([^"]*)"$`, requestMetricsEndpointPlainHTTP)
framework.RegisterStep(`^metrics endpoint should reject authorization random token request with status code "([^"]*)"$`, requestMetricsEndpointWithTLSAndRandomToken)
framework.RegisterStep(`^"([^"]*)" service account has bounded "([^"]*)" cluster role$`, createClusterRoleBinding)
framework.RegisterStep(`^metrics endpoint should accept https request with "([^"]*)" service account token$`, acceptServiceAccountMetricsRequest)
}
52 changes: 10 additions & 42 deletions charts/operator/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,48 +119,9 @@ func operatorContainers(dot *helmette.Dot, podTerminationGracePeriodSeconds *int
ReadinessProbe: readinessProbe(dot, podTerminationGracePeriodSeconds),
Resources: values.Resources,
},
{
Name: "kube-rbac-proxy",
Args: []string{
"--secure-listen-address=0.0.0.0:8443",
"--upstream=http://127.0.0.1:8080/",
"--logtostderr=true",
fmt.Sprintf("--v=%d", values.KubeRBACProxy.LogLevel),
},
Image: fmt.Sprintf("%s:%s", values.KubeRBACProxy.Image.Repository, *values.KubeRBACProxy.Image.Tag),
ImagePullPolicy: values.KubeRBACProxy.Image.PullPolicy,
Ports: []corev1.ContainerPort{
{
ContainerPort: 8443,
Name: "https",
},
},
VolumeMounts: kubeRBACProxyVolumeMounts(dot),
},
}
}

func kubeRBACProxyVolumeMounts(dot *helmette.Dot) []corev1.VolumeMount {
values := helmette.Unwrap[Values](dot.Values)

if !values.ServiceAccount.Create {
return nil
}

mountName := ServiceAccountVolumeName
for _, vol := range operatorPodVolumes(dot) {
if strings.HasPrefix(ServiceAccountVolumeName+"-", vol.Name) {
mountName = vol.Name
}
}

return []corev1.VolumeMount{{
Name: mountName,
ReadOnly: true,
MountPath: DefaultAPITokenMountPath,
}}
}

func livenessProbe(dot *helmette.Dot, podTerminationGracePeriodSeconds *int64) *corev1.Probe {
values := helmette.Unwrap[Values](dot.Values)

Expand Down Expand Up @@ -250,7 +211,7 @@ func operatorPodVolumes(dot *helmette.Dot) []corev1.Volume {
vol = append(vol, kubeTokenAPIVolume(ServiceAccountVolumeName))
}

if !values.Webhook.Enabled {
if !isWebhookEnabled(dot) {
return vol
}

Expand Down Expand Up @@ -337,7 +298,7 @@ func operatorPodVolumesMounts(dot *helmette.Dot) []corev1.VolumeMount {
})
}

if !values.Webhook.Enabled {
if !isWebhookEnabled(dot) {
return volMount
}

Expand All @@ -355,11 +316,18 @@ func operatorArguments(dot *helmette.Dot) []string {

args := []string{
"--health-probe-bind-address=:8081",
"--metrics-bind-address=127.0.0.1:8080",
"--metrics-bind-address=:8443",
"--leader-elect",
fmt.Sprintf("--webhook-enabled=%t", isWebhookEnabled(dot)),
}

if isWebhookEnabled(dot) {
args = append(args,
"--webhook-enabled=true",
"--webhook-cert-path=/tmp/k8s-webhook-server/serving-certs",
)
}

if values.Scope == Namespace {
args = append(args,
fmt.Sprintf("--namespace=%s", dot.Release.Namespace),
Expand Down
Loading

0 comments on commit 25e4fe2

Please sign in to comment.