From 463081e323829f8e365f1cbde307d96594baadaa Mon Sep 17 00:00:00 2001 From: Ori Shavit Date: Wed, 7 Jun 2023 14:33:03 +0300 Subject: [PATCH 1/2] Update docs to how kafka watcher will work in the future. --- .../network-mapper/helm-chart.mdx | 13 --- .../network-mapper/kafka-watcher.mdx | 27 +----- .../kafka-mtls/helm/values_debug_logging.yaml | 90 +++++++------------ 3 files changed, 31 insertions(+), 99 deletions(-) diff --git a/docs/reference/configuration/network-mapper/helm-chart.mdx b/docs/reference/configuration/network-mapper/helm-chart.mdx index 8eea4f90c..fbbd65fad 100644 --- a/docs/reference/configuration/network-mapper/helm-chart.mdx +++ b/docs/reference/configuration/network-mapper/helm-chart.mdx @@ -33,19 +33,6 @@ Checkout the network mapper [tutorial](/quick-tutorials/k8s-network-mapper) to s | `sniffer.tolerations` | Tolerations override. | `(none)` | | `sniffer.priorityClassName` | Set priorityClassName. | `(none)` | - -## Kafka watcher parameters -| Key | Description | Default | -|---------------------------------|-------------------------------------------------------------|--------------------------------| -| `kafkawatcher.enable` | Enable Kafka watcher deployment (beta). | `false` | -| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | -| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | -| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | -| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | -| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | -| `kafkawatcher.resources` | Resources override. | `(none)` | -| `kafkawatcher.kafkaServers` | Kafka servers to watch, specified as `pod.namespace` items. | `(none)` | - ## Istio watcher parameters | Key | Description | Default | |---------------------------------|-----------------------------------------|--------------------------------| diff --git a/docs/reference/configuration/network-mapper/kafka-watcher.mdx b/docs/reference/configuration/network-mapper/kafka-watcher.mdx index 160f6b45a..49b617183 100644 --- a/docs/reference/configuration/network-mapper/kafka-watcher.mdx +++ b/docs/reference/configuration/network-mapper/kafka-watcher.mdx @@ -3,32 +3,7 @@ sidebar_position: 2 title: Kafka Watcher --- -To deploy the network mapper with the Kafka watcher component, do the following: -```bash -helm repo add otterize https://helm.otterize.com -helm repo update -helm install network-mapper otterize/network-mapper -n otterize-system --create-namespace --set kafkawatcher.enable=true -``` -Make sure to include `--set kafkaServers={}` and provide a list of Kafka servers whose logs the Kafka watcher should watch. -Servers in the list should be specified as `name.namespace`. - -## Kafka watcher parameters -| Key | Description | Default | -|---------------------------------|-------------------------------------------------------------|--------------------------------| -| `kafkawatcher.enable` | Enable Kafka watcher deployment (beta). | `false` | -| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | -| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | -| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | -| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | -| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | -| `kafkawatcher.resources` | Resources override. | `(none)` | -| `kafkawatcher.kafkaServers` | Kafka servers to watch, specified as `pod.namespace` items. | `(none)` | - -## Enabling debug logs in Kafka servers -The Kafka watcher periodically examines logs of Kafka servers provided by the user through configuration, -parses them and deduces topic-level access to Kafka from pods in the Kubernetes cluster. -In order for the Kafka watcher to correctly examine topic-level access, the Kafka server's ACL authorizer logger should be configured -to log at debug level, and to stdout. +The Kafka Watcher is deployed in a Kubernetes environment as a sidecar container in the Kafka pod. Kafka is configured as below to generate logs from authorization decisions and write them to a volume shared with the Watcher container. The Kafka Watcher collects these logs and sends them to the network mapper. ### Install Kafka via Helm with debug logs preconfigured For the Bitnami Kafka Helm chart used in other Kafka tutorials, we can add the following configuration to the chart's diff --git a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml index 8a6bba0c1..d5a7dddf1 100644 --- a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml +++ b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml @@ -32,74 +32,44 @@ resources: requests: cpu: 50m memory: 256Mi +sidecars: + - name: otterize-sidecar + image: otterize/network-mapper-kafka-watcher:dev + imagePullPolicy: Never + volumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs + env: + - name: OTTERIZE_MAPPER_API_URL + value: http://otterize-network-mapper.otterize-system.svc:9090/query + - name: OTTERIZE_DEBUG + value: "False" + - name: OTTERIZE_POD + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTTERIZE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +extraVolumes: + - name: kafka-authz-logs + emptyDir: +extraVolumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs log4j: | - # Unspecified loggers and loggers with additivity=true output to server.log and stdout - # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise - - log4j.rootLogger=INFO, stdout, kafkaAppender + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - log4j.appender.kafkaAppender=org.apache.log4j.ConsoleAppender - log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.stateChangeAppender=org.apache.log4j.ConsoleAppender - log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.requestAppender=org.apache.log4j.ConsoleAppender - log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.cleanerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.controllerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.authorizerAppender=org.apache.log4j.ConsoleAppender + log4j.appender.authorizerAppender=org.apache.log4j.FileAppender log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - - # Change the line below to adjust ZK client logging - log4j.logger.org.apache.zookeeper=INFO - - # Change the two lines below to adjust the general broker logging level (output to server.log and stdout) - log4j.logger.kafka=INFO, stdout - log4j.logger.org.apache.kafka=INFO - - # Change to DEBUG or TRACE to enable request logging - log4j.logger.kafka.request.logger=WARN, requestAppender - log4j.additivity.kafka.request.logger=false - - # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output - # related to the handling of requests - #log4j.logger.kafka.network.Processor=TRACE, requestAppender - #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender - #log4j.additivity.kafka.server.KafkaApis=false - log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender - log4j.additivity.kafka.network.RequestChannel$=false - - # Change the line below to adjust KRaft mode controller logging - log4j.logger.org.apache.kafka.controller=INFO, controllerAppender - log4j.additivity.org.apache.kafka.controller=false - - # Change the line below to adjust ZK mode controller logging - log4j.logger.kafka.controller=TRACE, controllerAppender - log4j.additivity.kafka.controller=false - - log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender - log4j.additivity.kafka.log.LogCleaner=false - - log4j.logger.state.change.logger=INFO, stateChangeAppender - log4j.additivity.state.change.logger=false + log4j.appender.authorizerAppender.file=/opt/otterize/kafka-watcher/authz.log # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender - log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file + log4j.additivity.kafka.authorizer.logger=false From 1818ff6589993629adc2b64a3a4cf81271b91690 Mon Sep 17 00:00:00 2001 From: Ori Shoshan Date: Fri, 25 Aug 2023 14:29:19 +0200 Subject: [PATCH 2/2] Update docs to support both modes --- ...nforcement-and-kafka-watcher-no-netpols.md | 2 +- ...-otterize-no-netpols-with-kafka-watcher.md | 2 +- .../network-mapper/helm-chart.mdx | 13 ++ .../network-mapper/kafka-watcher.mdx | 173 ++++++------------ .../kafka-mtls/helm/values_debug_logging.yaml | 30 +-- .../values_debug_logging_k8s_watcher.yaml | 75 ++++++++ 6 files changed, 147 insertions(+), 148 deletions(-) create mode 100644 static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml diff --git a/docs/_common/install-otterize-from-cloud-with-enforcement-and-kafka-watcher-no-netpols.md b/docs/_common/install-otterize-from-cloud-with-enforcement-and-kafka-watcher-no-netpols.md index 5dd8202db..d12d1c0d4 100644 --- a/docs/_common/install-otterize-from-cloud-with-enforcement-and-kafka-watcher-no-netpols.md +++ b/docs/_common/install-otterize-from-cloud-with-enforcement-and-kafka-watcher-no-netpols.md @@ -5,7 +5,7 @@ If no Kubernetes clusters are connected to your account, click the "connect your 2. And add the following flags to the Helm command: ``` --set intentsOperator.operator.enableNetworkPolicyCreation=false \ - --set networkMapper.kafkawatcher.enable=true \ + --set networkMapper.kafkawatcher.enableK8sWatchMode=true \ --set networkMapper.kafkawatcher.kafkaServers={"kafka-0.kafka"} ``` diff --git a/docs/_common/install-otterize-no-netpols-with-kafka-watcher.md b/docs/_common/install-otterize-no-netpols-with-kafka-watcher.md index 64e35a9f1..2ccd290eb 100644 --- a/docs/_common/install-otterize-no-netpols-with-kafka-watcher.md +++ b/docs/_common/install-otterize-no-netpols-with-kafka-watcher.md @@ -3,7 +3,7 @@ helm repo update helm install otterize otterize/otterize-kubernetes -n otterize-system --create-namespace \ --set intentsOperator.operator.enableNetworkPolicyCreation=false \ - --set networkMapper.kafkawatcher.enable=true \ + --set networkMapper.kafkawatcher.enableK8sWatchMode=true \ --set networkMapper.kafkawatcher.kafkaServers={"kafka-0.kafka"} ``` This chart is a bundle of the Otterize intents operator, Otterize credentials operator, Otterize network mapper, and SPIRE. diff --git a/docs/reference/configuration/network-mapper/helm-chart.mdx b/docs/reference/configuration/network-mapper/helm-chart.mdx index fbbd65fad..beb3c7ca6 100644 --- a/docs/reference/configuration/network-mapper/helm-chart.mdx +++ b/docs/reference/configuration/network-mapper/helm-chart.mdx @@ -33,6 +33,19 @@ Checkout the network mapper [tutorial](/quick-tutorials/k8s-network-mapper) to s | `sniffer.tolerations` | Tolerations override. | `(none)` | | `sniffer.priorityClassName` | Set priorityClassName. | `(none)` | + +## Kafka watcher parameters +| Key | Description | Default | +|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------|--------------------------------| +| `kafkawatcher.enableK8sWatchModeK8sWatchMode` | Enable Kafka watcher deployment, which reads Kafka logs using the Kubernetes API, for pods in `kafkaServers` | `false` | +| `kafkawatcher.kafkaServers` | Kafka servers to watch, formatted as a list of `pod.namespace`. For example, `["kafka-0.kafka"]` | `(none)` | +| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | +| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | +| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | +| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | +| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | +| `kafkawatcher.resources` | Resources override. | `(none)` | + ## Istio watcher parameters | Key | Description | Default | |---------------------------------|-----------------------------------------|--------------------------------| diff --git a/docs/reference/configuration/network-mapper/kafka-watcher.mdx b/docs/reference/configuration/network-mapper/kafka-watcher.mdx index 49b617183..f063d342b 100644 --- a/docs/reference/configuration/network-mapper/kafka-watcher.mdx +++ b/docs/reference/configuration/network-mapper/kafka-watcher.mdx @@ -1,133 +1,70 @@ --- sidebar_position: 2 -title: Kafka Watcher +title: Kafka watcher --- -The Kafka Watcher is deployed in a Kubernetes environment as a sidecar container in the Kafka pod. Kafka is configured as below to generate logs from authorization decisions and write them to a volume shared with the Watcher container. The Kafka Watcher collects these logs and sends them to the network mapper. +The Kafka watcher periodically examines logs of Kafka brokers provided by the user through configuration, +parses them and deduces topic-level access to Kafka from pods in the Kubernetes cluster. -### Install Kafka via Helm with debug logs preconfigured -For the Bitnami Kafka Helm chart used in other Kafka tutorials, we can add the following configuration to the chart's -`values.yaml` to start Kafka with its ACL authorizer logging to stdout at debug level: +## Enabling debug logs for the Bitnami Kafka chart +In order for the Kafka watcher to be able to examine topic-level access, the Kafka server's ACL authorizer logger must be configured. +We do this by setting the logger for `kafka.authorizer` to DEBUG level. + + +The deployment examples below include examples on how to configure the Kafka logger and Kafka watcher to work together. +The examples are given in the form of a `values.yaml` for the [Bitnami Kafka](https://artifacthub.io/packages/helm/bitnami/kafka) chart, and work with our tutorials. + +## Deployment +The Kafka watcher can be deployed in a Kubernetes environment in two modes. As a sidecar, and as an independent pod that tails Kafka broker logs using the Kubernetes API. + +### Deploy a sidecar container in the Kafka pod +In this mode, Kafka is configured to generate logs from authorization decisions and write them to a volume shared with the Kafka watcher container, which is deployed as a sidecar in the same pod. + +The Kafka watcher collects these logs and sends them to the network mapper. + +#### Configure the Bitnami Kafka chart to output logs to file and the Kafka watcher as a sidecar container
-Kafka debug logs values.yaml + Expand to see the Helm values.yaml used with the Bitnami chart ```yaml -log4j: | - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed with - # this work for additional information regarding copyright ownership. - # The ASF licenses this file to You under the Apache License, Version 2.0 - # (the "License"); you may not use this file except in compliance with - # the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # Unspecified loggers and loggers with additivity=true output to server.log and stdout - # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise - log4j.rootLogger=INFO, stdout, kafkaAppender - - log4j.appender.stdout=org.apache.log4j.ConsoleAppender - log4j.appender.stdout.layout=org.apache.log4j.PatternLayout - log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.kafkaAppender=org.apache.log4j.ConsoleAppender - log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.stateChangeAppender=org.apache.log4j.ConsoleAppender - log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.requestAppender=org.apache.log4j.ConsoleAppender - log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.cleanerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.controllerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.authorizerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - - # Change the line below to adjust ZK client logging - log4j.logger.org.apache.zookeeper=INFO - - # Change the two lines below to adjust the general broker logging level (output to server.log and stdout) - log4j.logger.kafka=INFO, stdout - log4j.logger.org.apache.kafka=INFO - - # Change to DEBUG or TRACE to enable request logging - log4j.logger.kafka.request.logger=WARN, requestAppender - log4j.additivity.kafka.request.logger=false - - # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output - # related to the handling of requests - #log4j.logger.kafka.network.Processor=TRACE, requestAppender - #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender - #log4j.additivity.kafka.server.KafkaApis=false - log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender - log4j.additivity.kafka.network.RequestChannel$=false - - # Change the line below to adjust KRaft mode controller logging - log4j.logger.org.apache.kafka.controller=INFO, controllerAppender - log4j.additivity.org.apache.kafka.controller=false - - # Change the line below to adjust ZK mode controller logging - log4j.logger.kafka.controller=TRACE, controllerAppender - log4j.additivity.kafka.controller=false - - log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender - log4j.additivity.kafka.log.LogCleaner=false - - log4j.logger.state.change.logger=INFO, stateChangeAppender - log4j.additivity.state.change.logger=false - - # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses - log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender - log4j.additivity.kafka.authorizer.logger=false +{@include: ../../../../static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml} ``` - -Notice the `log4j.logger.kafka.authorizer.logger=DEBUG` line that sets the ACL authorizer logger to debug level.
-### Configure an already running Kafka server -Alternatively, we can also configure an already-running Kafka server and set its ACL authorizer logger level to debug. -The Kafka server must be configured to log to stdout so the Kafka watcher could examine its logs. -First, deploy an interactive Kafka client: -```shell -kubectl apply -f https://docs.otterize.com/code-examples/ibac-for-kafka/client-deployment-no-creds.yaml -``` -Connect to the interactive Kafka client using shell (replace the pod name with the name from your cluster): -```shell -kubectl exec -it -n ibac-for-kafka interactive-869fc7b89b-rgmfm -- /bin/bash -``` -Once connected, use the interactive shell to configure the ACL authorizer's logging level: -```shell -$ cd opt/bitnami/kafka/ -# query existing logging settings. Replace "kafka.kafka:9092" with the relevant service name, namespace and port. -$ bin/kafka-configs.sh --bootstrap-server kafka.kafka:9092 --describe --all --entity-type broker-loggers --entity-name 0 | grep authorizer - kafka.security.authorizer.AclAuthorizer=INFO sensitive=false synonyms={} - kafka.authorizer.logger=INFO sensitive=false synonyms={} -# enable authorizer debug logging -$ bin/kafka-configs.sh --bootstrap-server kafka.kafka:9092 --alter --add-config "kafka.authorizer.logger=DEBUG" --entity-type broker-loggers --entity-name 0 -Completed updating config for broker-logger 0. +### Deploy as part of the Otterize deployment, tailing Kafka logs +In this mode, an additional `kafka-watcher` pod is deployed alongside the Otterize network mapper, which uses the Kubernetes API to tail logs outputted to stdout by Kafka brokers in the cluster. +To configure the Kafka brokers, use the `kafkawatcher.kafkaServers` value in the Helm chart. + +To deploy the network mapper with the Kafka watcher component: +```bash +helm repo add otterize https://helm.otterize.com +helm repo update +helm install network-mapper otterize/network-mapper -n otterize-system --create-namespace --set kafkawatcher.enableK8sWatchModeK8sWatchMode=true ``` -Check out your Kafka server logs. You should now see log records indicating allow/denied connections -from the ACL authorizer (assuming you have clients producing/consuming data from topics): -```shell -[2023-03-22 16:06:22,746] DEBUG operation = READ on resource = ResourcePattern(resourceType=TOPIC, name=mytopic, patternType=LITERAL) from host = 10.244.0.12 is ALLOW based on acl = User:* has ALLOW permission for operations: ALL from hosts: * (kafka.authorizer.logger) +Make sure to include `--set kafkawatcher.kafkaServers=` and provide a list of Kafka servers whose logs the Kafka watcher should watch. +Servers in the list should be specified as `name.namespace`. For example, `--set kafkawatcher.kafkaServers=["kafka-0.kafka"]` + + +#### Configure the Bitnami Kafka chart to output logs to stdout for the Kafka watcher to read them +
+Expand to see the Helm values.yaml used with the Bitnami chart + +```yaml +{@include: ../../../../static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml} ``` +
+ + +## Kafka watcher parameters +| Key | Description | Default | +|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------|--------------------------------| +| `kafkawatcher.enableK8sWatchModeK8sWatchMode` | Enable Kafka watcher deployment, which reads Kafka logs using the Kubernetes API, for pods in `kafkaServers` | `false` | +| `kafkawatcher.kafkaServers` | Kafka servers to watch, formatted as a list of `pod.namespace`. For example, `["kafka-0.kafka"]` | `(none)` | +| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | +| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | +| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | +| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | +| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | +| `kafkawatcher.resources` | Resources override. | `(none)` | diff --git a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml index d5a7dddf1..98f1359b4 100644 --- a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml +++ b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml @@ -32,32 +32,6 @@ resources: requests: cpu: 50m memory: 256Mi -sidecars: - - name: otterize-sidecar - image: otterize/network-mapper-kafka-watcher:dev - imagePullPolicy: Never - volumeMounts: - - mountPath: /opt/otterize/kafka-watcher - name: kafka-authz-logs - env: - - name: OTTERIZE_MAPPER_API_URL - value: http://otterize-network-mapper.otterize-system.svc:9090/query - - name: OTTERIZE_DEBUG - value: "False" - - name: OTTERIZE_POD - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OTTERIZE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -extraVolumes: - - name: kafka-authz-logs - emptyDir: -extraVolumeMounts: - - mountPath: /opt/otterize/kafka-watcher - name: kafka-authz-logs log4j: | log4j.rootLogger=INFO, stdout @@ -65,10 +39,10 @@ log4j: | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - log4j.appender.authorizerAppender=org.apache.log4j.FileAppender + # Write to stdout for the k8s-watcher + log4j.appender.authorizerAppender=org.apache.log4j.ConsoleAppender log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - log4j.appender.authorizerAppender.file=/opt/otterize/kafka-watcher/authz.log # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender diff --git a/static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml b/static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml new file mode 100644 index 000000000..d5a7dddf1 --- /dev/null +++ b/static/code-examples/kafka-mtls/helm/values_debug_logging_k8s_watcher.yaml @@ -0,0 +1,75 @@ +# Configure Otterize as a super user to grant it access to configure ACLs +superUsers: "User:CN=kafka.kafka,O=SPIRE,C=US;User:CN=intents-operator-controller-manager.otterize,O=SPIRE,C=US" +# Use TLS for the Kafka listeners (Kafka calls them SSL) +listeners: + - "CLIENT://:9092" + - "INTERNAL://:9093" +advertisedListeners: + - "CLIENT://:9092" + - "INTERNAL://:9093" +listenerSecurityProtocolMap: "INTERNAL:SSL,CLIENT:SSL" +# For a gradual rollout scenario we will want to keep the default permission for topics as allowed, unless an ACL was set +allowEveryoneIfNoAclFound: true +# Annotations for Otterize to generate credentials +podAnnotations: + credentials-operator.otterize.com/cert-type: jks + credentials-operator.otterize.com/tls-secret-name: kafka-tls-secret + credentials-operator.otterize.com/truststore-file-name: kafka.truststore.jks + credentials-operator.otterize.com/keystore-file-name: kafka.keystore.jks + credentials-operator.otterize.com/dns-names: "kafka-0.kafka-headless.kafka.svc.cluster.local,kafka.kafka.svc.cluster.local" +# Authenticate clients using mTLS +auth: + clientProtocol: mtls + interBrokerProtocol: mtls + tls: + type: jks + existingSecrets: + - kafka-tls-secret + password: password +authorizerClassName: kafka.security.authorizer.AclAuthorizer +# Allocate resources +resources: + requests: + cpu: 50m + memory: 256Mi +sidecars: + - name: otterize-sidecar + image: otterize/network-mapper-kafka-watcher:dev + imagePullPolicy: Never + volumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs + env: + - name: OTTERIZE_MAPPER_API_URL + value: http://otterize-network-mapper.otterize-system.svc:9090/query + - name: OTTERIZE_DEBUG + value: "False" + - name: OTTERIZE_POD + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTTERIZE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +extraVolumes: + - name: kafka-authz-logs + emptyDir: +extraVolumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs +log4j: | + log4j.rootLogger=INFO, stdout + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + log4j.appender.authorizerAppender=org.apache.log4j.FileAppender + log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout + log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + log4j.appender.authorizerAppender.file=/opt/otterize/kafka-watcher/authz.log + + # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses + log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender + log4j.additivity.kafka.authorizer.logger=false