diff --git a/docs/reference/configuration/network-mapper/helm-chart.mdx b/docs/reference/configuration/network-mapper/helm-chart.mdx index 8eea4f90c..fbbd65fad 100644 --- a/docs/reference/configuration/network-mapper/helm-chart.mdx +++ b/docs/reference/configuration/network-mapper/helm-chart.mdx @@ -33,19 +33,6 @@ Checkout the network mapper [tutorial](/quick-tutorials/k8s-network-mapper) to s | `sniffer.tolerations` | Tolerations override. | `(none)` | | `sniffer.priorityClassName` | Set priorityClassName. | `(none)` | - -## Kafka watcher parameters -| Key | Description | Default | -|---------------------------------|-------------------------------------------------------------|--------------------------------| -| `kafkawatcher.enable` | Enable Kafka watcher deployment (beta). | `false` | -| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | -| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | -| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | -| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | -| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | -| `kafkawatcher.resources` | Resources override. | `(none)` | -| `kafkawatcher.kafkaServers` | Kafka servers to watch, specified as `pod.namespace` items. | `(none)` | - ## Istio watcher parameters | Key | Description | Default | |---------------------------------|-----------------------------------------|--------------------------------| diff --git a/docs/reference/configuration/network-mapper/kafka-watcher.mdx b/docs/reference/configuration/network-mapper/kafka-watcher.mdx index 160f6b45a..49b617183 100644 --- a/docs/reference/configuration/network-mapper/kafka-watcher.mdx +++ b/docs/reference/configuration/network-mapper/kafka-watcher.mdx @@ -3,32 +3,7 @@ sidebar_position: 2 title: Kafka Watcher --- -To deploy the network mapper with the Kafka watcher component, do the following: -```bash -helm repo add otterize https://helm.otterize.com -helm repo update -helm install network-mapper otterize/network-mapper -n otterize-system --create-namespace --set kafkawatcher.enable=true -``` -Make sure to include `--set kafkaServers={}` and provide a list of Kafka servers whose logs the Kafka watcher should watch. -Servers in the list should be specified as `name.namespace`. - -## Kafka watcher parameters -| Key | Description | Default | -|---------------------------------|-------------------------------------------------------------|--------------------------------| -| `kafkawatcher.enable` | Enable Kafka watcher deployment (beta). | `false` | -| `kafkawatcher.image.repository` | Kafka watcher image repository. | `otterize` | -| `kafkawatcher.image.image` | Kafka watcher image. | `network-mapper-kafka-watcher` | -| `kafkawatcher.image.tag` | Kafka watcher image tag. | `latest` | -| `kafkawatcher.pullPolicy` | Kafka watcher pull policy. | `(none)` | -| `kafkawatcher.pullSecrets` | Kafka watcher pull secrets. | `(none)` | -| `kafkawatcher.resources` | Resources override. | `(none)` | -| `kafkawatcher.kafkaServers` | Kafka servers to watch, specified as `pod.namespace` items. | `(none)` | - -## Enabling debug logs in Kafka servers -The Kafka watcher periodically examines logs of Kafka servers provided by the user through configuration, -parses them and deduces topic-level access to Kafka from pods in the Kubernetes cluster. -In order for the Kafka watcher to correctly examine topic-level access, the Kafka server's ACL authorizer logger should be configured -to log at debug level, and to stdout. +The Kafka Watcher is deployed in a Kubernetes environment as a sidecar container in the Kafka pod. Kafka is configured as below to generate logs from authorization decisions and write them to a volume shared with the Watcher container. The Kafka Watcher collects these logs and sends them to the network mapper. ### Install Kafka via Helm with debug logs preconfigured For the Bitnami Kafka Helm chart used in other Kafka tutorials, we can add the following configuration to the chart's diff --git a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml index 8a6bba0c1..d5a7dddf1 100644 --- a/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml +++ b/static/code-examples/kafka-mtls/helm/values_debug_logging.yaml @@ -32,74 +32,44 @@ resources: requests: cpu: 50m memory: 256Mi +sidecars: + - name: otterize-sidecar + image: otterize/network-mapper-kafka-watcher:dev + imagePullPolicy: Never + volumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs + env: + - name: OTTERIZE_MAPPER_API_URL + value: http://otterize-network-mapper.otterize-system.svc:9090/query + - name: OTTERIZE_DEBUG + value: "False" + - name: OTTERIZE_POD + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTTERIZE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +extraVolumes: + - name: kafka-authz-logs + emptyDir: +extraVolumeMounts: + - mountPath: /opt/otterize/kafka-watcher + name: kafka-authz-logs log4j: | - # Unspecified loggers and loggers with additivity=true output to server.log and stdout - # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise - - log4j.rootLogger=INFO, stdout, kafkaAppender + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - log4j.appender.kafkaAppender=org.apache.log4j.ConsoleAppender - log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.stateChangeAppender=org.apache.log4j.ConsoleAppender - log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.requestAppender=org.apache.log4j.ConsoleAppender - log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.cleanerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.controllerAppender=org.apache.log4j.ConsoleAppender - log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout - log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - log4j.appender.authorizerAppender=org.apache.log4j.ConsoleAppender + log4j.appender.authorizerAppender=org.apache.log4j.FileAppender log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n - - - # Change the line below to adjust ZK client logging - log4j.logger.org.apache.zookeeper=INFO - - # Change the two lines below to adjust the general broker logging level (output to server.log and stdout) - log4j.logger.kafka=INFO, stdout - log4j.logger.org.apache.kafka=INFO - - # Change to DEBUG or TRACE to enable request logging - log4j.logger.kafka.request.logger=WARN, requestAppender - log4j.additivity.kafka.request.logger=false - - # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output - # related to the handling of requests - #log4j.logger.kafka.network.Processor=TRACE, requestAppender - #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender - #log4j.additivity.kafka.server.KafkaApis=false - log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender - log4j.additivity.kafka.network.RequestChannel$=false - - # Change the line below to adjust KRaft mode controller logging - log4j.logger.org.apache.kafka.controller=INFO, controllerAppender - log4j.additivity.org.apache.kafka.controller=false - - # Change the line below to adjust ZK mode controller logging - log4j.logger.kafka.controller=TRACE, controllerAppender - log4j.additivity.kafka.controller=false - - log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender - log4j.additivity.kafka.log.LogCleaner=false - - log4j.logger.state.change.logger=INFO, stateChangeAppender - log4j.additivity.state.change.logger=false + log4j.appender.authorizerAppender.file=/opt/otterize/kafka-watcher/authz.log # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses log4j.logger.kafka.authorizer.logger=DEBUG, authorizerAppender - log4j.additivity.kafka.authorizer.logger=false \ No newline at end of file + log4j.additivity.kafka.authorizer.logger=false