composer: use logrus hook instead of k8s sidecar

for splunk log forwarding
Fixes COMPOSER-2051

Author: diaasami

cmd/osbuild-composer/config.go

@@ -18,6 +18,9 @@ type ComposerConfigFile struct {
 	LogLevel     string          `toml:"log_level"`
 	LogFormat    string          `toml:"log_format"`
 	DNFJson      string          `toml:"dnf-json"`
+	SplunkHost   string          `env:"SPLUNK_HEC_HOST"`
+	SplunkPort   string          `env:"SPLUNK_HEC_PORT"`
+	SplunkToken  string          `env:"SPLUNK_HEC_TOKEN"`
 }
 
 type KojiAPIConfig struct {

cmd/osbuild-composer/main.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
 	"github.com/coreos/go-systemd/activation"
+	slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger"
 	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/syslog"
 )
@@ -60,6 +62,15 @@ func main() {
 		logrus.Fatalf("Error printing configuration: %v", err)
 	}
 
+	if config.SplunkHost != "" {
+		hook, err := slogger.NewSplunkHook(context.Background(), config.SplunkHost, config.SplunkPort, config.SplunkToken, "composer")
+
+		if err != nil {
+			panic(err)
+		}
+		logrus.AddHook(hook)
+	}
+
 	stateDir, ok := os.LookupEnv("STATE_DIRECTORY")
 	if !ok {
 		logrus.Fatal("STATE_DIRECTORY is not set. Is the service file missing StateDirectory=?")

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/openshift-online/ocm-sdk-go v0.1.385
 	github.com/oracle/oci-go-sdk/v54 v54.0.0
 	github.com/osbuild/images v0.18.0
+	github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1
 	github.com/osbuild/pulp-client v0.1.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/segmentio/ksuid v1.0.4

go.sum

@@ -455,6 +455,8 @@ github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXch
 github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc=
 github.com/osbuild/images v0.18.0 h1:I/tOO7DCECciJptrXVq+oykJI5dP1rwkzJqmf2rKuqw=
 github.com/osbuild/images v0.18.0/go.mod h1:Zr+AkaX/Rpxyff6Zxh8kkwGKFtJsSukGo1Vv/j9HsxA=
+github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1 h1:UFEJIcPa46W8gtWgOYzriRKYyy1t6SWL0BI7fPTuVvc=
+github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1/go.mod h1:z+WA+dX6qMwc7fqY5jCzESDIlg4WR2sBQezxsoXv9Ik=
 github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8=
 github.com/osbuild/pulp-client v0.1.0/go.mod h1:rd/MLdfwwO2cQI1s056h8z32zAi3Bo90XhlAAryIvWc=
 github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=

templates/composer.yml

@@ -103,8 +103,21 @@ objects:
             value: "${PGSSLMODE}"
           - name: PGMAXCONNS
             value: "${PGMAXCONNS}"
-          - name: SYSLOG_SERVER
-            value: "localhost:5140"
+          # Splunk forwarding
+          - name: SPLUNK_HEC_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: splunk
+                key: token
+                optional: true
+          - name: SPLUNK_HEC_HOST
+            valueFrom:
+              secretKeyRef:
+                name: splunk
+                key: url
+                optional: true
+          - name: SPLUNK_HEC_PORT
+            value: "${SPLUNK_HEC_PORT}"
           ports:
           - name: composer-api
             protocol: TCP
@@ -123,32 +136,6 @@ objects:
             mountPath: "/var/lib/osbuild-composer"
           - name: cache-directory
             mountPath: "/var/cache/osbuild-composer"
-        - image: "quay.io/app-sre/fluentd-hec:1.2.13"
-          name: fluentd-sidecar
-          resources:
-            requests:
-              cpu: "${FLUENTD_CPU_REQUEST}"
-              memory: "${MEMORY_REQUEST}"
-            limits:
-              cpu: "${FLUENTD_CPU_LIMIT}"
-              memory: "${MEMORY_LIMIT}"
-          env:
-            - name: SPLUNK_HEC_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: splunk
-                  key: token
-                  optional: false
-            - name: SPLUNK_HEC_URL
-              valueFrom:
-                secretKeyRef:
-                  name: splunk
-                  key: url
-                  optional: false
-          volumeMounts:
-            - name: fluentd-config
-              mountPath: /fluentd/etc
-              readOnly: true
         volumes:
         - name: composer-config
           configMap:
@@ -157,9 +144,6 @@ objects:
           emptyDir: {}
         - name: cache-directory
           emptyDir: {}
-        - name: fluentd-config
-          configMap:
-            name: fluentd-config
         initContainers:
         - name: composer-migrate
           image: "${IMAGE_NAME}:${IMAGE_TAG}"
@@ -273,30 +257,6 @@ objects:
       jwt_keys_urls = ["${RH_SSO_BASE_URL}/protocol/openid-connect/certs"]
       jwt_acl_file = "${COMPOSER_CONFIG_DIR}/acl.yml"
       jwt_tenant_provider_fields = ["rh-org-id", "account_id"]
-- apiVersion: v1
-  kind: ConfigMap
-  metadata:
-    name: fluentd-config
-  data:
-    fluent.conf: |
-      <source>
-        @type syslog
-        port 5140
-        bind 127.0.0.1
-        <transport tcp>
-        </transport>
-        tag osbuild-composer
-        <parse>
-          time_format %Y-%m-%dT%H:%M:%SZ
-        </parse>
-      </source>
-
-      <match **>
-        @type splunk_hec
-        hec_host "#{ENV['SPLUNK_HEC_URL']}"
-        hec_port "${SPLUNK_HEC_PORT}"
-        hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
-      </match>
 - apiVersion: batch/v1
   kind: CronJob
   metadata:
@@ -523,7 +483,6 @@ parameters:
     name: MAINTENANCE_MAX_CONCURRENT_REQUESTS
     value: "10"
     required: true
-  - description: fluentd-hec splunk port
+  - description: Splunk HTTP Event Collector port
     name: SPLUNK_HEC_PORT
     value: "443"
-    required: true