remove url string credential injection (#465)

* add nginx and keycloak to quickstart setup
* add basic_auth to quickstart
* add credentials via commandline
* remove possibility to inject auth credentials via commandline

Co-authored-by: dtrai2 <[email protected]>

CHANGELOG.md

@@ -28,6 +28,16 @@
 * new rule `id` could possibly break configurations if the same rule is used in both rule trees
   - can be fixed by adding a unique `id` to each rule or delete the possibly redundant rule
 
+### Breaking
+
+* remove possibility to inject auth credentials via url string, because of the risk leaking credentials in logs
+    - if you want to use basic auth, then you have to set the environment variables
+        * :code:`LOGPREP_CONFIG_AUTH_USERNAME=<your_username>`
+        * :code:`LOGPREP_CONFIG_AUTH_PASSWORD=<your_password>`
+    - if you want to use oauth, then you have to set the environment variables
+        * :code:`LOGPREP_CONFIG_AUTH_TOKEN=<your_token>`
+        * :code:`LOGPREP_CONFIG_AUTH_METHOD=oauth`
+
 ### Features
 
 * add possibility to convert hex to int in `calculator` processor with new added function `from_hex`

README.md

@@ -425,22 +425,36 @@ The environment can either be started with a Logprep container or without one:
     docker-compose --profile logprep up -d
     ```
 
+### Run with getting config from http server with basic authentication
+
+  * Run from within the `quickstart` directory: 
+    ```bash
+    docker-compose --profile basic_auth up -d
+    ```
+  * Run within the project root directory:
+    ```bash
+    export LOGPREP_CONFIG_AUTH_USERNAME="user"
+    export LOGPREP_CONFIG_AUTH_PASSWORD="password"
+    logprep http://localhost:8081/config/pipeline.yml
+    ```
+
 ### Interacting with the Quickstart Environment
 
 The start up takes a few seconds to complete, but once everything is up
 and running it is possible to write JSON events into Kafka and read the processed events in
 Opensearch Dashboards. Following services are available after start up:
 
-| Service | Location |
-|:----------|:----|
-| Kafka: | `localhost:9092` |
-| Logprep metrics: | `localhost:8000` |
-| Opensearch: | `localhost:9200` |
-| Opensearch Dashboards: | `localhost:5601` |
-| Grafana Dashboards: | `localhost:3000` |
-| Prometheus: | `localhost:9090` |
-
-The credentials for Grafana are `admin` and `admin`.
+| Service | Location | Credentials |
+|:----------|:----|:-----|
+| Kafka: | `localhost:9092` | |
+| Logprep metrics: | `localhost:8000` | |
+| Opensearch: | `localhost:9200` | |
+| Opensearch Dashboards: | `localhost:5601` | |
+| Grafana Dashboards: | `localhost:3000` | admin:admin |
+| Prometheus: | `localhost:9090` | |
+| Nginx: | `localhost:8081` | user:password |
+| Keycloak: | `localhost:8080` | admin:admin |
+| Postgres: | `localhost:5432` | keycloak:bitnami |
 
 The example rules that are used in the docker instance of Logprep can be found 
 in `quickstart/exampledata/rules`.

logprep/run_logprep.py

@@ -195,8 +195,12 @@ def main():
     if args.version:
         print(get_versions_string(args))
         sys.exit(0)
+    try:
+        config = _load_configuration(args)
+    except BaseException as error:  # pylint: disable=broad-except
+        logging.getLogger("Logprep").error(error)
+        sys.exit(1)
 
-    config = _load_configuration(args)
     logger = _setup_logger(args, config)
 
     if args.validate_rules or args.auto_test:

logprep/util/getter.py

@@ -8,9 +8,10 @@
 from string import Template
 from typing import Tuple
 from urllib.parse import urlparse
+
 import requests
+from attrs import define, field, validators
 from requests.auth import HTTPBasicAuth
-from attrs import field, validators, define
 
 from logprep._version import get_versions
 from logprep.abc.getter import Getter
@@ -91,33 +92,53 @@ class HttpGetter(Getter):
 
     * Simple http target: :code:`http://your.target/file.yml`
     * Simple https target: :code:`https://your.target/file.json`
-    * Target with asic authentication: :code:`https://username:password@your_web_target`
-    * Target with oauth compliant authentication with bearer token header:
-      :code:`https://oauth:<your bearer token>@your_web_target`
+
+    if you want to use basic auth, then you have to set the environment variables
+
+        * :code:`LOGPREP_CONFIG_AUTH_USERNAME=<your_username>`
+        * :code:`LOGPREP_CONFIG_AUTH_PASSWORD=<your_password>`
+
+    if you want to use oauth, then you have to set the environment variables
+
+        * :code:`LOGPREP_CONFIG_AUTH_TOKEN=<your_token>`
+        * :code:`LOGPREP_CONFIG_AUTH_METHOD=oauth`
 
     """
 
     _sessions: dict = {}
 
     _username: str = field(validator=validators.optional(validators.instance_of(str)), default=None)
     _password: str = field(validator=validators.optional(validators.instance_of(str)), default=None)
+    _headers: dict = field(validator=validators.instance_of(dict), factory=dict)
 
     def __attrs_post_init__(self):
+        user_agent = f"Logprep version {get_versions().get('version')}"
+        self._headers |= {"User-Agent": user_agent}
         target = self.target
-        auth_match = re.match(r"^((?P<username>.+):(?P<password>.+)@)?(?P<target>.+)", target)
-        self.target = auth_match.group("target")
-        self._username = auth_match.group("username")
-        self._password = auth_match.group("password")
+        target_match = re.match(r"^((?P<username>.+):(?P<password>.+)@)?(?P<target>.+)", target)
+        self.target = target_match.group("target")
+        if target_match.group("username") or target_match.group("password"):
+            raise NotImplementedError(
+                "Basic auth credentials via commandline are not supported."
+                "Please use environment variables "
+                "LOGPREP_CONFIG_AUTH_USERNAME and LOGPREP_CONFIG_AUTH_PASSWORD instead."
+            )
+        self._set_credentials()
+
+    def _set_credentials(self):
+        if os.environ.get("LOGPREP_CONFIG_AUTH_METHOD") == "oauth":
+            if token := os.environ.get("LOGPREP_CONFIG_AUTH_TOKEN"):
+                self._headers.update({"Authorization": f"Bearer {token}"})
+                self._username = None
+                self._password = None
+        self._username = os.environ.get("LOGPREP_CONFIG_AUTH_USERNAME")
+        self._password = os.environ.get("LOGPREP_CONFIG_AUTH_PASSWORD")
 
     def get_raw(self) -> bytearray:
         """gets the content from a http server via uri"""
-        user_agent = f"Logprep version {get_versions().get('version')}"
-        headers = {"User-Agent": user_agent}
         basic_auth = None
         username, password = self._username, self._password
-        if username == "oauth":
-            headers.update({"Authorization": f"Bearer {password}"})
-        if username is not None and username != "oauth":
+        if username is not None:
             basic_auth = HTTPBasicAuth(username, password)
         url = f"{self.protocol}://{self.target}"
         domain = urlparse(url).netloc
@@ -135,7 +156,7 @@ def get_raw(self) -> bytearray:
                     url=url,
                     timeout=5,
                     allow_redirects=True,
-                    headers=headers,
+                    headers=self._headers,
                 )
             except requests.exceptions.RequestException as error:
                 retries -= 1

quickstart/docker-compose.yml

@@ -98,6 +98,54 @@ services:
       - 9090
     volumes:
       - ../quickstart/exampledata/config/prometheus/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
-
+  config:
+    image: nginx:latest
+    container_name: config
+    profiles:
+      - basic_auth
+    network_mode: host
+    expose:
+      - 8081
+    volumes:
+      - ../quickstart/exampledata:/usr/share/nginx/html:ro
+      - ../quickstart/exampledata/config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ../quickstart/exampledata/config/nginx/conf.d:/etc/nginx/conf.d:ro
+  keycloak:
+    image: bitnami/keycloak:latest
+    container_name: keycloak
+    network_mode: host
+    expose:
+      - 8080
+    profiles:
+      - oauth2
+    environment:
+      KEYCLOAK_ADMIN_USER: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+      KEYCLOAK_DATABASE_HOST: localhost
+      KEYCLOAK_DATABASE_PORT: 5432
+      KEYCLOAK_DATABASE_NAME: keycloak
+      KEYCLOAK_DATABASE_USER: keycloak
+      KEYCLOAK_DATABASE_PASSWORD: bitnami
+      KEYCLOAK_HTTP_PORT: 8080
+      KEYCLOAK_HTTPS_PORT: 8443
+      KEYCLOAK_BIND_ADDRESS: 127.0.0.1
+  keycloak_db:
+    image: bitnami/postgresql:latest
+    container_name: keycloak_db
+    network_mode: host
+    expose:
+      - 5432
+    profiles:
+      - oauth2
+    environment:
+      POSTGRESQL_PASSWORD: bitnami
+      POSTGRESQL_POSTGRES_PASSWORD: bitnami
+      POSTGRESQL_DATABASE: keycloak
+      POSTGRESQL_USERNAME: keycloak
+    volumes:
+      # use this folder to persist your postgresql data by dumping the database to it after changes
+      # outside the container: chmod 777 quickstart/exampledata/config/postgresql
+      # inside the container: pg_dump keycloak -U keycloak -W --file /docker-entrypoint-initdb.d/keycloak_db.sql
+      - ../quickstart/exampledata/config/postgresql:/docker-entrypoint-initdb.d
 volumes:
   data:

quickstart/exampledata/config/nginx/conf.d/.htpasswd

@@ -0,0 +1 @@
+user:$apr1$YXI6.KFP$rAzwGvjl3uSyEGwU1LjoN1

quickstart/exampledata/config/nginx/conf.d/default.conf

@@ -0,0 +1,17 @@
+server {
+    listen       8081;
+    listen  [::]:8081;
+    server_name  localhost;
+    auth_basic "Restricted Content";
+    auth_basic_user_file /etc/nginx/conf.d/.htpasswd;
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+    }
+    # redirect server error pages to the static page /50x.html
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}
+

quickstart/exampledata/config/nginx/nginx.conf

@@ -0,0 +1,32 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log notice;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    include /etc/nginx/conf.d/*.conf;
+}