Using secrets versus env variables with Podman

[kirillmuravyev]

Hello Community!
Need some help with the question on how to manage secrets within netbox docker-compose file.
I am using Podman compose and using podman secret mechanism.
Doing the things like in instruction and tried several approaches but every time logs says that "no password supplied"

Podman secrets:

Docker-compose override file:

version: '3.4'
services:
  netbox:
    image: netbox:latest-plugins
    ports:
    - 8000:8080
    environment:
      REMOTE_AUTH_ENABLED: "True"
      REMOTE_AUTH_BACKEND: "netbox.authentication.LDAPBackend"
      AUTH_LDAP_SERVER_URI: "closed"
      AUTH_LDAP_BIND_DN: "closed"
      AUTH_LDAP_BIND_PASSWORD: "/run/secrets/auth_ldap_bind_password"
      AUTH_LDAP_USER_SEARCH_BASEDN: "closed"
      AUTH_LDAP_GROUP_SEARCH_BASEDN: "closed"
      AUTH_LDAP_REQUIRE_GROUP_DN: "closed"
      AUTH_LDAP_GROUP_TYPE: "NestedGroupOfNamesType"
      AUTH_LDAP_IS_ADMIN_DN: "closed"
      AUTH_LDAP_IS_SUPERUSER_DN: "Closed"
      LDAP_IGNORE_CERT_ERRORS: "False"
      DB_PASSWORD: "/run/secrets/db_password"
      REDIS_CACHE_PASSWORD: "/run/secrets/redis_cache_password"
      REDIS_PASSWORD: "/run/secrets/redis_password"
      SECRET_KEY: "/run/secrets/secret_key"
    secrets:
    - auth_ldap_bind_password
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key
    build:
      context: .
      dockerfile: Dockerfile-Plugins

  netbox-worker:
    image: netbox:latest-plugins
    environment:
      DB_PASSWORD: "/run/secrets/db_password"
      REDIS_CACHE_PASSWORD: "/run/secrets/redis_cache_password"
      REDIS_PASSWORD: "/run/secrets/redis_password"
      SECRET_KEY: "/run/secrets/secret_key"
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  netbox-housekeeping:
    image: netbox:latest-plugins
    environment:
      DB_PASSWORD: "/run/secrets/db_password"
      REDIS_CACHE_PASSWORD: "/run/secrets/redis_cache_password"
      REDIS_PASSWORD: "/run/secrets/redis_password"
      SECRET_KEY: "/run/secrets/secret_key"
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  postgres:
    environment:
      DB_PASSWORD: "/run/secrets/db_password"
    restart: unless-stopped
    secrets:
    - db_password
  redis:
    environment:
      REDIS_PASSWORD: "/run/secrets/redis_password"
    restart: unless-stopped
    secrets:
    - redis_password
  redis-cache:
    environment:
      REDIS_CACHE_PASSWORD: "/run/secrets/redis_cache_password"
    restart: unless-stopped
    secrets:
    - redis_cache_password

secrets:
  auth_ldap_bind_password:
    external: true
  db_password:
    external: true
  redis_cache_password:
    external: true
  redis_password:
    external: true
  secret_key:
    external: true

errors on logs:

netbox-docker_netbox_1 django.db.utils.OperationalError: connection to server at "postgres" (10.89.0.2), port 5432 failed: FATAL:  password authentication failed for user "netbox"
netbox-docker_netbox-worker_1 Traceback (most recent call last):
netbox-docker_netbox-housekeeping_1 🧬 loaded config '/etc/netbox/config/logging.py'
netbox-docker_postgres_1 The default database encoding has accordingly been set to "UTF8".
netbox-docker_redis_1 1:M 30 Aug 2023 12:48:28.123 # Server initialized
netbox-docker_redis_1 1:M 30 Aug 2023 12:48:28.123 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
netbox-docker_redis_1 1:M 30 Aug 2023 12:48:28.515 * Creating AOF base file appendonly.aof.1.base.rdb on server start
netbox-docker_redis-cache_1 Reading the configuration file, at line 2
netbox-docker_netbox_1 [ Use DB_WAIT_DEBUG=1 in netbox.env to print full traceback for errors here ]
netbox-docker_netbox-worker_1   File "/opt/netbox/venv/lib/python3.10/site-packages/django/db/backends/base/base.py", line 282, in ensure_connection
netbox-docker_netbox-housekeeping_1 🧬 loaded config '/etc/netbox/config/plugins.py'
netbox-docker_postgres_1 The default text search configuration will be set to "english".
netbox-docker_postgres_1
netbox-docker_redis_1 1:M 30 Aug 2023 12:48:28.752 * Creating AOF incr file appendonly.aof.1.incr.aof on server start

netbox-docker_redis-cache_1 *** FATAL CONFIG FILE ERROR (Redis 7.0.12) ***
netbox-docker_redis-cache_1 Reading the configuration file, at line 2
netbox-docker_redis-cache_1 >>> 'requirepass'
netbox-docker_redis-cache_1 wrong number of arguments

Application is working fine when I add passwords in plain text to ENV files within env/ folder.
Trying to test production deployment where storing secrets in ENV files is not good.
What am I doing wrong?

[keeperAndy]

I'm getting a similar outcome. I believe the function to consume secrets being used in configuration/configuration.py is not working as intended, but I haven't worked out exactly why. If I revert to using Env variables to populate the passwords for postgres & redis, I can get the stack running and I can inspect /run/secrets/* on the running containers and see my podman secrets were indeed consumed, they just aren't being referenced when the rest of the scripts run that configure the environment.

The reason your stack is failing specifically on wrong number of arguments is that without any Env value defined for $REDIS_PASSWORD, the docker-compose.yml Redis stanza will lack a value for --requirepass:

# redis
  redis:
    image: docker.io/redis:7-alpine
    command:
    - sh
    - -c # this is to evaluate the $REDIS_PASSWORD from the env
    - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
    env_file: env/redis.env
    volumes:
    - netbox-redis-data:/data
  redis-cache:
    image: docker.io/redis:7-alpine
    command:
    - sh
    - -c # this is to evaluate the $REDIS_PASSWORD from the env
    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
    env_file: env/redis-cache.env
    volumes:
    - netbox-redis-cache-data:/data

[keeperAndy]

I can only suggest deleting your db_password secret and creating it again with the same value you have been using in Env. Double check your YAML and be sure there are no typos in the definitions for the db_password secret.

The whole stack is working for me as configured above with a clean install and all the secrets Env variables commented out in the respective ./env/*.env files. Secrets values for a production environment will need to be backed-up elsewhere because the values are not exposed by podman and changing them will break authentication for the database at minimum.

[kirillmuravyev]

Hi @keeperAndy , thakns f or y our help!
Not sure why, but not working for me as of now, tried re-creating the secret but still logs says that postreg pass is missing
here is my podman secret:

ID                         NAME                     DRIVER      CREATED        UPDATED
ce1b963499830583932876025  redis_cache_password     file        4 weeks ago    4 weeks ago
fdd78162d4c0011766ba800f0  auth_ldap_bind_password  file        5 weeks ago    5 weeks ago
0b582353a23cad2ded9e02699  redis_password           file        4 weeks ago    4 weeks ago
8b870b9a66fae93d3de062d46  db_password              file        2 minutes ago  2 minutes ago
c46851fc6a8309205eb1d3213  secret_key               file        4 weeks ago    4 weeks ago

docker-compose.override:

versio:on: '3.4'
services:
  netbox:
    image: netbox:latest-plugins
    restart: unless-stopped
    ports:
    - 8000:8080
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key
    build:
      context: .
      dockerfile: Dockerfile-Plugins

  netbox-worker:
    image: netbox:latest-plugins
    restart: unless-stopped
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  netbox-housekeeping:
    image: netbox:latest-plugins
    restart: unless-stopped
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  postgres:
    restart: unless-stopped
    secrets:
    - db_password

  redis:
    restart: unless-stopped
    secrets:
    - redis_password
    command:
    - sh
    - -c
    - redis-server --appendonly yes --requirepass "$$(cat  /run/secrets/redis_password)"

  redis-cache:
    restart: unless-stopped
    secrets:
    - redis_cache_password
    command:
    - sh
    - -c
    - redis-server --requirepass "$$(cat  /run/secrets/redis_cache_password)"

secrets:
  db_password:
    external: true
  redis_cache_password:
    external: true
  redis_password:
    external: true
  secret_key:
    external: true

Tried uncommenting DB_Password variable in env file and getting errors now from other containers as well for the same, this is strange.
When I open container I can see secret files in run/secrets folder:

[netbox-docker]$ podman exec -it netbox-docker_postgres_1 bash
65554310b547:/# ls run/secrets/
db_password

[keeperAndy]

I think I see why. I discovered this when I completely reset my environment to test from a clean slate and ran in the same problem you are now.

Upon first run, where the postgres database is initialized, the Env variable for POSTGRES_PASSWORD must match your db_password value because the secret is not being consumed for postgres initialization. I haven't tracked down where that fault is in the code, but if I set the POSTGRES_PASSWORD value in ./env/postgres.env just for the first run of podman-compose up, I can then comment it back out and run with only the db_password providing the value to the container.

Edit: the problem seems to be with how the postgres container must have the POSTGRES_PASSWORD Env variable passed to it when it is first launched for initdb. Not great for security, but I'll be trying to think of a workaround.

Edit again: The postgres docker image can accept a variable designed for use with secrets so that no POSTGRES_PASSWORD need be defined:

  postgres:
    restart: unless-stopped
    environment:
      POSTGRES_PASSWORD_FILE=/run/secrets/db_password
    secrets:
    - db_password

Add the environment entry for POSTGRES_PASSWORD_FILE like above and you can omit POSTGRES_PASSWORD even during first run.

[kirillmuravyev]

Hi @keeperAndy , that worked! Thanks a lot, really much apperciate your help
I'll summarize here your inputs and mark as answer if you dont mind

modify docker-compose override file with redis, redis-cache and postgres entries, add secrets and declare secrets, whole file looks like:

 version: '3.4'
services:
  netbox:
    image: netbox:latest-plugins
    restart: unless-stopped
    environment:
      REMOTE_AUTH_ENABLED: "True"
      REMOTE_AUTH_BACKEND: "netbox.authentication.LDAPBackend"
      AUTH_LDAP_SERVER_URI: "ldap://ruidcx102.mars-ad.net"
      AUTH_LDAP_BIND_DN: "some DN"
      AUTH_LDAP_USER_SEARCH_BASEDN: "userbaseDN"
      AUTH_LDAP_GROUP_SEARCH_BASEDN: "groupbseDN"
      AUTH_LDAP_REQUIRE_GROUP_DN: "reqgroupDN"
      AUTH_LDAP_GROUP_TYPE: "NestedGroupOfNamesType"
      AUTH_LDAP_IS_ADMIN_DN: "adminDN"
      AUTH_LDAP_IS_SUPERUSER_DN: "superuserDN"
      LDAP_IGNORE_CERT_ERRORS: "False"
    ports:
    - 8000:8080
    secrets:
    - auth_ldap_bind_password
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key
    build:
      context: .
      dockerfile: Dockerfile-Plugins

  netbox-worker:
    image: netbox:latest-plugins
    restart: unless-stopped
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  netbox-housekeeping:
    image: netbox:latest-plugins
    restart: unless-stopped
    build:
      context: .
      dockerfile: Dockerfile-Plugins
    secrets:
    - db_password
    - redis_cache_password
    - redis_password
    - secret_key

  postgres:
    restart: unless-stopped
    environment:
      POSTGRES_PASSWORD_FILE=/run/secrets/db_password
    secrets:
    - db_password
  
  redis:
    restart: unless-stopped
    secrets:
    - redis_password
    command:
    - sh
    - -c 
    - redis-server --appendonly yes --requirepass "$$(cat  /run/secrets/redis_password)"
  
  redis-cache:
    restart: unless-stopped
    secrets:
    - redis_cache_password
    command:
    - sh
    - -c
    - redis-server --requirepass "$$(cat  /run/secrets/redis_cache_password)"

secrets:
  auth_ldap_bind_password:
    external: true
  db_password:
    external: true
  redis_cache_password:
    external: true
  redis_password:
    external: true
  secret_key:
    external: true