From 3330dc1d098e155a1c14f061b8bb5875d8cc35e6 Mon Sep 17 00:00:00 2001 From: Barrett Strausser Date: Tue, 7 Jan 2025 07:16:03 -0500 Subject: [PATCH] feat(postgres): ssl for postgres (#2473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * SSL for postgres * Add entrypoint wrapper * Add in init so we can test ssl+init path * Remove unused fields from options * Remove unused consts * Separate entrypoint from ssl * Use external cert generation * Make entrypoint not-optional * Add docstring * Spaces to tab in entrypoint * Add postgres ssl docs * Remove WithEntrypoint * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Embed resources + Use custom conf automatically * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update docs/modules/postgres.md Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Update modules/postgres/postgres_test.go Co-authored-by: Manuel de la Peña * Revert to use passed in conf * Update doc for required conf * Error checking in the customizer * Few formatting fix * Use non-nil error when err is nil * Update modules/postgres/postgres_test.go Co-authored-by: Steven Hartland * Update modules/postgres/postgres_test.go Co-authored-by: Steven Hartland * Update modules/postgres/postgres.go Co-authored-by: Steven Hartland * Update modules/postgres/postgres.go Co-authored-by: Steven Hartland * Update modules/postgres/postgres_test.go Co-authored-by: Steven Hartland * Addresses review modulo cleanup * Remove unused type * Use ContainerCleanup * Lint pass * Add t.Helper and Linting * Remove SSLSetting struct, use raw paths * Use single command for chown key material * docs: remove spaces * fix: use non-deprecated APIs * chore: rename variable --------- Co-authored-by: bstrausser Co-authored-by: Manuel de la Peña Co-authored-by: Steven Hartland Co-authored-by: Manuel de la Peña --- docs/modules/postgres.md | 26 ++++++ modules/postgres/go.mod | 1 + modules/postgres/go.sum | 2 + modules/postgres/postgres.go | 41 +++++++++ modules/postgres/postgres_test.go | 88 +++++++++++++++++++ .../postgres/resources/customEntrypoint.sh | 25 ++++++ modules/postgres/testdata/postgres-ssl.conf | 80 +++++++++++++++++ 7 files changed, 263 insertions(+) create mode 100644 modules/postgres/resources/customEntrypoint.sh create mode 100644 modules/postgres/testdata/postgres-ssl.conf diff --git a/docs/modules/postgres.md b/docs/modules/postgres.md index 930de50c15..4192cf7eca 100644 --- a/docs/modules/postgres.md +++ b/docs/modules/postgres.md @@ -74,9 +74,35 @@ An example of a `*.sh` script that creates a user and database is shown below: In the case you have a custom config file for Postgres, it's possible to copy that file into the container before it's started, using the `WithConfigFile(cfgPath string)` function. +This function can be used `WithSSLSettings` but requires your configuration correctly sets the SSL properties. See the below section for more information. + !!!tip For information on what is available to configure, see the [PostgreSQL docs](https://www.postgresql.org/docs/14/runtime-config.html) for the specific version of PostgreSQL that you are running. +#### SSL Configuration + +- Not available until the next release of testcontainers-go :material-tag: main + +If you would like to use SSL with the container you can use the `WithSSLSettings`. This function accepts a `SSLSettings` which has the required secret material, namely the ca-certificate, server certificate and key. The container will copy this material to `/tmp/testcontainers-go/postgres/ca_cert.pem`, `/tmp/testcontainers-go/postgres/server.cert` and `/tmp/testcontainers-go/postgres/server.key` + +This function requires a custom postgres configuration file that enables SSL and correctly sets the paths on the key material. + +If you use this function by itself or in conjuction with `WithConfigFile` your custom conf must set the require ssl fields. The configuration must correctly align the key material provided via `SSLSettings` with the server configuration, namely the paths. Your configuration will need to contain the following: + +``` +ssl = on +ssl_ca_file = '/tmp/testcontainers-go/postgres/ca_cert.pem' +ssl_cert_file = '/tmp/testcontainers-go/postgres/server.cert' +ssl_key_file = '/tmp/testcontainers-go/postgres/server.key' +``` + +!!!warning + This function assumes the postgres user in the container is `postgres` + + There is no current support for mutual authentication. + + The `SSLSettings` function will modify the container `entrypoint`. This is done so that key material copied over to the container is chowned by `postgres`. All other container arguments will be passed through to the original container entrypoint. + ### Container Methods #### ConnectionString diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index 01ee22b8bd..955edef1de 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -6,6 +6,7 @@ require ( github.com/docker/go-connections v0.5.0 github.com/jackc/pgx/v5 v5.5.4 github.com/lib/pq v1.10.9 + github.com/mdelapenya/tlscert v0.1.0 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.34.0 diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index ba337a188c..9d1c34d3df 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -71,6 +71,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mdelapenya/tlscert v0.1.0 h1:YTpF579PYUX475eOL+6zyEO3ngLTOUWck78NBuJVXaM= +github.com/mdelapenya/tlscert v0.1.0/go.mod h1:wrbyM/DwbFCeCeqdPX/8c6hNOqQgbf0rUDErE1uD+64= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= diff --git a/modules/postgres/postgres.go b/modules/postgres/postgres.go index ce3719575d..e25bc667a6 100644 --- a/modules/postgres/postgres.go +++ b/modules/postgres/postgres.go @@ -3,6 +3,7 @@ package postgres import ( "context" "database/sql" + _ "embed" "errors" "fmt" "io" @@ -19,6 +20,9 @@ const ( defaultSnapshotName = "migrated_template" ) +//go:embed resources/customEntrypoint.sh +var embeddedCustomEntrypoint string + // PostgresContainer represents the postgres container type used in the module type PostgresContainer struct { testcontainers.Container @@ -205,6 +209,43 @@ func WithSnapshotName(name string) SnapshotOption { } } +// WithSSLSettings configures the Postgres server to run with the provided CA Chain +// This will not function if the corresponding postgres conf is not correctly configured. +// Namely the paths below must match what is set in the conf file +func WithSSLCert(caCertFile string, certFile string, keyFile string) testcontainers.CustomizeRequestOption { + const defaultPermission = 0o600 + + return func(req *testcontainers.GenericContainerRequest) error { + const entrypointPath = "/usr/local/bin/docker-entrypoint-ssl.bash" + + req.Files = append(req.Files, + testcontainers.ContainerFile{ + HostFilePath: caCertFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/ca_cert.pem", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + HostFilePath: certFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/server.cert", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + HostFilePath: keyFile, + ContainerFilePath: "/tmp/testcontainers-go/postgres/server.key", + FileMode: defaultPermission, + }, + testcontainers.ContainerFile{ + Reader: strings.NewReader(embeddedCustomEntrypoint), + ContainerFilePath: entrypointPath, + FileMode: defaultPermission, + }, + ) + req.Entrypoint = []string{"sh", entrypointPath} + + return nil + } +} + // Snapshot takes a snapshot of the current state of the database as a template, which can then be restored using // the Restore method. By default, the snapshot will be created under a database called migrated_template, you can // customize the snapshot name with the options. diff --git a/modules/postgres/postgres_test.go b/modules/postgres/postgres_test.go index 8c02e68476..e83b8e1454 100644 --- a/modules/postgres/postgres_test.go +++ b/modules/postgres/postgres_test.go @@ -3,7 +3,9 @@ package postgres_test import ( "context" "database/sql" + "errors" "fmt" + "os" "path/filepath" "testing" "time" @@ -12,6 +14,8 @@ import ( "github.com/jackc/pgx/v5" _ "github.com/jackc/pgx/v5/stdlib" _ "github.com/lib/pq" + "github.com/mdelapenya/tlscert" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -25,6 +29,40 @@ const ( password = "password" ) +func createSSLCerts(t *testing.T) (*tlscert.Certificate, *tlscert.Certificate, error) { + t.Helper() + tmpDir := t.TempDir() + certsDir := tmpDir + "/certs" + + require.NoError(t, os.MkdirAll(certsDir, 0o755)) + + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(tmpDir)) + }) + + caCert := tlscert.SelfSignedFromRequest(tlscert.Request{ + Host: "localhost", + Name: "ca-cert", + ParentDir: certsDir, + }) + + if caCert == nil { + return caCert, nil, errors.New("unable to create CA Authority") + } + + cert := tlscert.SelfSignedFromRequest(tlscert.Request{ + Host: "localhost", + Name: "client-cert", + Parent: caCert, + ParentDir: certsDir, + }) + if cert == nil { + return caCert, cert, errors.New("unable to create Server Certificates") + } + + return caCert, cert, nil +} + func TestPostgres(t *testing.T) { ctx := context.Background() @@ -171,6 +209,56 @@ func TestWithConfigFile(t *testing.T) { defer db.Close() } +func TestWithSSL(t *testing.T) { + ctx := context.Background() + + caCert, serverCerts, err := createSSLCerts(t) + require.NoError(t, err) + + ctr, err := postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithConfigFile(filepath.Join("testdata", "postgres-ssl.conf")), + postgres.WithInitScripts(filepath.Join("testdata", "init-user-db.sh")), + postgres.WithDatabase(dbname), + postgres.WithUsername(user), + postgres.WithPassword(password), + testcontainers.WithWaitStrategy(wait.ForLog("database system is ready to accept connections").WithOccurrence(2).WithStartupTimeout(5*time.Second)), + postgres.WithSSLCert(caCert.CertPath, serverCerts.CertPath, serverCerts.KeyPath), + ) + + testcontainers.CleanupContainer(t, ctr) + require.NoError(t, err) + + connStr, err := ctr.ConnectionString(ctx, "sslmode=require") + require.NoError(t, err) + + db, err := sql.Open("postgres", connStr) + require.NoError(t, err) + assert.NotNil(t, db) + defer db.Close() + + result, err := db.Exec("SELECT * FROM testdb;") + require.NoError(t, err) + assert.NotNil(t, result) +} + +func TestSSLValidatesKeyMaterialPath(t *testing.T) { + ctx := context.Background() + + _, err := postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithConfigFile(filepath.Join("testdata", "postgres-ssl.conf")), + postgres.WithInitScripts(filepath.Join("testdata", "init-user-db.sh")), + postgres.WithDatabase(dbname), + postgres.WithUsername(user), + postgres.WithPassword(password), + testcontainers.WithWaitStrategy(wait.ForLog("database system is ready to accept connections").WithOccurrence(2).WithStartupTimeout(5*time.Second)), + postgres.WithSSLCert("", "", ""), + ) + + require.Error(t, err, "Error should not have been nil. Container creation should have failed due to empty key material") +} + func TestWithInitScript(t *testing.T) { ctx := context.Background() diff --git a/modules/postgres/resources/customEntrypoint.sh b/modules/postgres/resources/customEntrypoint.sh new file mode 100644 index 0000000000..ff4ffa4291 --- /dev/null +++ b/modules/postgres/resources/customEntrypoint.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -Eeo pipefail + + +pUID=$(id -u postgres) +pGID=$(id -g postgres) + +if [ -z "$pUID" ] +then + echo "Unable to find postgres user id, required in order to chown key material" + exit 1 +fi + +if [ -z "$pGID" ] +then + echo "Unable to find postgres group id, required in order to chown key material" + exit 1 +fi + +chown "$pUID":"$pGID" \ + /tmp/testcontainers-go/postgres/ca_cert.pem \ + /tmp/testcontainers-go/postgres/server.cert \ + /tmp/testcontainers-go/postgres/server.key + +/usr/local/bin/docker-entrypoint.sh "$@" diff --git a/modules/postgres/testdata/postgres-ssl.conf b/modules/postgres/testdata/postgres-ssl.conf new file mode 100644 index 0000000000..5e49f16a4f --- /dev/null +++ b/modules/postgres/testdata/postgres-ssl.conf @@ -0,0 +1,80 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: ms = milliseconds +# kB = kilobytes s = seconds +# MB = megabytes min = minutes +# GB = gigabytes h = hours +# TB = terabytes d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory + # (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file + # (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) + +# - SSL - + +ssl = on +ssl_ca_file = '/tmp/testcontainers-go/postgres/ca_cert.pem' +ssl_cert_file = '/tmp/testcontainers-go/postgres/server.cert' +#ssl_crl_file = '' +ssl_key_file = '/tmp/testcontainers-go/postgres/server.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + +