diff --git a/.changeset/dry-gifts-buy.md b/.changeset/dry-gifts-buy.md new file mode 100644 index 0000000000..cfc7ec1f3d --- /dev/null +++ b/.changeset/dry-gifts-buy.md @@ -0,0 +1,5 @@ +--- +"electric-sql": patch +--- + +Fix the Docker Compose file that's bundled with the client CLI to support databases that are only reachable over IPv6, such as Supabase. diff --git a/components/cli/src/config-options.ts b/components/cli/src/config-options.ts index 82e868bab4..d051822ad1 100644 --- a/components/cli/src/config-options.ts +++ b/components/cli/src/config-options.ts @@ -335,7 +335,14 @@ export const configOptions: Record = { valueType: String, valueTypeName: 'features', defaultVal: '', - doc: 'Flags to enable experimental features', + doc: 'Flags to enable experimental features.', + groups: ['electric'], + }, + DOCKER_NETWORK_USE_EXTERNAL: { + valueType: String, + valueTypeName: "'host' | name", + defaultVal: '', + doc: "Name of an existing Docker network to use or 'host' to run the container using the host OS networking.", groups: ['electric'], }, } as const diff --git a/components/cli/src/docker-commands/command-start.ts b/components/cli/src/docker-commands/command-start.ts index ce827c9fae..0977a7a103 100644 --- a/components/cli/src/docker-commands/command-start.ts +++ b/components/cli/src/docker-commands/command-start.ts @@ -6,7 +6,7 @@ import { Config, printConfig, } from '../config' -import { dockerCompose } from './docker-utils' +import { dockerCompose, dockerComposeUp } from './docker-utils' export function makeStartCommand() { const command = new Command('start') @@ -88,9 +88,8 @@ export function start(options: StartSettings) { console.log('Docker compose config:') printConfig(dockerConfig) - const proc = dockerCompose( - 'up', - [...(options.detach ? ['--detach'] : [])], + const proc = dockerComposeUp( + options.detach ? ['--detach'] : [], options.config.CONTAINER_NAME, dockerConfig ) diff --git a/components/cli/src/docker-commands/docker-utils.ts b/components/cli/src/docker-commands/docker-utils.ts index fc8ec66020..553fe6765d 100644 --- a/components/cli/src/docker-commands/docker-utils.ts +++ b/components/cli/src/docker-commands/docker-utils.ts @@ -2,17 +2,24 @@ import { spawn } from 'child_process' import path from 'path' import { fileURLToPath } from 'url' -const composeFile = path.join( - path.dirname(fileURLToPath(import.meta.url)), - 'docker', - 'compose.yaml' -) +function composeFilePath(filename: string) { + return path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'docker', + filename + ) +} + +function useExternalDockerNetwork(networkName?: string): boolean { + return !!(networkName && networkName.length > 0) +} -const composeFileWithPostgres = path.join( - path.dirname(fileURLToPath(import.meta.url)), - 'docker', - 'compose-with-postgres.yaml' -) +// Derive network name from the current working directory, matching Docker Compose's default +// naming. +function deriveNetworkName(networkName?: string): string { + if (networkName && networkName.length > 0) return networkName + return path.basename(process.cwd()) + '_ip6net' +} export function dockerCompose( command: string, @@ -20,22 +27,49 @@ export function dockerCompose( containerName?: string, env: { [key: string]: string } = {} ) { - const withPostgres = env?.COMPOSE_PROFILES === 'with-postgres' + const composeFile = 'compose.yaml' + const extraComposeFile = + env.DOCKER_NETWORK_USE_EXTERNAL === 'host' + ? 'compose.hostnet.yaml' + : 'compose.ip6net.yaml' const args = [ 'compose', '--ansi', 'always', '-f', - withPostgres ? composeFileWithPostgres : composeFile, + composeFilePath(composeFile), + '-f', + composeFilePath(extraComposeFile), command, ...userArgs, ] return spawn('docker', args, { stdio: 'inherit', env: { + ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL: useExternalDockerNetwork( + env.DOCKER_NETWORK_USE_EXTERNAL + ).toString(), + ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME: deriveNetworkName( + env.DOCKER_NETWORK_USE_EXTERNAL + ), ...process.env, ...(containerName ? { COMPOSE_PROJECT_NAME: containerName } : {}), ...env, }, }) } + +export function dockerComposeUp( + userArgs: string[] = [], + containerName?: string, + env: { [key: string]: string } = {} +) { + // We use the same compose.yaml file for `electric-sql start` and `electric-sql start + // --with-postgres` and vary the services started by passing them as arguments to `docker + // compose up`. + const services = + env.COMPOSE_PROFILES === 'with-postgres' + ? ['postgres', 'electric'] + : ['electric'] + return dockerCompose('up', userArgs.concat(services), containerName, env) +} diff --git a/components/cli/src/docker-commands/docker/compose-with-postgres.yaml b/components/cli/src/docker-commands/docker/compose-with-postgres.yaml deleted file mode 100644 index 9ce7aeefbd..0000000000 --- a/components/cli/src/docker-commands/docker/compose-with-postgres.yaml +++ /dev/null @@ -1,63 +0,0 @@ -configs: - postgres_config: - file: './postgres.conf' - -volumes: - pg_data: - -services: - postgres: - profiles: ['with-postgres'] - image: '${POSTGRESQL_IMAGE:-postgres:14-alpine}' - environment: - POSTGRES_DB: ${DATABASE_NAME:-electric} - POSTGRES_USER: ${DATABASE_USER:-postgres} - POSTGRES_PASSWORD: ${DATABASE_PASSWORD:-db_password} - command: - - -c - - config_file=/etc/postgresql.conf - - -p - - ${DATABASE_PORT:-5432} - configs: - - source: postgres_config - target: /etc/postgresql.conf - healthcheck: - test: - [ - 'CMD-SHELL', - 'pg_isready -U ${DATABASE_USER:-postgres} -p ${DATABASE_PORT:-5432}', - ] - extra_hosts: - - 'host.docker.internal:host-gateway' - ports: - - ${DATABASE_PORT:-5432}:${DATABASE_PORT:-5432} - volumes: - - pg_data:/var/lib/postgresql/data - - electric: - image: '${ELECTRIC_IMAGE:-electricsql/electric:latest}' - init: true - stop_signal: SIGINT # use SIGINT as the more speedy alternative to SIGTERM - ports: - - ${HTTP_PORT:-5133}:${HTTP_PORT:-5133} - - ${PG_PROXY_PORT_PARSED:-65432}:${PG_PROXY_PORT_PARSED:-65432} - environment: - DATABASE_REQUIRE_SSL: ${DATABASE_REQUIRE_SSL:-} - DATABASE_URL: ${DATABASE_URL:-} - DATABASE_USE_IPV6: ${DATABASE_USE_IPV6:-} - ELECTRIC_USE_IPV6: ${ELECTRIC_USE_IPV6:-} - HTTP_PORT: ${HTTP_PORT:-5133} - ELECTRIC_WRITE_TO_PG_MODE: ${ELECTRIC_WRITE_TO_PG_MODE:-} - LOGICAL_PUBLISHER_HOST: ${LOGICAL_PUBLISHER_HOST:-} - LOGICAL_PUBLISHER_PORT: ${LOGICAL_PUBLISHER_PORT:-5433} - PG_PROXY_PASSWORD: ${PG_PROXY_PASSWORD:-proxy_password} - PG_PROXY_PORT: ${PG_PROXY_PORT:-65432} - AUTH_MODE: ${AUTH_MODE:-insecure} - AUTH_JWT_ALG: ${AUTH_JWT_ALG:-} - AUTH_JWT_AUD: ${AUTH_JWT_AUD:-} - AUTH_JWT_ISS: ${AUTH_JWT_ISS:-} - AUTH_JWT_KEY: ${AUTH_JWT_KEY:-} - AUTH_JWT_NAMESPACE: ${AUTH_JWT_NAMESPACE:-} - ELECTRIC_FEATURES: ${ELECTRIC_FEATURES:-} - depends_on: - - postgres diff --git a/components/cli/src/docker-commands/docker/compose.hostnet.yaml b/components/cli/src/docker-commands/docker/compose.hostnet.yaml new file mode 100644 index 0000000000..5101861db3 --- /dev/null +++ b/components/cli/src/docker-commands/docker/compose.hostnet.yaml @@ -0,0 +1,13 @@ +# This extra compose file is meant to be used together with the main compose.yaml file when +# running Docker Compose commands. +# +# In this file we configure both services to use the host network mode, side-stepping Docker +# networking entirely. + +services: + electric: + network_mode: host + + postgres: + # Postgres must be on the same network as the sync service. + network_mode: host diff --git a/components/cli/src/docker-commands/docker/compose.ip6net.yaml b/components/cli/src/docker-commands/docker/compose.ip6net.yaml new file mode 100644 index 0000000000..acd7cac5ca --- /dev/null +++ b/components/cli/src/docker-commands/docker/compose.ip6net.yaml @@ -0,0 +1,55 @@ +# This extra compose file is meant to be used together with the main compose.yaml file when +# running Docker Compose commands. +# +# In this file we define an IPv6-enabled network and assign it to both "electric" and +# "postgres" services. + +networks: + # An IPv6-enabled network is necessary for the sync service to connect to + # databases that are only reachable at IPv6 addresses. IPv4 can also be used as usual with + # this network. + ip6net: + enable_ipv6: true + # These two options provide an escape hatch that allows users to define their own Docker + # network using `docker network create` and use that for the services defined in this + # compose file. + # + # These environment variables are set by the CLI and are derived from the single + # user-facing `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` variable. + external: ${ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL} + name: '${ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME}' + ipam: + config: + # Subnet definition isn't required if the Docker daemon has a default address pool for + # IPv6 addresses configured. However, since that's not the case for Docker + # out-of-the-box (at least until version 27.0.1) and since we want to free our users + # from additional manual configuration when possible, we include a default subnet + # definition here that should work with any Docker daemon configuration. + # + # The fd00:: prefix is part of the address space reserved for Unique Local Addresses, + # i.e. private networks. This is analogous to 192.168.x.x in the IPv4 land. + # + # There is a possibility that this subnet overlaps with another network configured + # separately for the same Docker daemon. That situation would result in `docker compose + # up` failing with the error message + # + # Error response from daemon: Pool overlaps with other one on this address space + # + # To resolve this conflict, an external Docker network can be used by setting + # `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` to its name. + # + # The default subnet here has 2 randomly generated bytes in the 2nd and the 6th groups, + # it can accommodate 4 addresses which is small enough to further reduce any chance of conflicts. + - subnet: 'fd00:56f0::4acd:f0fc/126' + +services: + electric: + networks: + # Despite the name, assigning this network to the sync service does not preclude the use + # of IPv4. + - ip6net + + postgres: + networks: + # Postgres must be on the same network as the sync service. + - ip6net diff --git a/components/cli/src/docker-commands/docker/compose.yaml b/components/cli/src/docker-commands/docker/compose.yaml index 8691aeecfb..cb3c0391bd 100644 --- a/components/cli/src/docker-commands/docker/compose.yaml +++ b/components/cli/src/docker-commands/docker/compose.yaml @@ -1,3 +1,17 @@ +# This compose file is used by the CLI `start` command to run the sync service and, optionally, +# a Postgres database inside Docker containers. +# +# When a user wants to start only the sync service by executing `electric-sql start`, the CLI +# will execute `docker compose up ... electric`, omitting the "postgres" service, so only the +# "electric" service will be running. +# +# When `electric-sql start --with-postgres` is invoked, the CLI will execute `docker compose up +# ... postgres electric`. In this case, the sync service may spend some time in a reconnection loop +# until the Postgres database is ready to accept connections. This is a normal mode of operation. +# +# Docker Compose should be invoked with this file and one of the two extra +# files, compose.hostnest.yaml or compose.ip6net.yaml, depending on the chosen network mode. + configs: postgres_config: file: './postgres.conf' diff --git a/components/cli/test/config-options.test.ts b/components/cli/test/config-options.test.ts index 1a43df57f6..63038cb6d6 100644 --- a/components/cli/test/config-options.test.ts +++ b/components/cli/test/config-options.test.ts @@ -35,6 +35,7 @@ const expectedEnvVars = [ 'ELECTRIC_IMAGE', 'CONTAINER_NAME', 'ELECTRIC_FEATURES', + 'DOCKER_NETWORK_USE_EXTERNAL', ] test('assert that all expected env vars are options for CLI', (t) => {