chore(electric)!: Change some configuration options name for better DX (

#305)

This is related to the documentation changes discussed in
electric-sql/legacy-website#29.

.changeset/clever-keys-argue.md

@@ -0,0 +1,15 @@
+---
+"@core/electric": patch
+---
+
+Make some configuration option names more descriptive. Namely,
+
+- rename `ELECTRIC_HOST` to `LOGICAL_PUBLISHER_HOST`
+- rename `POSTGRES_REPLICATION_PORT` to `LOGICAL_PUBLISHER_PORT`
+- rename `STATUS_PORT` to `HTTP_API_PORT`
+
+The first two options together define the host:port pair that the PostgreSQL database will connect to
+in order to start following Electric's logical replication stream that publishes inbound changes from all clients.
+
+The `HTTP_API_PORT` option is now more aptly named since Electric exposes more than just the status endpoint
+on that port.

components/electric/Dockerfile

@@ -6,7 +6,7 @@ ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-$
 ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
 
 FROM ${BUILDER_IMAGE} AS builder
-LABEL maintainer="[email protected]"
+LABEL maintainer="[email protected]"
 
 RUN apt-get update -y && \
     apt-get install -y build-essential git curl && \
@@ -20,12 +20,12 @@ ENV MIX_ENV=prod
 WORKDIR /app
 COPY Makefile /app/
 
-COPY mix.*  /app/
+COPY mix.* /app/
 RUN mix deps.get
-COPY config/config.exs config/prod.exs /app/config/
+COPY config/config.exs /app/config/
 RUN mix deps.compile
 
-COPY lib    /app/lib/
+COPY lib /app/lib/
 COPY config/*runtime.exs /app/config/
 
 ARG ELECTRIC_VERSION=local

components/electric/config/config.exs

@@ -1,43 +1,3 @@
-# This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
-#
-# This configuration file is loaded before any dependency and
-# is restricted to this project.
-
-# General application configuration
 import Config
 
-# Configures Elixir's Logger
-config :logger, :console,
-  format: "$time $metadata[$level] $message\n",
-  metadata: [
-    :connection,
-    :origin,
-    :pid,
-    :pg_client,
-    :pg_producer,
-    :pg_slot,
-    :sq_client,
-    :component,
-    :instance_id,
-    :client_id,
-    :user_id,
-    :metadata
-  ]
-
-config :logger,
-  handle_otp_reports: true,
-  handle_sasl_reports: false,
-  level: :debug
-
-config :electric, Electric.Replication.Postgres,
-  pg_client: Electric.Replication.Postgres.Client,
-  producer: Electric.Replication.Postgres.LogicalReplicationProducer
-
 config :electric, Electric.Postgres.CachedWal.Api, adapter: Electric.Postgres.CachedWal.EtsBacked
-
-config :electric, Electric.StatusPlug, port: 5050
-
-# Import environment specific config. This must remain at the bottom
-# of this file so it overrides the configuration defined above.
-import_config "#{config_env()}.exs"

components/electric/config/dev.exs → components/electric/config/runtime.dev.exs

@@ -1,8 +1,9 @@
 import Config
 
-config :electric, Electric.VaxRepo, hostname: "localhost", port: 8087
+config :logger, level: :debug
 
-config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.dev.dat"
+auth_provider = System.get_env("AUTH_MODE", "secure") |> Electric.Satellite.Auth.build_provider!()
+config :electric, Electric.Satellite.Auth, provider: auth_provider
 
 config :electric, Electric.Replication.Connectors,
   postgres_1: [
@@ -25,4 +26,4 @@ config :electric, Electric.Replication.Connectors,
     ]
   ]
 
-config :logger, level: :debug
+config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.dev.dat"

components/electric/config/runtime.exs

@@ -7,65 +7,65 @@
 
 import Config
 
-alias Electric.Satellite.Auth
-
-auth_provider =
-  if config_env() == :test do
-    auth_config =
-      Auth.Secure.build_config!(
-        alg: "HS256",
-        key: "test-signing-key-at-least-32-bytes-long",
-        iss: "electric-sql-test-issuer"
-      )
-
-    {Auth.Secure, auth_config}
-  else
-    case System.get_env("AUTH_MODE", "secure") do
-      "insecure" ->
-        namespace = System.get_env("AUTH_JWT_NAMESPACE")
-        auth_config = Auth.Insecure.build_config(namespace: namespace)
-        {Auth.Insecure, auth_config}
-
-      "secure" ->
-        auth_config =
-          [
-            alg: System.get_env("AUTH_JWT_ALG"),
-            key: System.get_env("AUTH_JWT_KEY"),
-            namespace: System.get_env("AUTH_JWT_NAMESPACE"),
-            iss: System.get_env("AUTH_JWT_ISS"),
-            aud: System.get_env("AUTH_JWT_AUD")
-          ]
-          |> Enum.filter(fn {_, val} -> is_binary(val) and String.trim(val) != "" end)
-          |> Auth.Secure.build_config!()
-
-        {Auth.Secure, auth_config}
-
-      other ->
-        raise "Unsupported auth mode: #{inspect(other)}"
-    end
-  end
-
-config :electric, Electric.Satellite.Auth, provider: auth_provider
+default_log_level = "info"
+default_instance_id = "electric"
+default_auth_mode = "secure"
+default_http_api_port = "5050"
+default_ws_server_port = "5133"
+default_pg_server_port = "5433"
+default_offset_storage_path = "./offset_storage_data.dat"
+
+###
+
+config :logger,
+  handle_otp_reports: true,
+  handle_sasl_reports: false,
+  level: System.get_env("LOG_LEVEL", default_log_level) |> String.to_existing_atom()
+
+config :logger, :console,
+  format: "$time $metadata[$level] $message\n",
+  metadata: [
+    :connection,
+    :origin,
+    :pid,
+    :pg_client,
+    :pg_producer,
+    :pg_slot,
+    :sq_client,
+    :component,
+    :instance_id,
+    :client_id,
+    :user_id,
+    :metadata
+  ]
 
 config :electric,
   # Used only to send server identification upon connection,
   # can stay default while we're not working on multi-instance setups
-  instance_id: System.get_env("ELECTRIC_INSTANCE_ID", "electric")
+  instance_id: System.get_env("ELECTRIC_INSTANCE_ID", default_instance_id)
 
-if config_env() == :prod do
-  config :logger, level: String.to_existing_atom(System.get_env("LOG_LEVEL", "info"))
+config :electric, Electric.Replication.Postgres,
+  pg_client: Electric.Replication.Postgres.Client,
+  producer: Electric.Replication.Postgres.LogicalReplicationProducer
+
+config :electric,
+  http_api_port: System.get_env("HTTP_API_PORT", default_http_api_port) |> String.to_integer()
 
-  config :electric, Electric.StatusPlug,
-    port: System.get_env("STATUS_PORT", "5050") |> String.to_integer()
+config :electric, Electric.Satellite.WsServer,
+  port: System.get_env("WEBSOCKET_PORT", default_ws_server_port) |> String.to_integer()
 
-  electric_host = System.get_env("ELECTRIC_HOST") || raise "Env variable ELECTRIC_HOST is not set"
+pg_server_port =
+  System.get_env("LOGICAL_PUBLISHER_PORT", default_pg_server_port) |> String.to_integer()
 
-  electric_port = System.get_env("POSTGRES_REPLICATION_PORT", "5433") |> String.to_integer()
+config :electric, Electric.PostgresServer, port: pg_server_port
 
-  config :electric, Electric.PostgresServer, port: electric_port
+# The :prod environment is inlined here because by default Mix won't copy any config/runtime.*.exs files when assembling
+# a release, and we want a single configuration file in our release.
+if config_env() == :prod do
+  auth_provider =
+    System.get_env("AUTH_MODE", default_auth_mode) |> Electric.Satellite.Auth.build_provider!()
 
-  config :electric, Electric.Satellite.WsServer,
-    port: System.get_env("WEBSOCKET_PORT", "5133") |> String.to_integer()
+  config :electric, Electric.Satellite.Auth, provider: auth_provider
 
   postgresql_connection =
     System.fetch_env!("DATABASE_URL")
@@ -76,14 +76,18 @@ if config_env() == :prod do
     |> Keyword.update(:timeout, 5_000, &String.to_integer/1)
     |> Keyword.put(:replication, "database")
 
+  pg_server_host =
+    System.get_env("LOGICAL_PUBLISHER_HOST") ||
+      raise("Env variable LOGICAL_PUBLISHER_HOST is not set")
+
   connectors = [
     {"postgres_1",
      producer: Electric.Replication.Postgres.LogicalReplicationProducer,
      connection: postgresql_connection,
      replication: [
        electric_connection: [
-         host: electric_host,
-         port: electric_port,
+         host: pg_server_host,
+         port: pg_server_port,
          dbname: "electric",
          connect_timeout: postgresql_connection[:timeout]
        ]
@@ -93,5 +97,7 @@ if config_env() == :prod do
   config :electric, Electric.Replication.Connectors, connectors
 
   config :electric, Electric.Replication.OffsetStorage,
-    file: System.get_env("OFFSET_STORAGE_FILE", "./offset_storage_data.dat")
+    file: System.get_env("OFFSET_STORAGE_FILE", default_offset_storage_path)
+else
+  Code.require_file("runtime.#{config_env()}.exs", __DIR__)
 end

components/electric/config/runtime.test.exs

@@ -0,0 +1,14 @@
+import Config
+
+alias Electric.Satellite.Auth
+
+auth_config =
+  Auth.Secure.build_config!(
+    alg: "HS256",
+    key: "test-signing-key-at-least-32-bytes-long",
+    iss: "electric-sql-test-issuer"
+  )
+
+config :electric, Electric.Satellite.Auth, provider: {Auth.Secure, auth_config}
+
+config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.test.dat"

components/electric/lib/electric/application.ex

@@ -21,16 +21,16 @@ defmodule Electric.Application do
       Electric.Telemetry,
       Electric.Postgres.OidDatabase,
       Electric.Replication.OffsetStorage,
-      {Plug.Cowboy, scheme: :http, plug: Electric.Plug.Router, options: [port: status_port()]},
+      {Plug.Cowboy, scheme: :http, plug: Electric.Plug.Router, options: [port: http_api_port()]},
       Electric.Satellite.SubscriptionManager,
       Electric.Satellite.ClientManager,
       Electric.Replication.Connectors,
       Electric.Satellite.WsServer.child_spec(
-        port: sqlite_server_port(),
+        port: ws_server_port(),
         auth_provider: auth_provider,
         pg_connector_opts: postgres_connector_opts
       ),
-      Electric.PostgresServer.child_spec(port: postgres_server_port())
+      Electric.PostgresServer.child_spec(port: pg_server_port())
     ]
 
     opts = [strategy: :one_for_one, name: Electric.Supervisor]
@@ -47,16 +47,12 @@ defmodule Electric.Application do
     {:ok, supervisor}
   end
 
-  defp status_port(),
-    do: Application.fetch_env!(:electric, Electric.StatusPlug) |> Keyword.fetch!(:port)
+  defp http_api_port,
+    do: Application.fetch_env!(:electric, :http_api_port)
 
-  defp sqlite_server_port(),
-    do:
-      Application.get_env(:electric, Electric.Satellite.WsServer, [])
-      |> Keyword.get(:port, 5133)
+  defp ws_server_port,
+    do: Application.fetch_env!(:electric, Electric.Satellite.WsServer) |> Keyword.fetch!(:port)
 
-  defp postgres_server_port(),
-    do:
-      Application.get_env(:electric, Electric.PostgresServer, [])
-      |> Keyword.get(:port, 5433)
+  defp pg_server_port,
+    do: Application.fetch_env!(:electric, Electric.PostgresServer) |> Keyword.fetch!(:port)
 end

components/electric/lib/electric/satellite/auth.ex

@@ -6,6 +6,7 @@ defmodule Electric.Satellite.Auth do
   implementation module's documentation for more info.
   """
 
+  alias __MODULE__
   require Logger
 
   @enforce_keys [:user_id]
@@ -37,4 +38,39 @@ defmodule Electric.Satellite.Auth do
     {_module, _config} = provider = Keyword.fetch!(config, :provider)
     provider
   end
+
+  @doc """
+  Build an auth provider from the given auth mode and runtime configuration options.
+
+  This is a helper function to be used in runtime config.
+  """
+  @spec build_provider!(String.t()) :: provider
+  def build_provider!("insecure") do
+    auth_config =
+      [
+        namespace: System.get_env("AUTH_JWT_NAMESPACE")
+      ]
+      |> Auth.Insecure.build_config()
+
+    {Auth.Insecure, auth_config}
+  end
+
+  def build_provider!("secure") do
+    auth_config =
+      [
+        alg: System.get_env("AUTH_JWT_ALG"),
+        key: System.get_env("AUTH_JWT_KEY"),
+        namespace: System.get_env("AUTH_JWT_NAMESPACE"),
+        iss: System.get_env("AUTH_JWT_ISS"),
+        aud: System.get_env("AUTH_JWT_AUD")
+      ]
+      |> Enum.filter(fn {_, val} -> is_binary(val) and String.trim(val) != "" end)
+      |> Auth.Secure.build_config!()
+
+    {Auth.Secure, auth_config}
+  end
+
+  def build_prodiver!(other) do
+    raise "Unsupported auth mode: #{inspect(other)}"
+  end
 end

e2e/satellite_client/src/client.ts

@@ -24,7 +24,7 @@ export const electrify_db = async (
     url: `electric://${host}:${port}`,
     debug: true,
     auth: {
-      token: await authToken(process.env.AUTH_SIGNING_ISS, process.env.AUTH_SIGNING_KEY)
+      token: await authToken()
     }
   }
   console.log(`(in electrify_db) config: ${JSON.stringify(config)}`)

e2e/tests/compose.yaml

@@ -14,8 +14,8 @@ services:
       file: ../services_templates.yaml
       service: electric
     environment:
-      ELECTRIC_HOST: electric_1
       DATABASE_URL: postgresql://postgres:password@pg_1:5432/electric
+      LOGICAL_PUBLISHER_HOST: electric_1
     ports:
       - "5050:5050"
       - "5133:5133"

local-stack/docker-compose.yaml

@@ -29,8 +29,8 @@ services:
       service: electric_template
     environment:
       <<: *preshared-auth-variables
-      ELECTRIC_HOST: electric_1
       DATABASE_URL: postgresql://postgres:password@postgres_1:5432/electric
+      LOGICAL_PUBLISHER_HOST: electric_1
     ports:
       - 5050:5050
       - 5133:5133