From 82bad06df66f05a1cb229f99ad74197fb7948d65 Mon Sep 17 00:00:00 2001 From: Tyler Treat Date: Wed, 16 Dec 2020 11:40:11 -0700 Subject: [PATCH] Prepare v1.4.0 release --- documentation/configuration.md | 2 +- documentation/quick_start.md | 4 +- go.mod | 12 +- go.sum | 15 ++ server/version.go | 2 +- website/i18n/en.json | 33 +++ .../version-v1.4.0/configuration.md | 220 ++++++++++++++++++ .../version-v1.4.0/quick_start.md | 105 +++++++++ website/versions.json | 1 + 9 files changed, 384 insertions(+), 10 deletions(-) create mode 100644 website/versioned_docs/version-v1.4.0/configuration.md create mode 100644 website/versioned_docs/version-v1.4.0/quick_start.md diff --git a/documentation/configuration.md b/documentation/configuration.md index cf086678..5d46483b 100644 --- a/documentation/configuration.md +++ b/documentation/configuration.md @@ -24,7 +24,7 @@ USAGE: liftbridge [global options] command [command options] [arguments...] VERSION: - v1.3.0 + v1.4.0 COMMANDS: help, h Shows a list of commands or help for one command diff --git a/documentation/quick_start.md b/documentation/quick_start.md index c8c5935f..06b6cf76 100644 --- a/documentation/quick_start.md +++ b/documentation/quick_start.md @@ -43,7 +43,7 @@ only be set on one server when bootstrapping a cluster.** ```shell $ liftbridge --raft-bootstrap-seed -INFO[2020-10-15 14:29:50] Liftbridge Version: v1.3.0 +INFO[2020-10-15 14:29:50] Liftbridge Version: v1.4.0 INFO[2020-10-15 14:29:50] Server ID: 4nbhBr66WnRsy0I5oKF9bo INFO[2020-10-15 14:29:50] Namespace: liftbridge-default INFO[2020-10-15 14:29:50] Default Retention Policy: [Age: 1 week, Compact: false] @@ -57,7 +57,7 @@ We set the `--data-dir` and `--port` flags to avoid clobbering the first server. ```shell $ liftbridge --data-dir /tmp/liftbridge/server-2 --port=9293 -INFO[2020-10-15 14:30:48] Liftbridge Version: v1.3.0 +INFO[2020-10-15 14:30:48] Liftbridge Version: v1.4.0 INFO[2020-10-15 14:30:48] Server ID: lbW05esZTab3guEwcmWD9M INFO[2020-10-15 14:30:48] Namespace: liftbridge-default INFO[2020-10-15 14:30:48] Default Retention Policy: [Age: 1 week, Compact: false] diff --git a/go.mod b/go.mod index 4e87219c..8710f2cb 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/raft v1.1.2 github.com/liftbridge-io/go-liftbridge/v2 v2.0.2-0.20201118225953-b849cccb6467 - github.com/liftbridge-io/liftbridge-api v1.1.1-0.20201029165056-10f2aa65f256 + github.com/liftbridge-io/liftbridge-api v1.4.0 github.com/liftbridge-io/nats-on-a-log v0.0.0-20200818183806-bb17516cf3a3 github.com/liftbridge-io/raft-boltdb v0.0.0-20200414234651-aaf6e08d8f73 github.com/mattn/go-colorable v0.1.7 // indirect @@ -36,15 +36,15 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.0 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 github.com/urfave/cli v1.22.4 go.etcd.io/bbolt v1.3.5 // indirect golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de // indirect - golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect - golang.org/x/sys v0.0.0-20201118182958-a01c418693c7 // indirect + golang.org/x/net v0.0.0-20201216054612-986b41b23924 // indirect + golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect golang.org/x/text v0.3.4 // indirect - google.golang.org/genproto v0.0.0-20201117123952-62d171c70ae1 // indirect - google.golang.org/grpc v1.33.2 + google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect + google.golang.org/grpc v1.34.0 gopkg.in/ini.v1 v1.57.0 // indirect launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index 3adff4ef..7d323500 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,7 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -63,6 +64,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= @@ -193,6 +195,8 @@ github.com/liftbridge-io/go-liftbridge/v2 v2.0.2-0.20201118225953-b849cccb6467/g github.com/liftbridge-io/liftbridge-api v1.0.1-0.20200703154436-52124ef6c36f/go.mod h1:6IFEFZ4ncnOgeDVjSt0vh1lKNhlJ5YT9xnG1eRa9LC8= github.com/liftbridge-io/liftbridge-api v1.1.1-0.20201029165056-10f2aa65f256 h1:2pZtC3v6IBTwE70xfb/k0DPlOJ6BlXpthCUWrxCnhwo= github.com/liftbridge-io/liftbridge-api v1.1.1-0.20201029165056-10f2aa65f256/go.mod h1:6IFEFZ4ncnOgeDVjSt0vh1lKNhlJ5YT9xnG1eRa9LC8= +github.com/liftbridge-io/liftbridge-api v1.4.0 h1:2axkVGBEnQdXvXVHG+GyTYQucqxuvk7W9vJLXOCJy68= +github.com/liftbridge-io/liftbridge-api v1.4.0/go.mod h1:6IFEFZ4ncnOgeDVjSt0vh1lKNhlJ5YT9xnG1eRa9LC8= github.com/liftbridge-io/nats-on-a-log v0.0.0-20200818183806-bb17516cf3a3 h1:O4mg1NEmukgY8hxen3grrG5RY34LadMTzpbjf8kM2tA= github.com/liftbridge-io/nats-on-a-log v0.0.0-20200818183806-bb17516cf3a3/go.mod h1:wmIIYVq+psahPlB1rvtTkGiltdihsKJbqwE1DkIPwj4= github.com/liftbridge-io/raft-boltdb v0.0.0-20200414234651-aaf6e08d8f73 h1:8r/ReB1ns87pVDwSnPj87HIbOu/5y0uDyGChx9mUGSQ= @@ -320,6 +324,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -387,6 +392,8 @@ golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c h1:dk0ukUIHmGHqASjP0iue2261i golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c/go.mod h1:iQL9McJNjoIa5mjH6nYTCTZXUN6RP+XW3eib7Ya3XcI= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -429,6 +436,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7 h1:Z991aAXPjz0tLnj74pVXW3eWJ5lHMIBvbRfMq4M2jHA= golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -485,6 +496,8 @@ google.golang.org/genproto v0.0.0-20201002142447-3860012362da h1:DTQYk4u7nICKkkV google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201117123952-62d171c70ae1 h1:EVow1AaDgdoMjdO64/fntn4+RGTVor8YE/mkmIYsqFM= google.golang.org/genproto v0.0.0-20201117123952-62d171c70ae1/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d h1:HV9Z9qMhQEsdlvxNFELgQ11RkMzO3CMkjEySjCtuLes= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -496,6 +509,8 @@ google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/server/version.go b/server/version.go index 07fbc788..907ab7ce 100644 --- a/server/version.go +++ b/server/version.go @@ -1,4 +1,4 @@ package server // Version of the Liftbridge server. -const Version = "v1.3.0" +const Version = "v1.4.0" diff --git a/website/i18n/en.json b/website/i18n/en.json index 06cc574e..afba7577 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -202,6 +202,39 @@ }, "version-v1.2.0/version-v1.2.0-roadmap": { "title": "Product Roadmap" + }, + "version-v1.3.0/version-v1.3.0-client-implementation": { + "title": "Client Implementation Guidance" + }, + "version-v1.3.0/version-v1.3.0-clients": { + "title": "Clients" + }, + "version-v1.3.0/version-v1.3.0-concepts": { + "title": "Concepts" + }, + "version-v1.3.0/version-v1.3.0-configuration": { + "title": "Configuration" + }, + "version-v1.3.0/version-v1.3.0-cursors": { + "title": "Cursors" + }, + "version-v1.3.0/version-v1.3.0-feature-comparison": { + "title": "Feature Comparison" + }, + "version-v1.3.0/version-v1.3.0-pausing-streams": { + "title": "Pausing Streams" + }, + "version-v1.3.0/version-v1.3.0-quick-start": { + "title": "Quick Start" + }, + "version-v1.3.0/version-v1.3.0-roadmap": { + "title": "Product Roadmap" + }, + "version-v1.4.0/version-v1.4.0-configuration": { + "title": "Configuration" + }, + "version-v1.4.0/version-v1.4.0-quick-start": { + "title": "Quick Start" } }, "links": { diff --git a/website/versioned_docs/version-v1.4.0/configuration.md b/website/versioned_docs/version-v1.4.0/configuration.md new file mode 100644 index 00000000..c1aea7f4 --- /dev/null +++ b/website/versioned_docs/version-v1.4.0/configuration.md @@ -0,0 +1,220 @@ +--- +id: version-v1.4.0-configuration +title: Configuration +original_id: configuration +--- + +Liftbridge provides limited configuration through command-line flags and full +configuration using a configuration file. Flags will always take precedent +over settings in the configuration file. + +The configuration file is passed in using the `--config` flag: + +```shell +$ liftbridge --config liftbridge.yaml +``` + +To get a full list of the CLI flags, use the `--help` flag: + +```shell +$ liftbridge --help +NAME: + liftbridge - Lightweight, fault-tolerant message streams + +USAGE: + liftbridge [global options] command [command options] [arguments...] + +VERSION: + v1.4.0 + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --config FILE, -c FILE load configuration from FILE + --server-id value, --id value ID of the server in the cluster if there is no stored ID (default: random ID) + --namespace value, --ns value cluster namespace (default: "liftbridge-default") + --nats-servers ADDR[,ADDR], -n ADDR[,ADDR] connect to NATS cluster at ADDR[,ADDR] (default: "nats://127.0.0.1:4222") + --data-dir DIR, -d DIR store data in DIR (default: "/tmp/liftbridge/") + --port value, -p value port to bind to (default: 9292) + --tls-cert value server certificate file + --tls-key value private key for server certificate + --level value, -l value logging level [debug|info|warn|error] (default: "info") + --raft-bootstrap-seed bootstrap the Raft cluster by electing self as leader if there is no existing state + --raft-bootstrap-peers value bootstrap the Raft cluster with the provided list of peer IDs if there is no existing state + --help, -h show help + --version, -v print the version +``` + +## Configuration File Format + +The configuration file uses a YAML format. The configuration settings are +described below. Settings follow a hierarchical pattern delimited by periods. +The full, flattened name or nested names can be used. For example: + +```yaml +logging.level: debug +logging.recovery: true +logging.raft: true +``` + +The above configuration is equivalent to: + +```yaml +logging: + level: debug + recovery: true + raft: true +``` + +## Example Configuration File + +An example configuration file is shown below. + +```yaml +--- +listen: localhost:9293 +host: localhost +data.dir: /tmp/liftbridge/server-2 +activity.stream.enabled: true + +# Configure logging. +logging: + level: debug + raft: true + +# Define NATS cluster to connect to. +nats.servers: + - nats://localhost:4300 + - nats://localhost:4301 + +# Specify message stream settings. +streams: + retention.max: + age: 24h + messages: 100 + compact.enabled: true + +# Specify cluster settings. +clustering: + server.id: server-2 + raft.bootstrap.seed: true + replica.max.lag.time: 20s +``` + +## Overriding configuration settings with environment variables + +For configuration set in the configuration file the value can be overridden +with environment variables prefixed with `LIFTBRIDGE_`. The key must exist in +the config file to be overridden. + +For example using the config file from above one could override the host and +logging level with: + +```sh +env LIFTBRIDGE_HOST=liftbridge.example.com \ + LIFTBRIDGE_LOGGING_LEVEL=error \ + liftbridge --config config.yaml +``` + +## Configuration Settings + +Below is the list of Liftbridge configuration settings, including the name of +the setting in the configuration file and the CLI flag if it exists. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| listen | | The server listen host/port. This is the host and port the server will bind to. If this is not specified but `host` and `port` are specified, these values will be used. If neither `listen` nor `host`/`port` are specified, the default listen address will be used. | string | 0:0:0:0:9292 | | +| host | | The server host that is advertised to clients, i.e. the address clients will attempt to connect to based on metadata API responses. If not set, `listen` will be returned to clients. This value may differ from `listen` in situations where the external address differs from the internal address, e.g. when running in a container. If `listen` is not specified, the server will also bind to this host. | string | localhost | | +| port | port | The server port that is advertised to clients. See `host` for more information on how this behaves. | int | 9292 | | +| tls.key | tls-key | The private key file for server certificate. This must be set in combination with `tls.cert` to enable TLS. | string | | +| tls.cert | tls-cert | The server certificate file. This must be set in combination with `tls.key` to enable TLS. | string | | +| tls.client.auth.enabled | tls-client-auth | Enforce client-side authentication via certificate. | bool | false | +| tls.client.auth.ca | tls-client-auth-ca | The CA certificate file to use when authenticating clients. | string | | +| logging.level | level | The logging level. | string | info | [debug, info, warn, error] | +| logging.recovery | | Log messages resulting from the replay of the Raft log on server recovery. | bool | false | | +| logging.raft | | Enables logging in the Raft subsystem. | bool | false | | +| data.dir | data-dir | The directory to store data in. | string | /tmp/liftbridge/namespace | | +| batch.max.messages | | The maximum number of messages to batch when writing to disk. | int | 1024 | +| batch.max.time | | The maximum time to wait to batch more messages when writing to disk. | duration | 0 | | +| metadata.cache.max.age | | The maximum age of cached broker metadata. | duration | 2m | | +| nats | | NATS configuration. | map | | [See below](#nats-configuration-settings) | +| streams | | Write-ahead log configuration for message streams. | map | | [See below](#streams-configuration-settings) | +| clustering | | Broker cluster configuration. | map | | [See below](#clustering-configuration-settings) | +| activity | | Meta activity event stream configuration. | map | | [See below](#activity-configuration-settings) | +| cursors | | Cursor management configuration. | map | | [See below](#cursors-configuration-settings) | + +### NATS Configuration Settings + +Below is the list of the configuration settings for the `nats` section of +the configuration file. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| servers | nats-servers | List of NATS hosts to connect to. | list | nats://localhost:4222 | | +| user | | Username to use to connect to NATS servers. | string | | | +| password | | Password to use to connect to NATS servers. | string | | | +| tls.cert | | Path to NATS certificate file. | string | | | +| tls.key | | Path to NATS key file. | string | | | +| tls.ca | | Path to NATS CA Root file. | string | | | + +### Streams Configuration Settings + +Below is the list of the configuration settings for the `streams` section of the +configuration file. These settings are applied globally to all streams. +However, streams can be individually configured when they are created, +overriding these settings. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| retention.max.bytes | | The maximum size a stream's log can grow to, in bytes, before we will discard old log segments to free up space. A value of 0 indicates no limit. | int64 | 0 | | +| retention.max.messages | | The maximum size a stream's log can grow to, in number of messages, before we will discard old log segments to free up space. A value of 0 indicates no limit. | int64 | 0 | | +| retention.max.age | | The TTL for stream log segment files, after which they are deleted. A value of 0 indicates no TTL. | duration | 168h | | +| cleaner.interval | | The frequency to check if a new stream log segment file should be rolled and whether any segments are eligible for deletion based on the retention policy or compaction if enabled. | duration | 5m | | +| segment.max.bytes | | The maximum size of a single stream log segment file in bytes. Retention is always done a file at a time, so a larger segment size means fewer files but less granular control over retention. | int64 | 268435456 | | +| segment.max.age | | The maximum time before a new stream log segment is rolled out. A value of 0 means new segments will only be rolled when `segment.max.bytes` is reached. Retention is always done a file at a time, so a larger value means fewer files but less granular control over retention. | duration | value of `retention.max.age` | | +| compact.enabled | | Enables stream log compaction. Compaction works by retaining only the latest message for each key and discarding older messages. The frequency in which compaction runs is controlled by `cleaner.interval`. | bool | false | | +| compact.max.goroutines | | The maximum number of concurrent goroutines to use for compaction on a stream log (only applicable if `compact.enabled` is `true`). | int | 10 | | +| auto.pause.time | | The amount of time a stream partition can go idle, i.e. not receive a message, before it is automatically paused. A value of 0 disables auto pausing. | duration | 0 | | +| auto.pause.disable.if.subscribers | | Disables automatic stream partition pausing when there are subscribers. | bool | false | | + +### Clustering Configuration Settings + +Below is the list of the configuration settings for the `clustering` section of +the configuration file. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| server.id | server-id | ID of the server in the cluster. | string | random id | string with no spaces or periods | +| namespace | namespace | Cluster namespace. | string | liftbridge-default | string with no spaces or periods | +| raft.snapshot.retain | | The number Raft log snapshots to retain on disk. | int | 2 | | +| raft.snapshot.threshold | | Controls how many outstanding logs there must be before taking a snapshot. This prevents excessive snapshots when a small set of logs can be replayed. | int | 8192 | | +| raft.cache.size | | The number of Raft logs to hold in memory for quick lookup. | int | 512 | | +| raft.bootstrap.seed | raft-bootstrap-seed | Bootstrap the Raft cluster by electing self as leader if there is no existing state. If this is enabled, `raft.bootstrap.peers` should generally not be used, either on this node or peer nodes, since cluster topology is not being explicitly defined. Instead, peers should be started without bootstrap flags which will cause them to automatically discover the bootstrapped leader and join the cluster. | bool | false | | +| raft.bootstrap.peers | raft-bootstrap-peers | Bootstrap the Raft cluster with the provided list of peer IDs if there is no existing state. This should generally not be used in combination with `raft.bootstrap.seed` since it is explicitly defining cluster topology and the configured topology will elect a leader. Note that once the cluster is established, new nodes can join without setting bootstrap flags since they will automatically discover the elected leader and join the cluster. | list | | | +| replica.max.lag.time | | If a follower hasn't sent any replication requests or hasn't caught up to the leader's log end offset for at least this time, the leader will remove the follower from ISR. | duration | 15s | | +| replica.max.leader.timeout | | If a leader hasn't sent any replication responses for at least this time, the follower will report the leader to the controller. If a majority of the replicas report the leader, a new leader is selected by the controller. | duration | 15s | | +| replica.max.idle.wait | | The maximum amount of time a follower will wait before making a replication request once the follower is caught up with the leader. This value should always be less than `replica.max.lag.time` to avoid frequent shrinking of ISR for low-throughput streams. | duration | 10s | | +| replica.fetch.timeout | | Timeout duration for follower replication requests. | duration | 3s | | +| min.insync.replicas | | Specifies the minimum number of replicas that must acknowledge a stream write before it can be committed. If the ISR drops below this size, messages cannot be committed. | int | 1 | [1,...] | + +### Activity Configuration Settings + +Below is the list of the configuration settings for the `activity` section of +the configuration file. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| stream.enabled | | Enables the activity stream. This will create an internal stream called `__activity` which events will be published to. | bool | false | | +| stream.publish.timeout | | The timeout for publishes to the activity stream. This is the time to wait for an ack from the activity stream, which means it's related to `stream.publish.ack.policy`. If the ack policy is `none`, this has no effect. | duration | 5s | | +| stream.publish.ack.policy | | The ack policy to use for publishes to the activity stream. The value `none` means publishes will not wait for an ack, `leader` means publishes will wait for the ack sent when the leader has committed the event, and `all` means publishes will wait for the ack sent when all replicas have committed the event. | string | all | [none, leader, all] | + +### Cursors Configuration Settings + +Below is the list of the configuration settings for the `cursors` section of +the configuration file. + +| Name | Flag | Description | Type | Default | Valid Values | +|:----|:----|:----|:----|:----|:----| +| stream.partitions | | Sets the number of partitions for the internal `__cursors` stream which stores consumer cursors. A value of 0 disables the cursors stream. This cannot be changed once it is set. | int | 0 | | +| stream.auto.pause.time | | The amount of time a partition in the internal `__cursors` stream can go idle, i.e. not receive a cursor update or fetch, before it is automatically paused. A value of 0 disables auto pausing. | duration | 1m | | diff --git a/website/versioned_docs/version-v1.4.0/quick_start.md b/website/versioned_docs/version-v1.4.0/quick_start.md new file mode 100644 index 00000000..f48bb2ab --- /dev/null +++ b/website/versioned_docs/version-v1.4.0/quick_start.md @@ -0,0 +1,105 @@ +--- +id: version-v1.4.0-quick-start +title: Quick Start +original_id: quick-start +--- + +There are three ways to get started running Liftbridge on your machine: +[downloading a pre-built binary](#binary), [building from source](#building-from-source), +or [running a Docker container](#docker-container). There are also several +options for running a Liftbridge cluster [described below](#running-a-liftbridge-cluster-locally). + +## Binary + +A pre-built Liftbridge binary can be downloaded for a specific platform from +the [releases](https://github.com/liftbridge-io/liftbridge/releases) page. Once +you have installed the binary, refer to the steps under [Building From +Source](#building-from-source) for running the server. + +## Building From Source + +A Liftbridge binary can be built and installed from source using +[Go](https://golang.org/doc/install). Follow the below step to install from +source. + +```shell +$ go get github.com/liftbridge-io/liftbridge +``` +*Liftbridge uses [Go modules](https://github.com/golang/go/wiki/Modules), so +ensure this is enabled, e.g. `export GO111MODULE=on`.* + +Liftbridge currently relies on an externally running +[NATS server](https://github.com/nats-io/gnatsd). By default, it will connect +to a NATS server running on localhost. The `--nats-servers` flag allows +configuring the NATS server(s) to connect to. + +Also note that Liftbridge is clustered by default and relies on Raft for +coordination. This means a cluster of three or more servers is normally run +for high availability, and Raft manages electing a leader. A single server is +actually a cluster of size 1. For safety purposes, the server cannot elect +itself as leader without using the `--raft-bootstrap-seed` flag, which will +indicate to the server to elect itself as leader. This will start a single +server that can begin handling requests. **Use this flag with caution as it should +only be set on one server when bootstrapping a cluster.** + +```shell +$ liftbridge --raft-bootstrap-seed +INFO[2020-10-15 14:29:50] Liftbridge Version: v1.4.0 +INFO[2020-10-15 14:29:50] Server ID: 4nbhBr66WnRsy0I5oKF9bo +INFO[2020-10-15 14:29:50] Namespace: liftbridge-default +INFO[2020-10-15 14:29:50] Default Retention Policy: [Age: 1 week, Compact: false] +INFO[2020-10-15 14:29:50] Default Partition Pausing: disabled +INFO[2020-10-15 14:29:50] Starting server on 0.0.0.0:9292... +INFO[2020-10-15 14:29:51] Server became metadata leader, performing leader promotion actions +``` + +Once a leader has been elected, other servers will automatically join the cluster. +We set the `--data-dir` and `--port` flags to avoid clobbering the first server. + +```shell +$ liftbridge --data-dir /tmp/liftbridge/server-2 --port=9293 +INFO[2020-10-15 14:30:48] Liftbridge Version: v1.4.0 +INFO[2020-10-15 14:30:48] Server ID: lbW05esZTab3guEwcmWD9M +INFO[2020-10-15 14:30:48] Namespace: liftbridge-default +INFO[2020-10-15 14:30:48] Default Retention Policy: [Age: 1 week, Compact: false] +INFO[2020-10-15 14:30:48] Default Partition Pausing: disabled +INFO[2020-10-15 14:30:48] Starting server on 0.0.0.0:9293... +``` + +We can also bootstrap a cluster by providing the explicit cluster configuration. +To do this, we provide the IDs of the participating peers in the cluster using the +`--raft-bootstrap-peers` flag. Raft will then handle electing a leader. + +```shell +$ liftbridge --raft-bootstrap-peers server-2,server-3 +``` + +## Docker Container + +Instead of running a binary, you can run Liftbridge using a container. There is +a [container image](https://hub.docker.com/r/liftbridge/standalone-dev) +available which runs an instance of Liftbridge and NATS inside a single Docker +container. This is meant for development and testing purposes. Use the +following Docker commands to run this container: + +```shell +$ docker pull liftbridge/standalone-dev +$ docker run -d --name=liftbridge-main -p 4222:4222 -p 9292:9292 -p 8222:8222 -p 6222:6222 liftbridge/standalone-dev +``` + +This will run the container which will start both the NATS and Liftbridge +servers. To check the logs to see if the container started properly, run: + +```shell +$ docker logs liftbridge-main +``` + +See the [deployment guide](./deployment.md) for more information. + +## Running a Liftbridge Cluster Locally + +The quickest way to get a multi-node Liftbridge cluster up and running on your +machine is with either [Docker Compose](https://docs.docker.com/compose) or +[Kind](https://kind.sigs.k8s.io) (Kubernetes in Docker). Follow the +[deployment guide](./deployment.md) for help running a cluster locally for +development or testing. diff --git a/website/versions.json b/website/versions.json index e59ce7f6..c84e64ee 100644 --- a/website/versions.json +++ b/website/versions.json @@ -1,4 +1,5 @@ [ + "v1.4.0", "v1.3.0", "v1.2.0", "v1.1.0",