diff --git a/README.md b/README.md index d62427b7..70b15ef4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: Apache-2.0](https://img.shields.io/github/license/strangelove-ventures/cosmos-operator.svg?style=flat-square)](https://github.com/strangelove-ventures/cosmos-operator/blob/main/LICENSE) [![Version](https://img.shields.io/github/tag/strangelove-ventures/cosmos-operator.svg?style=flat-square)](https://github.com/cosmos/strangelove-ventures/cosmos-operator) -Cosmos Operator is a [Kubernetes Operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) for blockchains built with the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk). +Cosmos Operator is a [Kubernetes Operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) primarily for blockchains built with the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk). It also supports [Penumbra](https://github.com/penumbra-zone/penumbra) and other chains which use [CometBFT](https://github.com/cometbft/cometbft) for consensus. The long-term vision of this operator is to allow you to "configure it and forget it". @@ -39,12 +39,16 @@ The CosmosFullNode controller is like a StatefulSet for running Cosmos SDK block A CosmosFullNode can be configured to run as an RPC node, a validator sentry, or a seed node. All configurations can be used as persistent peers. -As of this writing, Strangelove has been running CosmosFullNode in production for many months. +As of this writing, Strangelove has been running CosmosFullNode in production for over a year. + +## Samples [Minimal example yaml](./config/samples/cosmos_v1_cosmosfullnode.yaml) [Full example yaml](./config/samples/cosmos_v1_cosmosfullnode_full.yaml) +[Penumbra example yaml](./config/samples/cosmos_v1_cosmosfullnode_penumbra.yaml) + ### Why not a StatefulSet? Each pod requires different config, such as peer settings in config.toml and mounted node keys. Therefore, a blanket template as found in StatefulSet did not suffice. @@ -75,7 +79,7 @@ See the [best practices guide for CosmosFullNode](./docs/fullnode_best_practices Disclaimer: Strangelove has not committed to these enhancements and cannot estimate when they will be completed. -- [ ] Scheduled upgrades. Set a halt height and image version. The controller performs a rolling update with the new image version after the committed halt height. +- [x] Scheduled upgrades. Set the upgrade height and image version, optionally setting halt height. The controller performs a rolling update with the new image version after the committed height. - [x] Support configuration suitable for validator sentries. - [x] Reliable, persistent peer support. - [x] Quicker p2p discovery using private peers. diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go index 0f2984c8..24d13e2c 100644 --- a/api/v1/cosmosfullnode_types.go +++ b/api/v1/cosmosfullnode_types.go @@ -436,6 +436,8 @@ type ChainSpec struct { Comet CometConfig `json:"config"` // App configuration applied to app.toml. + // Although optional, it's highly recommended you configure this field. + // +optional App SDKAppConfig `json:"app"` // One of trace|debug|info|warn|error|fatal|panic. diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml index 528e81a7..92fde62f 100644 --- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml +++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml @@ -81,7 +81,9 @@ spec: Use AddrbookScript if the chain has an unconventional file format or address book location. type: string app: - description: App configuration applied to app.toml. + description: |- + App configuration applied to app.toml. + Although optional, it's highly recommended you configure this field. properties: apiEnableUnsafeCORS: description: Defines if CORS should be enabled for the API @@ -383,7 +385,6 @@ spec: type: object type: array required: - - app - binary - chainID - network diff --git a/config/samples/cosmos_v1_cosmosfullnode_penumbra.yaml b/config/samples/cosmos_v1_cosmosfullnode_penumbra.yaml new file mode 100644 index 00000000..0d24300e --- /dev/null +++ b/config/samples/cosmos_v1_cosmosfullnode_penumbra.yaml @@ -0,0 +1,134 @@ +# Example Penumbra Sentry configuration +# 3 sentries, 1 per worker node +apiVersion: cosmos.strange.love/v1 +kind: CosmosFullNode +metadata: + name: penumbra-testnet-sentry +spec: + type: Sentry + replicas: 3 + chain: + chainID: penumbra-testnet-deimos-8 + network: testnet + versions: + # Genesis version + - height: 0 + image: ghcr.io/strangelove-ventures/heighliner/cometbft:v0.37.5 + containers: + pd: ghcr.io/strangelove-ventures/heighliner/penumbra:v0.77.2 + # Stage upgrades at future block heights by adding the version information here + config: + maxInboundPeers: 20 + maxOutboundPeers: 20 + overrides: |- + moniker = "strangelove" + fast_sync = true + + [consensus] + timeout_commit = "5000ms" + + [fastsync] + version = "v0" + seeds: 778faa70d7426d6cfc2561293d3108cea14029e5@35.225.116.144:26656,2a1c938b033750194b3b94ec94c45530af64da06@223.130.134.24:26656,d90429e1dda28210b71bdf265e004afc0f77590c@84.46.247.166:26656,65a1f42ac9be28dcd50c472a30e2532e5566559a@161.97.90.165:26656,b419b447a126a31d2396e87fb99f92a0bd66f324@84.46.247.160:26656,66eabb033f37a0d405e61c4609c8db1d78c193b6@84.46.247.17:26656,476d4ac6146807ed605d794d84f7235c982af595@78.46.84.125:42656 + genesisScript: echo "No need for genesis" + addrbookScript: echo "No need for address book" + skipInvariants: false + binary: cometbft + + podTemplate: + affinity: + podAntiAffinity: + # One sentry per node + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: penumbra-testnet-sentry + topologyKey: kubernetes.io/hostname + initContainers: + # Initialize the cometbft tmp configuration for the correct version, used for config overlay + - name: chain-init + command: + - cometbft + - init + - --home /home/operator/.tmp + volumeMounts: + - mountPath: /home/operator/.tmp + name: vol-tmp + env: + - name: HOME + value: /home/operator/cosmos + + # Initialize the cometbft and pd data from a snapshot if it is not already initialized + - name: pd-init + image: ghcr.io/strangelove-ventures/heighliner/penumbra:v0.77.2 + command: + - sh + - -c + - | + HOME=/home/operator/cosmos pd testnet join --archive-url "https://snapshots.penumbra.zone/testnet/pd-migrated-state-76-77.tar.gz" || echo "Already initialized" + volumeMounts: + - mountPath: /home/operator/cosmos + name: vol-chain-home + + # If pd-init initialized the data, move the cometbft data to the operator expected location and + # move the pd data also to the root of the vol-chain-home volume so that it is included in snapshots + - name: pd-move + image: busybox:latest + command: + - sh + - -c + - | + HOME=/home/operator/cosmos + NODE0=$HOME/.penumbra/testnet_data/node0 + mv $NODE0/cometbft/* $HOME/ || echo "Already moved cometbft data" + mv $NODE0/pd $HOME/ || echo "Already moved pd data" + volumeMounts: + - mountPath: /home/operator/cosmos + name: vol-chain-home + containers: + # pd sidecar container, communicates with cometbft "node" container + - name: pd + command: + - sh + - -c + - | + pd start --home /home/operator/cosmos/pd + ports: + # CometBFT connects to PD on port 26658 + - containerPort: 26658 + name: abci + protocol: TCP + - containerPort: 8080 + name: grpc + protocol: TCP + volumeMounts: + - mountPath: /home/operator/cosmos + name: vol-chain-home + probes: + # Disable liveness and readiness probes for sentries + strategy: None + resources: + requests: + cpu: 4000m + memory: 10Gi + limits: + memory: 64Gi + selfHeal: + heightDriftMitigation: + threshold: 10 + pvcAutoScale: + increaseQuantity: 10% + maxSize: 5Ti + usedSpacePercentage: 75 + service: + # Create a kubernetes NodePort service for the P2P port on each sentry + maxP2PExternalAddresses: 3 + p2pTemplate: + type: NodePort + volumeClaimTemplate: + resources: + requests: + # Initial storage size for the PVC, needs to be enough for snapshot restore + storage: 20Gi + # Storage class for the PVC, will use default storage class if not specified + storageClassName: topolvm-provisioner-thin-striped