add signer and update versions

Added signer docker compose file as extra service, which can be started with an optional flag on all available networks. Updated node and API versions to the latest ones.

Author: bowtiedbot

.gitignore

@@ -28,9 +28,11 @@ Session.vim
 node_modules/
 stacks-node-follower/Config.toml
 configurations/*/Config.toml
+configurations/*/Signer.toml
 configurations/private-testnet/*.toml
 
 conf/*/Config.toml
+conf/*/Signer.toml
 conf/*/Config-with-bitcoin-flag.toml
 conf/*/bitcoin.conf
 conf/private-testnet/*.toml

README.md

@@ -25,7 +25,7 @@ cp sample.env .env
 ### Seed chainstate from Hiro Archiver
 
 Using data from the [Hiro Archiver](https://docs.hiro.so/hiro-archive) service, this script will download the latest files, extract them and restore the postgres data. \
-_**Note**: it can take a long time to process the data, and you'll need at a minimum roughly 150GB of free space_
+_**Note**: it can take a long time to process the data, and you'll need at a minimum roughly 350GB of free space_
 
 ```bash
 sudo ./scripts/seed-chainstate.sh

conf/mainnet/Config.toml.sample

@@ -3,11 +3,7 @@ working_dir = "/root/stacks-blockchain/data"
 rpc_bind = "0.0.0.0:20443"
 p2p_bind = "0.0.0.0:20444"
 bootstrap_node = "02196f005965cebe6ddc3901b7b1cc1aa7a88f305bb8c5893456b8f9a605923893@seed.mainnet.hiro.so:20444"
-
-[[events_observer]]
-endpoint = "stacks-blockchain-api:3700"
-retry_count = 255
-events_keys = ["*"]
+stacker = true
 
 [burnchain]
 chain = "bitcoin"
@@ -17,3 +13,16 @@ username = "stacks"
 password = "foundation"
 rpc_port = 8332
 peer_port = 8333
+
+[connection_options]
+auth_token = "1234"
+private_neighbors = false
+
+[[events_observer]]
+events_keys = ["stackerdb", "block_proposal", "burn_blocks"]
+endpoint = "stacks-signer:30000"
+
+[[events_observer]]
+endpoint = "stacks-blockchain-api:3700"
+events_keys = ["*"]
+timeout_ms = 300_000

conf/mocknet/Config.toml.sample

@@ -3,17 +3,22 @@ rpc_bind = "0.0.0.0:20443"
 p2p_bind = "0.0.0.0:20444"
 wait_time_for_microblocks = 10000
 use_test_genesis_chainstate = true
-
-[[events_observer]]
-endpoint = "stacks-blockchain-api:3700"
-retry_count = 255
-events_keys = ["*"]
+stacker = true
 
 [burnchain]
 chain = "bitcoin"
 mode = "mocknet"
 commit_anchor_block_within = 5000
 
+[[events_observer]]
+events_keys = ["stackerdb", "block_proposal", "burn_blocks"]
+endpoint = "stacks-signer:30000"
+
+[[events_observer]]
+endpoint = "stacks-blockchain-api:3700"
+events_keys = ["*"]
+timeout_ms = 60_000
+
 [[ustx_balance]]
 # "mnemonic": "point approve language letter cargo rough similar wrap focus edge polar task olympic tobacco cinnamon drop lawn boring sort trade senior screen tiger climb",
 # "keyInfo": {
@@ -59,6 +64,8 @@ address = "STSTW15D618BSZQB85R058DS46THH86YQQY6XCB7"
 amount = 100000000000000
 
 [connection_options]
+auth_token = "1234"
+private_neighbors = false
 read_only_call_limit_write_length = 0
 read_only_call_limit_read_length = 100000
 read_only_call_limit_write_count = 0

conf/testnet/Config.toml.sample

@@ -2,22 +2,31 @@
 working_dir = "/root/stacks-blockchain/data"
 rpc_bind = "0.0.0.0:20443"
 p2p_bind = "0.0.0.0:20444"
-bootstrap_node="029266faff4c8e0ca4f934f34996a96af481df94a89b0c9bd515f3536a95682ddc@seed.testnet.hiro.so:30444"
-always_use_affirmation_maps = false
+bootstrap_node = "029266faff4c8e0ca4f934f34996a96af481df94a89b0c9bd515f3536a95682ddc@seed.testnet.hiro.so:30444"
+always_use_affirmation_maps = true
+require_affirmed_anchor_blocks = true
+pox_sync_sample_secs = 30
+stacker = true
 
 [burnchain]
-chain = "bitcoin"
-mode = "xenon"
-peer_host = "bitcoin.testnet.stacks.org"
-username = "stacks"
-password = "foundation"
-rpc_port = 18332
-peer_port = 18333
+mode = "krypton"
+peer_host = "bitcoin.regtest.hiro.so"
+peer_port = 18444
+pox_prepare_length = 100
+pox_reward_length = 900
+
+[connection_options]
+auth_token = "1234"
+private_neighbors = false
+
+[[events_observer]]
+events_keys = ["stackerdb", "block_proposal", "burn_blocks"]
+endpoint = "stacks-signer:30000"
 
 [[events_observer]]
 endpoint = "stacks-blockchain-api:3700"
-retry_count = 255
 events_keys = ["*"]
+timeout_ms = 60_000
 
 [[ustx_balance]]
 address = "ST2QKZ4FKHAH1NQKYKYAYZPY440FEPK7GZ1R5HBP2"
@@ -34,3 +43,39 @@ amount = 10000000000000000
 [[ustx_balance]]
 address = "ST2TFVBMRPS5SSNP98DQKQ5JNB2B6NZM91C4K3P7B"
 amount = 10000000000000000
+
+[[burnchain.epochs]]
+epoch_name = "1.0"
+start_height = 0
+
+[[burnchain.epochs]]
+epoch_name = "2.0"
+start_height = 0
+
+[[burnchain.epochs]]
+epoch_name = "2.05"
+start_height = 1
+
+[[burnchain.epochs]]
+epoch_name = "2.1"
+start_height = 2
+
+[[burnchain.epochs]]
+epoch_name = "2.2"
+start_height = 3
+
+[[burnchain.epochs]]
+epoch_name = "2.3"
+start_height = 4
+
+[[burnchain.epochs]]
+epoch_name = "2.4"
+start_height = 5
+
+[[burnchain.epochs]]
+epoch_name = "2.5"
+start_height = 6
+
+[[burnchain.epochs]]
+epoch_name = "3.0"
+start_height = 56_457

docs/config.md

@@ -5,7 +5,9 @@ _Note: Burnchchain environment variables defined in `./env` will overwrite value
 
 - `.env`
 - `./conf/mainnet/Config.toml`
+- `./conf/mainnet/Signer.toml`
 - `./conf/testnet/Config.toml`
+- `./conf/testnet/Signer.toml`
 
 ## Environment Variables
 
@@ -19,10 +21,22 @@ Most variables in `.env` shouldn't be modified, but there are a few you may wish
 | `DOCKER_NETWORK`                | Name of docker network used to launch services        | `stacks`                  |
 | `EXPOSE_POSTGRES`               | Expose postgres service to the host OS                | `false`                   |
 | `STACKS_BLOCKCHAIN_VERSION`     | Stacks Blockchain Docker image version                | `latest released version` |
+| `STACKS_SIGNER_VERSION`         | Stacks Signer Docker image version                    | `latest released version` |
 | `STACKS_BLOCKCHAIN_API_VERSION` | Stacks Blockchain API Docker image version            | `latest released version` |
 | `POSTGRES_VERSION`              | Postgres Docker image version                         | `14`                      |
 | `NGINX_PROXY_PORT`              | HTTP port for the nginx proxy                         | `80`                      |
 
+### Stacks Signer Settings
+
+#### You must set values for the _SIGNER_PRIVATE_KEY_ on the specific network if you're running a signer, otherwise they can be left empty.
+
+| Name                         | Description                                                                                      | Default Value |
+| ---------------------------- | ------------------------------------------------------------------------------------------------ | ------------- |
+| `AUTH_TOKEN`                 | Authorization token for HTTP requests made from the signer to your Stacks node                   | `1234`        |
+| `SIGNER_PRIVATE_KEY`         | The private key of the signer, on mainnet.                                                       |               |
+| `TESTNET_SIGNER_PRIVATE_KEY` | The private key of the signer, on testnet.                                                       |               |
+| `STACKS_SIGNER_PORT`         | The port where the signer will expose an RPC endpoint for receiving events from your Stacks node | `30000`       |
+
 ### API Settings
 
 #### Recommened to leave these settings _as is_
@@ -81,8 +95,6 @@ Most variables in `.env` shouldn't be modified, but there are a few you may wish
 
 | Name            | Description                           | Default Value                |
 | --------------- | ------------------------------------- | ---------------------------- |
-| `TBTC_HOST`     | FQDN of bitcoin mainnnet host         | `bitcoin.testnet.stacks.org` |
-| `TBTC_RPC_USER` | RPC username for bitcoin mainnet host | `stacks`                     |
-| `TBTC_RPC_PASS` | RPC password for bitcoin mainnet host | `foundation`                 |
-| `TBTC_RPC_PORT` | RPC port for bitcoin mainnet host     | `18332`                      |
-| `TBTC_P2P_PORT` | P2P port for bitcoin mainnet host     | `18333`                      |
+| `TBTC_HOST`     | FQDN of bitcoin testnet host         | `bitcoin.regtest.hiro.so` |
+| `TBTC_RPC_PORT` | RPC port for bitcoin testnet host     | `18332`                      |
+| `TBTC_P2P_PORT` | P2P port for bitcoin testnet host     | `18333`                      |

docs/usage.md

@@ -8,9 +8,10 @@ Usage:
         -n|--network: [ mainnet | testnet | mocknet ]
         -a|--action: [ start | stop | logs | reset | upgrade | import | export | bns ]
     optional args:
-        -f|--flags: [ proxy ]
+        -f|--flags: [ signer,proxy ]
         export: combined with 'logs' action, exports logs to a text file
     ex: ./manage.sh -n mainnet -a start -f proxy
+	ex: ./manage.sh -n mainnet -a start -f signer,proxy
     ex: ./manage.sh --network mainnet --action start --flags proxy
     ex: ./manage.sh -n mainnet -a logs export
 ```
@@ -25,6 +26,16 @@ Usage:
 ./manage.sh -n <network> -a restart
 ```
 
+#### With optional signer
+
+```bash
+./manage.sh -n <network> -a start -f signer
+```
+
+```bash
+./manage.sh -n <network> -a restart -f signer
+```
+
 #### With optional proxy
 
 ```bash

manage.sh

@@ -8,6 +8,7 @@ shopt -s expand_aliases
 export NETWORK="mainnet"
 export ACTION=""
 export PROFILE="stacks-blockchain"
+SIGNER=false
 STACKS_CHAIN_ID="2147483648"
 STACKS_SHUTDOWN_TIMEOUT=1200 # default to 20 minutes, during sync it can take a long time to stop the runloop
 LOG_TAIL="100"
@@ -154,9 +155,10 @@ usage() {
 	log "        -n|--network: [ mainnet | testnet | mocknet ]"
 	log "        -a|--action: [ start | stop | logs | reset | upgrade | import | export | bns ]"
 	log "    optional args:"
-	log "        -f|--flags: [ proxy ]"
+	log "        -f|--flags: [ signer,proxy ]"
 	log "        export: combined with 'logs' action, exports logs to a text file"
 	log "    ex: ${COLCYAN}${0} -n mainnet -a start -f proxy${COLRESET}"
+	log "    ex: ${COLCYAN}${0} -n mainnet -a start -f signer,proxy${COLRESET}"
 	log "    ex: ${COLCYAN}${0} --network mainnet --action start --flags proxy${COLRESET}"
 	log "    ex: ${COLCYAN}${0} -n mainnet -a logs export${COLRESET}"
 	echo
@@ -434,14 +436,62 @@ events_file_env(){
 
 # Function that updates Config.toml
 update_configs(){
-    if [ "${NETWORK}" == "testnet" ]; then
+	if [ "${NETWORK}" == "testnet" ]; then
 		BTC_HOST=${TBTC_HOST}
 		BTC_RPC_USER=${TBTC_RPC_USER}
 		BTC_RPC_PASS=${TBTC_RPC_PASS}
 		BTC_RPC_PORT=${TBTC_RPC_PORT}
 		BTC_P2P_PORT=${TBTC_P2P_PORT}
-    fi
+		SIGNER_PRIVATE_KEY=${TESTNET_SIGNER_PRIVATE_KEY}
+	fi
 	CONFIG_TOML="${SCRIPTPATH}/conf/${NETWORK}/Config.toml"
+	SIGNER_TOML="${SCRIPTPATH}/conf/${NETWORK}/Signer.toml"
+
+		## update Config.toml with signer options
+		if [ "${SIGNER}" != "true" ]; then
+			${VERBOSE} && log "${COLYELLOW}Disabling signer options in ${CONFIG_TOML}${COLRESET}"
+			sed -i.tmp "
+				/^\[\[events_observer\]\]/{
+					:a
+					N
+					/endpoint.*stacks-signer/!ba
+					s/^/#/mg
+				}
+				/^stacker = true/ s/^/#/
+			" "${CONFIG_TOML}" || {
+					log_exit "Unable to update values in Config.toml file: ${COLCYAN}${CONFIG_TOML}${COLRESET}"
+			}
+    else
+			[ ! ${SIGNER_PRIVATE_KEY} ] && log_exit "Signer private key not set!"
+			${VERBOSE} && log "${COLYELLOW}Enabling signer options in ${CONFIG_TOML}${COLRESET}"
+			sed -i.tmp "
+				/^#\[\[events_observer\]\]/{
+					:a
+					N
+					/endpoint.*stacks-signer/!ba
+					s/^#//mg
+				}
+				/^#stacker = true/ s/^#//
+			" "${CONFIG_TOML}" || {
+					log_exit "Unable to update values in Config.toml file: ${COLCYAN}${CONFIG_TOML}${COLRESET}"
+			}
+
+				## update Signer.toml with env vars
+			[[ ! -f "${SIGNER_TOML}" ]] && cp "${SIGNER_TOML}.sample" "${SIGNER_TOML}"
+			${VERBOSE} && log "${COLYELLOW}Updating values in ${SIGNER_TOML} from .env${COLRESET}"
+				$(sed -i.tmp "
+				/^node_host/s/.*/node_host = \"${STACKS_CORE_RPC_HOST}:${STACKS_CORE_RPC_PORT}\"/;
+				/^endpoint/s/.*/endpoint = \"0.0.0.0:${STACKS_SIGNER_PORT}\"/;
+				/^auth_password/s/.*/auth_password = \"${AUTH_TOKEN}\"/;
+				/^stacks_private_key/s/.*/stacks_private_key = \"${SIGNER_PRIVATE_KEY}\"/;
+			" "${SIGNER_TOML}" 2>&1) || {
+						log_exit "Unable to update values in Signer.toml file: ${COLCYAN}${SIGNER_TOML}${COLRESET}"
+				}
+				${VERBOSE} && log "${COLYELLOW}Deleting temp Signer.toml file: ${SIGNER_TOML}.tmp${COLRESET}"
+				$(rm "${SIGNER_TOML}.tmp" 2>&1) || {
+						log_exit "Unable to delete tmp Signer.toml file: ${COLCYAN}${SIGNER_TOML}.tmp${COLRESET}"
+				}
+    fi
 
     ## update Config.toml with btc vars
 	[[ ! -f "${CONFIG_TOML}" ]] && cp "${CONFIG_TOML}.sample" "${CONFIG_TOML}"
@@ -452,6 +502,8 @@ update_configs(){
 		/^password/s/.*/password = \"${BTC_RPC_PASS}\"/;
 		/^rpc_port/s/.*/rpc_port = ${BTC_RPC_PORT}/;
 		/^peer_port/s/.*/peer_port = ${BTC_P2P_PORT}/;
+		/^auth_token/s/.*/auth_token = \"${AUTH_TOKEN}\"/;
+		/^endpoint = \"stacks-signer/s/.*/endpoint = \"stacks-signer:${STACKS_SIGNER_PORT}\"/;
 	" "${CONFIG_TOML}" 2>&1) || {
         log_exit "Unable to update values in Config.toml file: ${COLCYAN}${CONFIG_TOML}${COLRESET}"
     }
@@ -547,6 +599,12 @@ docker_up() {
 	if ! check_event_replay; then
 		log_exit "Event-replay in progress. Refusing to start services"
 	fi
+
+	# Set signer env based on flag
+	if [[ "${FLAGS_ARRAY[*]}" == *"signer"* ]]; then
+		SIGNER=true
+	fi
+
 	# Sanity checks before starting services
 	local param="-d"
 	if [ "${PROFILE}" == "bns" ]; then
@@ -567,9 +625,10 @@ docker_up() {
 			${VERBOSE} && log "created (recursive) persistent-data dir ${SCRIPTPATH}/persistent-data/${NETWORK}/event-replay"
 		fi
 		${VERBOSE} && log "Using existing data dir: ${SCRIPTPATH}/persistent-data/${NETWORK}"
-		update_configs
 	fi
-	
+
+	update_configs
+
     # # See if we can detect a Hiro API major version change requiring an event-replay import
 	# if check_api; then
 	# 	log_warn "    Required to perform a stacks-blockchain-api event-replay:"