Skip to content

Commit 5a5e42b

Browse files
echobtechobt
committed
test(validation): improve multi-validator checks
Streamlined scripts/test-comprehensive.sh and refactored tests/docker/test-multi-validator.sh with clearer output and logic adjustments for validator monitoring.
1 parent 9607e9f commit 5a5e42b

2 files changed

Lines changed: 120 additions & 143 deletions

File tree

‎scripts/test-comprehensive.sh‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -170,18 +170,12 @@ log_info "======================================================================
170170
if platform_should_run_docker; then
171171
if platform_require_compose; then
172172
platform_ensure_network
173-
log_info "Starting compose stack..."
174-
if platform_compose -f "${PLATFORM_TEST_COMPOSE_FILE}" up --build -d 2>&1 | tee "${PLATFORM_TEST_LOG_DIR}/compose-up.log"; then
175-
log_success "Compose stack started"
173+
log_info "Running multi-validator docker test harness..."
174+
if "${SCRIPT_DIR}/../tests/docker/test-multi-validator.sh" 2>&1 | tee "${PLATFORM_TEST_LOG_DIR}/multi-validator-docker.log"; then
175+
log_success "Multi-validator docker test completed"
176176
else
177-
log_failure "Compose stack failed to start"
177+
log_failure "Multi-validator docker test failed"
178178
fi
179-
180-
log_info "Collecting compose logs..."
181-
platform_compose -f "${PLATFORM_TEST_COMPOSE_FILE}" logs --no-color 2>&1 | tee "${PLATFORM_TEST_LOG_DIR}/compose.log" || true
182-
183-
log_info "Tearing down compose stack..."
184-
platform_compose -f "${PLATFORM_TEST_COMPOSE_FILE}" down -v 2>&1 | tee "${PLATFORM_TEST_LOG_DIR}/compose-down.log" || true
185179
else
186180
log_skip "Docker Compose not available"
187181
fi

‎tests/docker/test-multi-validator.sh‎

Lines changed: 116 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
#!/bin/bash
22
# =============================================================================
3-
# Platform Multi-Validator Integration Test
3+
# Platform Multi-Validator Integration Test (Docker)
44
# =============================================================================
5-
# Tests multiple validators in a P2P network without Docker build issues
6-
# Uses locally built binary
7-
# Note: This script does not require Docker; see scripts/test-comprehensive.sh for Docker-only test suites
5+
# Spins up a full multi-validator + mock Subtensor network via Docker Compose.
6+
# Verifies health, distributed DB creation, P2P connectivity, and mock chain
7+
# commit/reveal inspection endpoints. Logs and artifacts are written to
8+
# PLATFORM_TEST_ARTIFACTS_DIR.
89
# =============================================================================
910

1011
set -euo pipefail
@@ -14,153 +15,135 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
1415
source "${SCRIPT_DIR}/../../scripts/test-harness.sh"
1516

1617
platform_test_init
17-
trap platform_cleanup_run_dir EXIT
1818

19-
if [ "${PLATFORM_TEST_DOCKER_MODE:-auto}" = "required" ]; then
20-
platform_install_docker_if_needed
21-
fi
19+
ARTIFACT_DIR="${PLATFORM_TEST_ARTIFACTS_DIR}/multi-validator"
20+
LOG_DIR="${ARTIFACT_DIR}/logs"
21+
mkdir -p "${ARTIFACT_DIR}" "${LOG_DIR}"
2222

23-
NUM_VALIDATORS="${NUM_VALIDATORS:-3}"
24-
BASE_PORT="${BASE_PORT:-9100}"
25-
VALIDATOR_BINARY="${VALIDATOR_BINARY:-${PLATFORM_TEST_ROOT}/target/release/validator-node}"
26-
PLATFORM_TEST_RUN_DIR="${PLATFORM_TEST_RUN_DIR:-${PLATFORM_TEST_TMP_BASE}/multi-validator}"
23+
COMPOSE_FILE="${PLATFORM_TEST_COMPOSE_FILE}"
24+
COMPOSE_PROJECT="${PLATFORM_TEST_COMPOSE_PROJECT}"
2725

28-
log_info "Test directory: ${PLATFORM_TEST_RUN_DIR}"
29-
mkdir -p "${PLATFORM_TEST_RUN_DIR}"
26+
cleanup_compose() {
27+
if platform_has_compose; then
28+
platform_compose -f "${COMPOSE_FILE}" logs --no-color > "${LOG_DIR}/compose.log" 2>&1 || true
29+
platform_compose -f "${COMPOSE_FILE}" down -v > "${LOG_DIR}/compose-down.log" 2>&1 || true
30+
fi
31+
platform_cleanup_run_dir
32+
}
3033

31-
if [ ! -x "${VALIDATOR_BINARY}" ]; then
32-
log_failure "Validator binary not found at ${VALIDATOR_BINARY}"
33-
log_info "Building..."
34-
cargo build --release --bin validator-node
35-
fi
36-
log_success "Validator binary found"
37-
38-
declare -a VALIDATOR_PIDS
39-
40-
VALIDATOR_1_DIR="${PLATFORM_TEST_RUN_DIR}/validator-1"
41-
mkdir -p "${VALIDATOR_1_DIR}"
42-
VALIDATOR_1_KEY="0x0000000000000000000000000000000000000000000000000000000000000001"
43-
VALIDATOR_1_PORT="${BASE_PORT}"
44-
45-
log_info "Starting Validator 1 (bootstrap) on port ${VALIDATOR_1_PORT}..."
46-
RUST_LOG=info,validator_node=debug,platform_p2p_consensus=debug \
47-
"${VALIDATOR_BINARY}" \
48-
--secret-key "${VALIDATOR_1_KEY}" \
49-
--data-dir "${VALIDATOR_1_DIR}" \
50-
--listen-addr "/ip4/127.0.0.1/tcp/${VALIDATOR_1_PORT}" \
51-
--netuid 100 \
52-
--no-bittensor \
53-
> "${VALIDATOR_1_DIR}/output.log" 2>&1 &
54-
VALIDATOR_PIDS+=($!)
55-
log_info "Validator 1 PID: ${VALIDATOR_PIDS[0]}"
56-
57-
sleep 3
58-
59-
if ! kill -0 "${VALIDATOR_PIDS[0]}" 2>/dev/null; then
60-
log_failure "Validator 1 failed to start"
61-
tail -50 "${VALIDATOR_1_DIR}/output.log" || true
62-
exit 1
34+
trap cleanup_compose EXIT
35+
36+
if ! platform_should_run_docker; then
37+
log_skip "Docker not available; skipping multi-validator docker test"
38+
exit 0
6339
fi
64-
log_success "Validator 1 started"
65-
66-
for i in $(seq 2 "${NUM_VALIDATORS}"); do
67-
VALIDATOR_DIR="${PLATFORM_TEST_RUN_DIR}/validator-${i}"
68-
mkdir -p "${VALIDATOR_DIR}"
69-
VALIDATOR_KEY="0x000000000000000000000000000000000000000000000000000000000000000${i}"
70-
VALIDATOR_PORT=$((BASE_PORT + i - 1))
71-
72-
log_info "Starting Validator ${i} on port ${VALIDATOR_PORT}..."
73-
RUST_LOG=info,validator_node=debug,platform_p2p_consensus=debug \
74-
"${VALIDATOR_BINARY}" \
75-
--secret-key "${VALIDATOR_KEY}" \
76-
--data-dir "${VALIDATOR_DIR}" \
77-
--listen-addr "/ip4/127.0.0.1/tcp/${VALIDATOR_PORT}" \
78-
--bootstrap "/ip4/127.0.0.1/tcp/${VALIDATOR_1_PORT}" \
79-
--netuid 100 \
80-
--no-bittensor \
81-
> "${VALIDATOR_DIR}/output.log" 2>&1 &
82-
VALIDATOR_PIDS+=($!)
83-
log_info "Validator ${i} PID: ${VALIDATOR_PIDS[$((i-1))]}"
84-
sleep 1
85-
done
8640

87-
log_info "Waiting for validators to initialize..."
88-
sleep 5
41+
platform_require_compose
42+
platform_ensure_network
8943

90-
RUNNING=0
91-
for i in $(seq 1 "${NUM_VALIDATORS}"); do
92-
if kill -0 "${VALIDATOR_PIDS[$((i-1))]}" 2>/dev/null; then
93-
log_success "Validator ${i} is running"
94-
RUNNING=$((RUNNING + 1))
95-
else
96-
log_failure "Validator ${i} is not running"
97-
tail -50 "${PLATFORM_TEST_RUN_DIR}/validator-${i}/output.log" || true
98-
fi
99-
done
44+
log_info "Artifacts directory: ${ARTIFACT_DIR}"
45+
log_info "Using compose file: ${COMPOSE_FILE}"
46+
log_info "Compose project: ${COMPOSE_PROJECT}"
10047

101-
if [ "${RUNNING}" -ne "${NUM_VALIDATORS}" ]; then
102-
log_failure "Not all validators are running (${RUNNING}/${NUM_VALIDATORS})"
103-
exit 1
104-
fi
48+
log_info "Building docker images..."
49+
platform_compose -f "${COMPOSE_FILE}" build > "${LOG_DIR}/compose-build.log" 2>&1
10550

106-
log_info "Checking distributed storage initialization..."
107-
for i in $(seq 1 "${NUM_VALIDATORS}"); do
108-
DB_FILE="${PLATFORM_TEST_RUN_DIR}/validator-${i}/distributed.db"
109-
if [ -d "${DB_FILE}" ]; then
110-
log_success "Validator ${i}: distributed.db exists"
111-
else
112-
log_warning "Validator ${i}: distributed.db not found yet"
113-
fi
114-
done
51+
log_info "Starting compose stack..."
52+
platform_compose -f "${COMPOSE_FILE}" up -d > "${LOG_DIR}/compose-up.log" 2>&1
11553

116-
log_info "Checking P2P network initialization..."
117-
for i in $(seq 1 "${NUM_VALIDATORS}"); do
118-
LOG_FILE="${PLATFORM_TEST_RUN_DIR}/validator-${i}/output.log"
119-
if grep -q "P2P network initialized" "${LOG_FILE}" 2>/dev/null; then
120-
log_success "Validator ${i}: P2P network initialized"
121-
else
122-
log_warning "Validator ${i}: P2P initialization message not found"
123-
fi
54+
wait_for_health() {
55+
local container="$1"
56+
local timeout_seconds="$2"
57+
local start
58+
start=$(date +%s)
12459

125-
if grep -q "Peer connected\|Peer identified" "${LOG_FILE}" 2>/dev/null; then
126-
log_success "Validator ${i}: Has peer connections"
127-
fi
128-
done
60+
while true; do
61+
local status
62+
status=$(docker inspect --format '{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
63+
if [ "${status}" = "healthy" ]; then
64+
log_success "${container} is healthy"
65+
return 0
66+
fi
12967

130-
log_info "Letting the network stabilize for 10 seconds..."
131-
sleep 10
132-
133-
log_info ""
134-
log_info "==============================================="
135-
log_info " MULTI-VALIDATOR TEST RESULTS"
136-
log_info "==============================================="
137-
138-
HEALTHY=0
139-
for i in $(seq 1 "${NUM_VALIDATORS}"); do
140-
if kill -0 "${VALIDATOR_PIDS[$((i-1))]}" 2>/dev/null; then
141-
DB_FILE="${PLATFORM_TEST_RUN_DIR}/validator-${i}/distributed.db"
142-
if [ -d "${DB_FILE}" ]; then
143-
log_success "Validator ${i}: HEALTHY (running + DB initialized)"
144-
HEALTHY=$((HEALTHY + 1))
145-
else
146-
log_warning "Validator ${i}: RUNNING (no DB yet)"
68+
local now
69+
now=$(date +%s)
70+
if [ $((now - start)) -ge "${timeout_seconds}" ]; then
71+
log_failure "Timeout waiting for ${container} health (status=${status})"
72+
return 1
14773
fi
74+
75+
sleep 5
76+
done
77+
}
78+
79+
log_info "Waiting for services to become healthy..."
80+
wait_for_health "platform-mock-subtensor" 180
81+
wait_for_health "platform-validator-1" 180
82+
wait_for_health "platform-validator-2" 180
83+
wait_for_health "platform-validator-3" 180
84+
wait_for_health "platform-validator-4" 180
85+
86+
log_info "Verifying distributed storage initialization..."
87+
for i in 1 2 3 4; do
88+
if docker exec "platform-validator-${i}" test -f /data/distributed.db; then
89+
log_success "Validator ${i}: distributed.db created"
14890
else
149-
log_failure "Validator ${i}: DEAD"
91+
log_failure "Validator ${i}: distributed.db missing"
92+
exit 1
15093
fi
15194
done
15295

153-
log_info ""
154-
log_info "Summary: ${HEALTHY}/${NUM_VALIDATORS} validators healthy"
96+
log_info "Collecting compose logs for connectivity checks..."
97+
platform_compose -f "${COMPOSE_FILE}" logs --no-color > "${LOG_DIR}/compose.log" 2>&1
15598

156-
log_info ""
157-
log_info "Sample logs from Validator 1:"
158-
tail -30 "${PLATFORM_TEST_RUN_DIR}/validator-1/output.log" || true
99+
peer_connections=$(grep -c "Peer connected" "${LOG_DIR}/compose.log" || true)
100+
peer_identified=$(grep -c "Peer identified" "${LOG_DIR}/compose.log" || true)
101+
total_peers=$((peer_connections + peer_identified))
159102

160-
if [ "${HEALTHY}" -eq "${NUM_VALIDATORS}" ]; then
161-
log_success "All validators are healthy!"
162-
exit 0
103+
if [ "${total_peers}" -gt 0 ]; then
104+
log_success "Detected P2P peer activity (${total_peers} events)"
105+
else
106+
log_failure "No P2P peer activity detected"
107+
exit 1
108+
fi
109+
110+
log_info "Querying mock-subtensor health endpoint..."
111+
curl -fsS "http://localhost:9944/health" > "${ARTIFACT_DIR}/mock-subtensor-health.json"
112+
113+
log_info "Fetching mock-subtensor neurons for commit/reveal test..."
114+
hotkey_response=$(curl -fsS -X POST "http://localhost:9944/rpc" \
115+
-H "Content-Type: application/json" \
116+
-d '{"jsonrpc":"2.0","method":"subtensor_getNeurons","params":[100],"id":1}')
117+
118+
echo "${hotkey_response}" > "${ARTIFACT_DIR}/mock-subtensor-neurons.json"
119+
120+
hotkey=$(echo "${hotkey_response}" | grep -m1 -o '"hotkey":"[^"]*"' | cut -d '"' -f4)
121+
if [ -z "${hotkey}" ]; then
122+
log_failure "Failed to extract hotkey from mock-subtensor response"
123+
exit 1
124+
fi
125+
126+
log_info "Submitting mock weight commit..."
127+
commit_response=$(curl -fsS -X POST "http://localhost:9944/rpc" \
128+
-H "Content-Type: application/json" \
129+
-d "{\"jsonrpc\":\"2.0\",\"method\":\"subtensor_commitWeights\",\"params\":[100,[0,1,2],\"test_commit\",\"${hotkey}\"],\"id\":2}")
130+
echo "${commit_response}" > "${ARTIFACT_DIR}/mock-subtensor-commit.json"
131+
132+
log_info "Submitting mock weight reveal..."
133+
reveal_response=$(curl -fsS -X POST "http://localhost:9944/rpc" \
134+
-H "Content-Type: application/json" \
135+
-d "{\"jsonrpc\":\"2.0\",\"method\":\"subtensor_revealWeights\",\"params\":[100,[0,1,2],[65535,65535,65535],\"test_commit\",\"${hotkey}\"],\"id\":3}")
136+
echo "${reveal_response}" > "${ARTIFACT_DIR}/mock-subtensor-reveal.json"
137+
138+
log_info "Inspecting mock-subtensor weight commitments..."
139+
weights_response=$(curl -fsS "http://localhost:9944/test/weights")
140+
echo "${weights_response}" > "${ARTIFACT_DIR}/mock-subtensor-weights.json"
141+
142+
total_revealed=$(echo "${weights_response}" | grep -o '"total_revealed":[0-9]*' | head -1 | cut -d ':' -f2)
143+
if [ -z "${total_revealed}" ] || [ "${total_revealed}" -lt 1 ]; then
144+
log_failure "No revealed weight commits detected"
145+
exit 1
163146
fi
164147

165-
log_failure "Some validators are not healthy"
166-
exit 1
148+
log_success "Mock-subtensor commit/reveal flow verified"
149+
log_success "Multi-validator docker test completed successfully"

0 commit comments

Comments
 (0)