feat(go): share detection state through Redis (#43)

* feat(go): share detection state through Redis

* fix(helm): close validateSecret before the new shared-state define

The shared-state guard was added between validateSecret's inner {{ end }}
and the one closing the define, so the new define nested inside it and
the chart stopped parsing: 'unexpected <define> in command'. Neither the
guard nor any other template rendered.

Closes validateSecret first and drops the now-stray trailing end. Checked
with helm lint plus template renders at one replica, at three without
Redis (guard fires), and at three with Redis.

* ci(helm): give the multi-replica render combinations a Redis URL

autoscaling.enabled implies more than one pod, which the new shared-state
guard now refuses without Redis, so the scaling and full combinations
stopped rendering. Both get a Redis URL, which is what the chart is
telling operators to do.

Adds a guard test alongside the signing-key one: three replicas without
Redis, and autoscaling without Redis, must both refuse to render.

---------

Co-authored-by: Chris Portscheller <home@mac-mini.lan>

Author: cport1

.github/workflows/helm.yml

@@ -48,6 +48,21 @@ jobs:
           fi
           echo "guard holds"
 
+      # Multiple replicas without shared state give each pod its own challenge
+      # and replay tables, so a challenge issued by one is unknown to the next.
+      # Same treatment as the signing-key guard: prove it actually fires.
+      - name: Refuses to render multiple replicas without shared state
+        run: |
+          if helm template t charts/fcaptcha --set secret=ci --set replicaCount=3 >/dev/null 2>&1; then
+            echo "chart rendered 3 replicas with no Redis — the guard is broken"
+            exit 1
+          fi
+          if helm template t charts/fcaptcha --set secret=ci --set autoscaling.enabled=true >/dev/null 2>&1; then
+            echo "chart rendered with autoscaling and no Redis — the guard is broken"
+            exit 1
+          fi
+          echo "guard holds"
+
       - name: Install kubeconform
         run: |
           curl -sSL https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz \
@@ -64,11 +79,13 @@ jobs:
           render minimal        --set secret=ci
           render existing       --set existingSecret=mine
           render ingress        --set secret=ci --set ingress.enabled=true
-          render scaling        --set secret=ci --set autoscaling.enabled=true --set podDisruptionBudget.enabled=true
+          render scaling        --set secret=ci --set autoscaling.enabled=true --set podDisruptionBudget.enabled=true \
+                                --set redis.url=redis://r:6379
           render redis          --set secret=ci --set redis.url=redis://r:6379
           render full           --set secret=ci --set ingress.enabled=true --set autoscaling.enabled=true \
                                 --set podDisruptionBudget.enabled=true --set config.logVerdicts=true \
-                                --set config.allowedHostnames=a.example --set verifySecret=v
+                                --set config.allowedHostnames=a.example --set verifySecret=v \
+                                --set redis.url=redis://r:6379
 
   install:
     name: Install into a cluster

INSTALLATION.md

@@ -271,11 +271,10 @@ services:
 ```
 
 FCaptcha state is process-local by default. In the Go server, `REDIS_URL` now
-shares PoW challenges, token replay protection, and Siteverify idempotency
-results across instances. Challenge and token claims are atomic. Rate limits,
-suspicion, and fingerprints remain local, so continue to run one instance until
-the remaining stores and the Node/Python implementations gain shared-state
-support.
+shares PoW challenges, token replay protection, Siteverify idempotency, rate
+limits, suspicion, fingerprint cardinality, and site-key rotation guards.
+Challenge and token claims are atomic. Go may run multiple replicas when Redis
+is configured; Node and Python must remain single-instance.
 
 Run:
 
@@ -424,10 +423,9 @@ server {
 }
 ```
 
-**Important:** Do not run multiple instances yet. The Go server shares PoW,
-token replay, and Siteverify idempotency state through Redis, but rate limiting,
-suspicion, and fingerprints are still process-local. Node and Python remain
-entirely process-local.
+**Important:** Multiple Go instances require `REDIS_URL`; without it, all state
+is process-local. Node and Python remain entirely process-local and must run as
+one instance.
 
 ---
 
@@ -439,7 +437,7 @@ entirely process-local.
 |----------|----------|---------|-------------|
 | `FCAPTCHA_SECRET` | Yes | - | Secret key for signing tokens (min 16 chars) |
 | `FCAPTCHA_INSECURE_DEV_MODE` | No | off | Explicitly use the public development signing key for local-only development. Never expose a server with this enabled |
-| `REDIS_URL` | No | - | Go only: Redis URL for shared PoW, token replay, and Siteverify idempotency state. Configuration is fail-closed; it does not yet make every store distributed |
+| `REDIS_URL` | No | - | Go only: Redis URL for shared security state. Required for multiple replicas; configuration and runtime failures are fail-closed |
 | `FCAPTCHA_VERIFY_SECRET` | No | `FCAPTCHA_SECRET` | Credential your backend sends as `secret` when verifying a token. Split it from the signing key so a leaked verify credential cannot also mint tokens |
 | `FCAPTCHA_LEGACY_UNAUTH_VERIFY` | No | off | Restore the pre-1.22.0 behaviour where token verification accepted any caller. Migration cover for one release — see [Upgrading to 1.22.0](#upgrading-to-1220) |
 | `FCAPTCHA_ALLOWED_HOSTNAMES` | No | (any) | Comma-separated hostnames permitted to mint tokens, matched against the request `Origin` (then `Referer`) |

README.md

@@ -57,15 +57,14 @@ FCAPTCHA_SECRET=my-secret docker compose -f docker/docker-compose.yml up -d
 ```
 
 FCaptcha state is currently process-local by default. The Go server can use
-`REDIS_URL` to share PoW challenges, token replay protection, and Siteverify
-idempotency results across replicas; challenge and token consumption are atomic,
-it refuses to start if the configured Redis service is unavailable and returns
-503 rather than issuing an unpersisted challenge if Redis fails later.
-
-This is not yet permission to scale the entire service horizontally: rate
-limits, suspicion history, and fingerprint history remain per-process, and the
-Node and Python servers do not yet use Redis. Run one replica until those stores
-are also shared.
+`REDIS_URL` to share PoW challenges, token replay protection, Siteverify
+idempotency, rate limits, suspicion, fingerprint cardinality, and site-key
+rotation guards across replicas. Challenge and token consumption are atomic.
+It refuses to start if configured Redis is unavailable and fails closed if it
+becomes unavailable later.
+
+With `REDIS_URL`, the Go server can run multiple replicas. Node and Python do
+not yet use Redis and must remain single-instance.
 
 Kubernetes:
 

charts/fcaptcha/README.md

@@ -38,11 +38,11 @@ to the controller and rate limiting collapses onto one address — silently. The
 default covers the usual in-cluster pod CIDRs; narrow it to your controller's
 range.
 
-**Run one replica for now.** With the Go image, `redis.url` shares PoW challenges,
-token replay protection, and Siteverify idempotency results; challenge and token
-claims are atomic. Rate limits, suspicion, and fingerprints are still per-pod;
-Node and Python do not yet use Redis. `autoscaling` remains off until those
-remaining stores land.
+The default Go image may run multiple replicas when `redis.url` is configured.
+PoW, token replay, Siteverify idempotency, rate limits, suspicion, fingerprint
+cardinality, and site-key rotation guards are shared; one-time claims are
+atomic. Without Redis, run one replica. Node and Python images do not yet use
+Redis and must remain single-instance.
 
 The full list of deployment settings with security consequences is in
 [SECURITY.md](https://github.com/WebDecoy/FCaptcha/blob/main/SECURITY.md#deployment-notes-that-are-security-relevant).
@@ -62,7 +62,7 @@ The full list of deployment settings with security consequences is in
 | `config.logVerdictsIncludeRaw` | Also log free-text detection reasons. **Can contain visitor-derived data** | `false` |
 | `image.repository` / `image.tag` | Container image; tag defaults to the chart's `appVersion` | `ghcr.io/webdecoy/fcaptcha` |
 | `replicaCount` | See the note above before raising this | `1` |
-| `redis.url` / `redis.existingSecret` | Go only: shared PoW, token replay, and Siteverify idempotency state; remaining stores are still local | `""` |
+| `redis.url` / `redis.existingSecret` | Go only: shared security state; required for multiple replicas | `""` |
 | `service.type` / `service.port` | | `ClusterIP` / `80` |
 | `ingress.enabled` | | `false` |
 | `resources` | | 100m CPU / 128Mi requested |

charts/fcaptcha/templates/_helpers.tpl

@@ -61,3 +61,12 @@ to do by accident, and it costs one line of setup to satisfy.
 {{- fail "\n\nfcaptcha: a token signing key is required.\n\n  --set secret=$(openssl rand -hex 32)\n\nor point at one you already manage:\n\n  --set existingSecret=my-fcaptcha-secret\n\nWithout it the server falls back to a key published in its own source, and\nanyone can mint tokens your backend will accept.\n" -}}
 {{- end -}}
 {{- end -}}
+
+{{/* Multiple Go pods require shared security state. */}}
+{{- define "fcaptcha.validateSharedState" -}}
+{{- $multiple := or (gt (int .Values.replicaCount) 1) .Values.autoscaling.enabled -}}
+{{- $redisConfigured := or .Values.redis.url .Values.redis.existingSecret -}}
+{{- if and $multiple (not $redisConfigured) -}}
+{{- fail "\n\nfcaptcha: multiple replicas require Redis-backed shared state.\n\nSet redis.url or redis.existingSecret, or keep replicaCount=1 and autoscaling.enabled=false.\n" -}}
+{{- end -}}
+{{- end -}}

charts/fcaptcha/templates/deployment.yaml

@@ -1,4 +1,5 @@
 {{- include "fcaptcha.validateSecret" . -}}
+{{- include "fcaptcha.validateSharedState" . -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

charts/fcaptcha/values.yaml

@@ -68,9 +68,8 @@ extraEnv: []
 ## captcha chart, and the honest answer for anything you care about keeping is a
 ## managed instance or a purpose-built operator. Point this at one.
 ##
-## The Go server uses this for shared PoW, token replay, and Siteverify
-## idempotency state. Other stores remain in memory, and Node/Python do not use
-## it yet, so replicaCount still defaults to 1.
+## The Go server uses this for shared security state and may run multiple
+## replicas when it is configured. Node/Python do not use it yet.
 redis:
   url: ""
   existingSecret: ""
@@ -147,9 +146,9 @@ tolerations: []
 affinity: {}
 topologySpreadConstraints: []
 
-## Off by default because rate limiting, suspicion, and fingerprints remain
-## per-pod. Go shares PoW, token replay, and idempotency state, but that alone
-## does not make horizontal scaling safe.
+## Off by default because Redis is optional. Safe for the default Go image when
+## redis.url or redis.existingSecret is configured; Node/Python remain
+## single-instance.
 autoscaling:
   enabled: false
   minReplicas: 2

server-go/main.go

@@ -253,6 +253,9 @@ func main() {
 	// the first component of every rate-limit, fingerprint and challenge
 	// partition key. See sitekeys.go.
 	siteKeys := SiteKeyGuardFromEnv()
+	if engine.redisClient != nil {
+		siteKeys = SiteKeyGuardFromEnvWithRedis(engine.redisClient)
+	}
 	log.Printf("site keys: %s", siteKeys.Describe())
 
 	// Holds a JA4 fingerprint per live connection, populated during the TLS

server-go/redis_state_test.go

@@ -79,3 +79,54 @@ func TestRedisIdempotencyResultsCrossInstances(t *testing.T) {
 		t.Fatalf("another instance did not read the idempotency result: %#v", got)
 	}
 }
+
+func TestRedisDetectionStateCrossesInstances(t *testing.T) {
+	redisServer := miniredis.RunT(t)
+	redisURL := "redis://" + redisServer.Addr()
+	first := NewScoringEngineWithRedis("shared-test-secret", redisURL)
+	second := NewScoringEngineWithRedis("shared-test-secret", redisURL)
+
+	for i := 0; i < 3; i++ {
+		exceeded, _ := first.rateLimiter.Check("site|203.0.113.5", 60, 3)
+		if exceeded {
+			t.Fatalf("request %d was limited before the configured ceiling", i+1)
+		}
+	}
+	if exceeded, count := second.rateLimiter.Check("site|203.0.113.5", 60, 3); !exceeded || count != 3 {
+		t.Fatalf("another instance did not enforce the shared rate limit: exceeded=%v count=%d", exceeded, count)
+	}
+
+	first.suspicion.Record("site", "203.0.113.5", 0.95)
+	if count := second.suspicion.Count("site", "203.0.113.5"); count != 1 {
+		t.Fatalf("another instance saw %d shared suspicion hits, want 1", count)
+	}
+
+	first.fingerprintStore.Record("fp-a", "203.0.113.5", "site")
+	second.fingerprintStore.Record("fp-b", "203.0.113.5", "site")
+	if count := first.fingerprintStore.GetIPFingerprintCount("203.0.113.5"); count != 2 {
+		t.Fatalf("shared address fingerprint count=%d, want 2", count)
+	}
+	second.fingerprintStore.Record("fp-a", "198.51.100.8", "site")
+	if count := first.fingerprintStore.GetFingerprintIPCount("fp-a", "site"); count != 2 {
+		t.Fatalf("shared fingerprint address count=%d, want 2", count)
+	}
+}
+
+func TestRedisSiteKeyRotationLimitCrossesInstances(t *testing.T) {
+	redisServer := miniredis.RunT(t)
+	client := NewScoringEngineWithRedis("shared-test-secret", "redis://"+redisServer.Addr()).redisClient
+	first := NewSiteKeyGuard(nil, 2)
+	first.redis = client
+	second := NewSiteKeyGuard(nil, 2)
+	second.redis = client
+
+	if got := first.Normalize("site-a", "203.0.113.5"); got != "site-a" {
+		t.Fatalf("first key normalized to %q", got)
+	}
+	if got := second.Normalize("site-b", "203.0.113.5"); got != "site-b" {
+		t.Fatalf("second key normalized to %q", got)
+	}
+	if got := first.Normalize("site-c", "203.0.113.5"); got != overflowSiteKey {
+		t.Fatalf("cross-instance rotation was not folded into overflow: %q", got)
+	}
+}

server-go/scoring.go

@@ -307,13 +307,15 @@ const webBotAuthTimeout = 3 * time.Second
 type RateLimiter struct {
 	mu       sync.RWMutex
 	requests map[string][]int64
+	redis    *redis.Client
 }
 
 // FingerprintStore tracks fingerprint patterns
 type FingerprintStore struct {
 	mu             sync.RWMutex
 	fingerprints   map[string]*FingerprintData
 	ipFingerprints map[string]map[string]bool
+	redis          *redis.Client
 }
 
 type FingerprintData struct {
@@ -424,6 +426,9 @@ func NewScoringEngineWithRedis(secretKey, redisURL string) *ScoringEngine {
 	engine := NewScoringEngine(secretKey)
 	engine.powStore = newRedisPoWChallengeStore(client)
 	engine.tokenStore = newRedisTokenStore(client)
+	engine.rateLimiter = newRedisRateLimiter(client)
+	engine.fingerprintStore = newRedisFingerprintStore(client)
+	engine.suspicion = NewRedisSuspicionLedger(client)
 	engine.redisClient = client
 	return engine
 }
@@ -434,13 +439,21 @@ func newRateLimiter() *RateLimiter {
 	}
 }
 
+func newRedisRateLimiter(client *redis.Client) *RateLimiter {
+	return &RateLimiter{redis: client}
+}
+
 func newFingerprintStore() *FingerprintStore {
 	return &FingerprintStore{
 		fingerprints:   make(map[string]*FingerprintData),
 		ipFingerprints: make(map[string]map[string]bool),
 	}
 }
 
+func newRedisFingerprintStore(client *redis.Client) *FingerprintStore {
+	return &FingerprintStore{redis: client}
+}
+
 func compileUAPatterns() []*regexp.Regexp {
 	patterns := []string{
 		`(?i)headless`,
@@ -2425,7 +2438,43 @@ func (e *ScoringEngine) computeSignature(payload []byte) string {
 // Rate Limiter Methods
 // ============================================================
 
+var redisRateCheck = redis.NewScript(`
+redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[1])
+local count = redis.call('ZCARD', KEYS[1])
+local added = 0
+if count < tonumber(ARGV[3]) then
+  redis.call('ZADD', KEYS[1], ARGV[2], ARGV[4])
+  count = count + 1
+  added = 1
+end
+redis.call('EXPIRE', KEYS[1], ARGV[5])
+return {count, added}
+`)
+
+func redisOpaqueKey(kind, value string) string {
+	sum := sha256.Sum256([]byte(value))
+	return redisStatePrefix + kind + ":" + hex.EncodeToString(sum[:])
+}
+
+func randomHex(size int) string {
+	b := make([]byte, size)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Sprintf("%d", time.Now().UnixNano())
+	}
+	return hex.EncodeToString(b)
+}
+
 func (rl *RateLimiter) Check(key string, windowSeconds int64, maxRequests int) (bool, int) {
+	if rl.redis != nil {
+		now := time.Now().UnixMilli()
+		redisKey := redisOpaqueKey("rate", key)
+		member := fmt.Sprintf("%d:%s", now, randomHex(8))
+		values, err := redisRateCheck.Run(context.Background(), rl.redis, []string{redisKey}, now-windowSeconds*1000, now, maxRequests, member, windowSeconds+1).Int64Slice()
+		if err != nil {
+			return true, maxRequests
+		}
+		return values[1] == 0, int(values[0])
+	}
 	rl.mu.Lock()
 	defer rl.mu.Unlock()
 
@@ -2458,6 +2507,18 @@ func (rl *RateLimiter) Check(key string, windowSeconds int64, maxRequests int) (
 // ============================================================
 
 func (fs *FingerprintStore) Record(fingerprint, ip, siteKey string) {
+	if fs.redis != nil {
+		ctx := context.Background()
+		fpKey := redisOpaqueKey("fingerprint:ips", siteKey+"|"+fingerprint)
+		ipKey := redisOpaqueKey("fingerprint:fps", ip)
+		pipe := fs.redis.TxPipeline()
+		pipe.SAdd(ctx, fpKey, redisOpaqueKey("value:ip", ip))
+		pipe.Expire(ctx, fpKey, suspicionWindow)
+		pipe.SAdd(ctx, ipKey, redisOpaqueKey("value:fp", fingerprint))
+		pipe.Expire(ctx, ipKey, suspicionWindow)
+		_, _ = pipe.Exec(ctx)
+		return
+	}
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
 
@@ -2481,6 +2542,13 @@ func (fs *FingerprintStore) Record(fingerprint, ip, siteKey string) {
 }
 
 func (fs *FingerprintStore) GetIPFingerprintCount(ip string) int {
+	if fs.redis != nil {
+		count, err := fs.redis.SCard(context.Background(), redisOpaqueKey("fingerprint:fps", ip)).Result()
+		if err != nil {
+			return 100
+		}
+		return int(count)
+	}
 	fs.mu.RLock()
 	defer fs.mu.RUnlock()
 
@@ -2491,6 +2559,13 @@ func (fs *FingerprintStore) GetIPFingerprintCount(ip string) int {
 }
 
 func (fs *FingerprintStore) GetFingerprintIPCount(fingerprint, siteKey string) int {
+	if fs.redis != nil {
+		count, err := fs.redis.SCard(context.Background(), redisOpaqueKey("fingerprint:ips", siteKey+"|"+fingerprint)).Result()
+		if err != nil {
+			return 100
+		}
+		return int(count)
+	}
 	fs.mu.RLock()
 	defer fs.mu.RUnlock()