fix(notify): load config JWT secret and improve delivery diagnostics

docs/channels.md

@@ -410,23 +410,45 @@ The `services` table schema:
 
 ### Security
 
-- **Authentication** — the gateway's `POST /v1/api/notify` endpoint
-  requires authentication. Configure `TURNSTONE_JWT_SECRET` so the
-  server can mint JWTs with `aud: turnstone-channel` automatically.
-  If the secret is not set, the gateway fails closed and rejects all
-  requests with 401. Server JWTs (`aud: turnstone-server`) are
-  rejected.
+- **Authentication** — the gateway's `POST /v1/api/notify` endpoint requires authentication.
+  Configure the server's signing secret with `TURNSTONE_JWT_SECRET` or `[auth].jwt_secret` in
+  `config.toml`; the environment variable takes precedence. The gateway reads
+  `TURNSTONE_JWT_SECRET`, which must match the server's secret. The server automatically mints JWTs
+  with `aud: turnstone-channel`. If the gateway's secret is not set, it fails closed and rejects all
+  requests with 401. Server JWTs (`aud: turnstone-server`) are rejected.
 - **Rate limit** — maximum 5 notifications per turn. The counter only
   increments on successful delivery, so failures don't consume the
   budget.
-- **SSRF protection** — only `http://` and `https://` service URLs
-  are allowed. Other schemes are silently skipped.
+- **SSRF protection** — only `http://` and `https://` service URLs are allowed. Other schemes are
+  skipped with a warning.
 - **Mention sanitization** — `discord.utils.escape_mentions()` is
   applied before sending, preventing `@everyone` / `@here` abuse.
 - **Error redaction** — generic error messages are returned to the
   LLM. Internal details (service IDs, URLs, exception messages) are
   logged server-side only.
 
+### Troubleshooting notifications
+
+Check both the originating server and the channel gateway logs. Each `notify.gateway_failed` event
+identifies the gateway, its URL, the attempt, the workstream (`ws_id`), and the tool call (`call_id`).
+The event records whether an Authorization header was present, plus the HTTP status, request exception
+type, or delivery statuses such as `no_adapter`, `failed`, and `timeout`. Completion notifications log
+the same gateway and workstream details under `notify_completion.*`. Gateway URLs omit credentials,
+query parameters, and fragments; outbound diagnostics omit tokens, message content, and response
+bodies.
+
+A 401 from the gateway occurs before Discord or Slack delivery. `notify.auth_missing` on the server
+means it could not find notification credentials. On the gateway, `notify.auth_not_configured` means
+its signing secret is missing; `notify.auth_rejected` distinguishes a missing Authorization header,
+an invalid scheme or token format, and an invalid JWT. For `invalid_jwt`, check the shared secret, the
+`turnstone-channel` audience, token expiry, and host clocks. `notify.auth_insufficient_scope` indicates
+a valid JWT without the required `write` scope and returns 403.
+
+With debug logging enabled before credentials are first used, `notify.auth_configured` records the
+credential source without its value. A configured `TURNSTONE_CHANNEL_AUTH_TOKEN` overrides automatic
+JWT minting and must itself be a valid JWT for the channel audience with `write` scope. Working Discord
+or Slack conversations do not verify this outbound notification authentication path.
+
 ---
 
 ## Adding New Adapters

tests/test_notify_http.py

@@ -264,6 +264,32 @@ def test_whitespace_only_message(self, client):
 class TestNotifyAuth:
     """Tests for authentication on the /v1/api/notify endpoint."""
 
+    @pytest.mark.parametrize(
+        ("header", "reason"),
+        [
+            ("", "missing_authorization"),
+            ("Basic private-credentials", "invalid_auth_scheme"),
+            ("Bearer private-token", "invalid_token_format"),
+            ("Bearer private-token.invalid.jwt", "invalid_jwt"),
+        ],
+    )
+    def test_auth_rejection_logs_reason_without_credentials(self, authed_client, header, reason):
+        from unittest.mock import patch
+
+        with patch("turnstone.channels._http.log") as logger:
+            response = authed_client.post(
+                "/v1/api/notify",
+                json={"message": "private-message"},
+                headers={"Authorization": header} if header else {},
+            )
+
+        assert response.status_code == 401
+        assert response.json() == {"error": "Unauthorized"}
+        logger.warning.assert_called_once_with(
+            "notify.auth_rejected", reason=reason, client_host="testclient"
+        )
+        assert "private-" not in repr(logger.mock_calls)
+
     def test_reject_when_unconfigured(self, no_auth_client):
         """Requests are rejected (fail closed) when no auth is configured."""
         resp = no_auth_client.post(

tests/test_notify_tool.py

@@ -5,6 +5,8 @@
 from typing import TYPE_CHECKING
 from unittest.mock import MagicMock
 
+import pytest
+
 if TYPE_CHECKING:
     from turnstone.core.session import ChatSession
 
@@ -32,6 +34,216 @@ def _make_session() -> ChatSession:
     return session
 
 
+class TestNotifyAuthHeaders:
+    @pytest.fixture(autouse=True)
+    def auth_config(self, tmp_path, monkeypatch):
+        import turnstone.core.config as config
+        import turnstone.core.session as session
+
+        config_path = tmp_path / "config.toml"
+        config_path.touch(mode=0o600)
+        monkeypatch.setattr(config, "_config_path", config_path)
+        monkeypatch.setattr(config, "_cache", None)
+        monkeypatch.setattr(session, "_notify_token_manager", None)
+        monkeypatch.delenv("TURNSTONE_JWT_SECRET", raising=False)
+        monkeypatch.delenv("TURNSTONE_CHANNEL_AUTH_TOKEN", raising=False)
+        return config_path
+
+    def test_config_only_secret_authenticates_with_gateway(self, auth_config):
+        """The documented bare-metal config must authenticate outbound notifications."""
+        from unittest.mock import AsyncMock
+
+        from starlette.testclient import TestClient
+
+        from turnstone.channels._http import create_channel_app
+        from turnstone.core.session import _notify_auth_headers
+
+        secret = "a" * 32
+        auth_config.write_text(f'[auth]\njwt_secret = "  {secret}  "\n')
+        adapter = AsyncMock()
+        adapter.send.return_value = "message-1"
+        app = create_channel_app({"discord": adapter}, MagicMock(), jwt_secret=secret)
+        with TestClient(app) as client:
+            response = client.post(
+                "/v1/api/notify",
+                json={
+                    "target": {"channel_type": "discord", "channel_id": "123"},
+                    "message": "Hello!",
+                },
+                headers=_notify_auth_headers(),
+            )
+
+        assert response.status_code == 200
+        assert response.json()["results"][0]["status"] == "sent"
+        adapter.send.assert_awaited_once_with("123", "Hello!")
+
+    def test_environment_secret_takes_precedence(self, auth_config, monkeypatch):
+        from turnstone.core.auth import JWT_AUD_CHANNEL, validate_jwt
+        from turnstone.core.session import _notify_auth_headers
+
+        config_secret = "a" * 32
+        auth_config.write_text(f'[auth]\njwt_secret = "{config_secret}"\n')
+        secret = "b" * 32
+        monkeypatch.setenv("TURNSTONE_JWT_SECRET", f"  {secret}  ")
+
+        token = _notify_auth_headers()["Authorization"].removeprefix("Bearer ")
+        auth = validate_jwt(token, secret, audience=JWT_AUD_CHANNEL)
+        assert auth is not None
+        assert "write" in auth.scopes
+
+    def test_static_token_takes_precedence(self, auth_config, monkeypatch):
+        from turnstone.core.session import _notify_auth_headers
+
+        config_secret = "a" * 32
+        auth_config.write_text(f'[auth]\njwt_secret = "{config_secret}"\n')
+        monkeypatch.setenv("TURNSTONE_JWT_SECRET", "b" * 32)
+        monkeypatch.setenv("TURNSTONE_CHANNEL_AUTH_TOKEN", "  static-token  ")
+        assert _notify_auth_headers() == {"Authorization": "Bearer static-token"}
+
+    def test_missing_secret_returns_no_headers(self):
+        from turnstone.core.session import _notify_auth_headers
+
+        assert _notify_auth_headers() == {}
+
+
+class TestNotifyDiagnostics:
+    @pytest.fixture(params=["tool", "completion"])
+    def notify_caller(self, request, monkeypatch):
+        """Exercise both outbound paths with the same gateway failures."""
+        import turnstone.core.session as session_module
+        import turnstone.server as server_module
+
+        session = _make_session()
+        session._ws_id = "abcdef1234567890"
+        monkeypatch.setattr(session, "_backoff_or_cancelled", MagicMock())
+        monkeypatch.setattr(server_module.time, "sleep", MagicMock())
+        monkeypatch.setattr(
+            session_module,
+            "_notify_auth_headers",
+            lambda: {"Authorization": "Bearer private-auth-token"},
+        )
+        storage = MagicMock()
+        monkeypatch.setattr(session_module, "get_storage", lambda: storage)
+        caller_module = session_module if request.param == "tool" else server_module
+        logger = MagicMock()
+        monkeypatch.setattr(caller_module, "log", logger)
+        post = MagicMock()
+        monkeypatch.setattr(session_module.httpx, "post", post)
+
+        def deliver():
+            if request.param == "tool":
+                _, result = session._exec_notify(
+                    {
+                        "call_id": "call-1",
+                        "channel_type": "discord",
+                        "channel_id": "123",
+                        "message": "private-message-content",
+                    }
+                )
+                assert result == "Error: notification delivery failed"
+                assert session._notify_count == 0
+            else:
+                server_module._deliver_notification(
+                    storage,
+                    {"ws_id": session._ws_id, "message": "private-message-content"},
+                    {"Authorization": "Bearer private-auth-token"},
+                )
+
+        return storage, post, logger, deliver, request.param
+
+    def test_preserves_each_gateway_failure_without_secrets(self, notify_caller):
+        storage, post, logger, deliver, caller = notify_caller
+        storage.list_services.return_value = [
+            {"service_id": "gateway-1", "url": "http://user:private-password@gw.example.com:8091"},
+            {"service_id": "gateway-2", "url": "http://gw2.example.com:8091"},
+        ]
+        post.side_effect = [
+            MagicMock(status_code=401),
+            ConnectionError("private-exception-content"),
+        ] * 3
+
+        deliver()
+
+        failures = [c.kwargs for c in logger.warning.call_args_list if "gateway_id" in c.kwargs]
+        assert len(failures) == 6
+        for attempt in range(1, 4):
+            rejected, unreachable = failures[(attempt - 1) * 2 : attempt * 2]
+            assert rejected["gateway_id"] == "gateway-1"
+            assert rejected["gateway_url"] == "http://gw.example.com:8091/v1/api/notify"
+            assert rejected.get("status_code", rejected.get("status")) == 401
+            assert unreachable["gateway_id"] == "gateway-2"
+            assert unreachable["error_type"] == "ConnectionError"
+            for failure in (rejected, unreachable):
+                assert failure["attempt"] == attempt
+                assert failure["ws_id"] == "abcdef1234567890"
+                assert failure["auth_present"] is True
+                if caller == "tool":
+                    assert failure["call_id"] == "call-1"
+        assert "private-" not in repr(logger.mock_calls)
+
+    @pytest.mark.parametrize(
+        "response_kind", ["failed_deliveries", "invalid_json", "invalid_results"]
+    )
+    def test_unsuccessful_response_logs_safe_details(self, notify_caller, response_kind):
+        storage, post, logger, deliver, _ = notify_caller
+        storage.list_services.return_value = [
+            {"service_id": "gateway-1", "url": "http://gw.example.com:8091"},
+        ]
+        response = MagicMock(status_code=200)
+        if response_kind == "invalid_json":
+            response.json.side_effect = ValueError("private-response-content")
+        elif response_kind == "invalid_results":
+            response.json.return_value = {"results": "private-response-content"}
+        else:
+            response.json.return_value = {
+                "results": [
+                    {"status": "no_adapter", "channel_id": "private-target"},
+                    {"status": "private-response-content"},
+                    {"status": ["private-response-content"]},
+                ]
+            }
+        post.return_value = response
+
+        deliver()
+
+        failures = [c for c in logger.warning.call_args_list if "gateway_id" in c.kwargs]
+        assert len(failures) == 3
+        for failure in failures:
+            if response_kind == "invalid_json":
+                assert (
+                    failure.kwargs.get("reason") == "invalid_response"
+                    or failure.args[0] == "notify_completion.response_parse_error"
+                )
+            else:
+                expected = (
+                    ["invalid_response"]
+                    if response_kind == "invalid_results"
+                    else ["no_adapter", "unknown"]
+                )
+                assert failure.kwargs["delivery_statuses"] == expected
+        assert "private-" not in repr(logger.mock_calls)
+
+    @pytest.mark.parametrize(
+        ("url", "expected"),
+        [
+            (
+                "https://user:password@gw.example.com/prefix?token=secret#fragment",
+                "https://gw.example.com/prefix",
+            ),
+            (
+                "http://user:password@[2001:db8::1]:8091/v1/api/notify",
+                "http://[2001:db8::1]:8091/v1/api/notify",
+            ),
+            ("http://[invalid", "<invalid URL>"),
+            ("file:///private-credential", "<invalid URL>"),
+        ],
+    )
+    def test_gateway_url_redaction(self, url, expected):
+        from turnstone.core.session import _notify_log_url
+
+        assert _notify_log_url(url) == expected
+
+
 class TestPrepareNotify:
     def test_valid_username_target(self):
         session = _make_session()

turnstone/channels/_http.py

@@ -42,13 +42,19 @@ async def _handle_health(request: Request) -> JSONResponse:
 def _check_auth(request: Request) -> JSONResponse | None:
     """Validate the request's Authorization header.  Returns an error response or None."""
     jwt_secret: str = getattr(request.app.state, "jwt_secret", "")
+    client_host = request.client.host if request.client else ""
 
     if not jwt_secret:
-        log.warning("notify.auth_not_configured")
+        log.warning("notify.auth_not_configured", client_host=client_host)
         return JSONResponse({"error": "authentication not configured"}, status_code=401)
 
     header = request.headers.get("Authorization", "")
     if not header.startswith("Bearer "):
+        log.warning(
+            "notify.auth_rejected",
+            reason="missing_authorization" if not header else "invalid_auth_scheme",
+            client_host=client_host,
+        )
         return JSONResponse({"error": "Unauthorized"}, status_code=401)
 
     token = header[7:]
@@ -67,10 +73,16 @@ def _check_auth(request: Request) -> JSONResponse | None:
                     "notify.auth_insufficient_scope",
                     user_id=result.user_id,
                     scopes=sorted(result.scopes),
+                    client_host=client_host,
                 )
                 return JSONResponse({"error": "insufficient scope"}, status_code=403)
             return None
 
+    log.warning(
+        "notify.auth_rejected",
+        reason="invalid_jwt" if "." in token else "invalid_token_format",
+        client_host=client_host,
+    )
     return JSONResponse({"error": "Unauthorized"}, status_code=401)
 
 
@@ -105,7 +117,7 @@ async def _handle_notify(request: Request) -> JSONResponse:
     if "username" in target:
         user = await asyncio.to_thread(storage.get_user_by_username, target["username"])
         if user is None:
-            log.warning("notify.user_not_found", username=target["username"])
+            log.warning("notify.user_not_found", username=target["username"], ws_id=ws_id)
             return JSONResponse(
                 {"error": "target not found or has no linked channels"},
                 status_code=404,
@@ -114,7 +126,7 @@ async def _handle_notify(request: Request) -> JSONResponse:
         for link in links:
             targets.append((link["channel_type"], link["channel_user_id"]))
         if not targets:
-            log.warning("notify.user_no_linked_channels", username=target["username"])
+            log.warning("notify.user_no_linked_channels", username=target["username"], ws_id=ws_id)
             return JSONResponse(
                 {"error": "target not found or has no linked channels"},
                 status_code=404,
@@ -159,6 +171,7 @@ async def _handle_notify(request: Request) -> JSONResponse:
                 "notify.no_adapter",
                 channel_type=channel_type,
                 channel_id=channel_id,
+                ws_id=ws_id,
             )
             continue
         try:
@@ -181,12 +194,14 @@ async def _handle_notify(request: Request) -> JSONResponse:
                 channel_type=channel_type,
                 channel_id=channel_id,
                 message_id=msg_id,
+                ws_id=ws_id,
             )
         except TimeoutError:
             log.warning(
                 "notify.timeout",
                 channel_type=channel_type,
                 channel_id=channel_id,
+                ws_id=ws_id,
             )
             results.append(
                 {
@@ -200,6 +215,7 @@ async def _handle_notify(request: Request) -> JSONResponse:
                 "notify.delivery_failed",
                 channel_type=channel_type,
                 channel_id=channel_id,
+                ws_id=ws_id,
             )
             results.append(
                 {