diff --git a/.env.example b/.env.example index bdf135ab..f6b4936b 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,9 @@ # -- Agent behavior ----------------------------------------------------------- # SKIP_PERMISSIONS=true # auto-approve all tool calls (dev only) # MCP_CONFIG=/workspace/mcp.json # MCP server config file +# Exact, operator-controlled private hosts allowed during MCP OAuth discovery. +# Comma-separated; set on every console/node process in non-Compose deployments. +# TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS=gitlab.internal.example,auth.internal.example # -- Channel gateway (Discord / Slack) ---------------------------------------- # TURNSTONE_DISCORD_TOKEN= diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3f244d..50018de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,12 @@ the 1.7 line are not repeated here. ### Added +- **Trusted private hosts for MCP OAuth discovery.** Operators can allow exact, + controlled private-network hostnames or IP addresses through + `TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` and the MCP Servers admin page. + Environment entries remain read-only and merge with database-managed entries; + the exception preserves HTTPS, same-origin, port, userinfo, and dangerous-address + protections. - **Immutable memory-index snapshots (#902).** At the first real model call, a workstream captures the complete memory metadata visible to its acting user and project. That index stays cache-stable for the life of the workstream; diff --git a/compose.yaml b/compose.yaml index 89c6c8f8..540a8f14 100644 --- a/compose.yaml +++ b/compose.yaml @@ -167,6 +167,9 @@ services: TURNSTONE_DB_BACKEND: *db-backend TURNSTONE_DB_URL: *db-url TURNSTONE_CONSOLE_URL: http://console:8090 + # Exact private-network hosts trusted during MCP OAuth discovery. The + # console exposes these as read-only environment entries in the admin UI. + TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS: "${TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS:-}" # Separate from TURNSTONE_CONSOLE_URL: this is the canonical responder # base embedded in ACME directory/order URLs for cross-host enrollment. TURNSTONE_ACME_EXTERNAL_URL: "${TURNSTONE_ACME_EXTERNAL_URL:-}" @@ -308,6 +311,9 @@ services: TURNSTONE_SEARXNG_URL: ${TURNSTONE_SEARXNG_URL:-http://searxng:8080} MODEL: ${MODEL:-} MCP_CONFIG: ${MCP_CONFIG:-} + # Must match the console value so OAuth discovery behaves consistently + # on every node that handles token refresh or connection work. + TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS: "${TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS:-}" SKIP_PERMISSIONS: ${SKIP_PERMISSIONS:-} TURNSTONE_NODE_ID: node-1 TURNSTONE_ADVERTISE_URL: http://node-1:8080 diff --git a/docs/api-reference.md b/docs/api-reference.md index 696c1eab..bde1103d 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -2505,11 +2505,60 @@ the `admin.settings` permission. | DELETE | `/v1/api/admin/mcp-servers/{server_id}` | Delete an MCP server definition. | | POST | `/v1/api/admin/mcp-servers/reload` | Tell all cluster nodes to re-read the `mcp_servers` DB table and reconcile (add new, remove stale, reconnect changed). | | POST | `/v1/api/admin/mcp-servers/import` | Import servers from a pasted JSON config. Body: `{config: {mcpServers: {...}}}`. Skips existing names. | +| GET | `/v1/api/admin/mcp-servers/trusted-private-hosts` | Return the merged MCP OAuth private-host allow-list with source and read-only metadata. | +| PUT | `/v1/api/admin/mcp-servers/trusted-private-hosts` | Replace only the user-managed portion of the MCP OAuth private-host allow-list. Body: `{hosts: ["gitlab.internal.example"]}`. | Permission: `admin.mcp` Secrets (`env`, `headers` fields) are masked with `***` by default. Use `?reveal=true` on GET endpoints to see actual values. +#### Trusted private OAuth hosts + +`GET /v1/api/admin/mcp-servers/trusted-private-hosts` returns both deployment +entries from `TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` and entries managed by +users through the API/Web UI: + +```json +{ + "hosts": [ + { + "host": "gitlab.internal.example", + "source": "environment", + "readonly": true + }, + { + "host": "auth.internal.example", + "source": "manual", + "readonly": false + } + ], + "environment_variable": "TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS", + "help": "Add only exact hostnames or IP addresses ..." +} +``` + +`PUT /v1/api/admin/mcp-servers/trusted-private-hosts` replaces the complete +**manual** list: + +```json +{ + "hosts": ["auth.internal.example", "192.168.5.120"] +} +``` + +The response has the same merged shape as GET. Environment entries are never +changed by PUT and take precedence over duplicate manual entries. Each value +must be an exact hostname or IP address; schemes, ports, paths, credentials, +and wildcards return `400`. The merged environment and manual list may contain +at most 100 unique hosts. A malformed deployment environment value makes GET +fail closed with `500`; correct the environment variable and restart the +affected process. Both methods require `admin.mcp`; missing configuration +storage returns `503`. + +Trusting a host relaxes only private-address rejection during MCP OAuth +discovery. See [MCP OAuth: Private-network OAuth hosts](mcp-oauth.md#private-network-oauth-hosts) +for the remaining SSRF protections. + --- ### MCP Registry diff --git a/docs/console.md b/docs/console.md index 50626d11..b0be844e 100644 --- a/docs/console.md +++ b/docs/console.md @@ -477,7 +477,14 @@ in-flight requests keep their original definition snapshot; see [Settings](settings.md#model-definition-reloads) for the full contract. The **Nodes** tab edits per-node metadata, and the **TLS** tab manages CA and leaf certificates for the internal mTLS fabric. The **Settings** tab edits ConfigStore values -live; edits apply without restart. +live; edits apply without restart. Under its **MCP** section, the +`oauth_trusted_private_hosts` row manages exact-host exceptions for +operator-controlled MCP/OAuth services on private networks. Entries supplied +through `TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` are labeled **environment** +and cannot be changed in the UI; manually added entries are stored in the +database and can be removed. See +[MCP OAuth: Private-network OAuth hosts](mcp-oauth.md#private-network-oauth-hosts) +for the security boundary and deployment syntax. **Users tab:** diff --git a/docs/docker.md b/docs/docker.md index 74ed62cc..5a74b867 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -281,6 +281,7 @@ interface, or anyone who can reach it can search through your instance. | `TURNSTONE_WORKSPACE` | `/workspace` (image env) | Directory named as the user's workspace in the model's tool descriptions; informational only — see [Working directory](#working-directory) | | `SKIP_PERMISSIONS` | — | Set to any value to auto-approve all tool calls (dev only) | | `MCP_CONFIG` | — | Path to an MCP server config file | +| `TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` | — | Comma- or newline-separated exact hostnames/IPs allowed to resolve to private addresses during MCP OAuth discovery. Set on every server and console container. Environment entries are read-only under **Settings → MCP** and merge with database-managed entries. See [MCP OAuth](mcp-oauth.md#private-network-oauth-hosts). | | `TURNSTONE_IMAGE_TAG` | `latest` | ghcr.io image tag — production stack | ## Building diff --git a/docs/mcp-oauth.md b/docs/mcp-oauth.md index 821166eb..3fe3ae36 100644 --- a/docs/mcp-oauth.md +++ b/docs/mcp-oauth.md @@ -52,6 +52,55 @@ Switching `auth_type` away from `oauth_user` / `oauth_obo` **deletes** that serv | Scopes | No | Space-separated default scope set requested at the authorize endpoint. Per-tool step-up may union additional scopes from a server's `insufficient_scope` response. | | Audience | No | RFC 8707 `resource=` parameter sent on every authorize and token request. Defaults to the MCP server URL when unset. Validate against the `aud` claim in returned JWT tokens. | +### Private-network OAuth hosts + +OAuth discovery rejects endpoints that resolve to private addresses by default. This +prevents an untrusted MCP server or discovery document from turning Turnstone into an +SSRF proxy to internal services. If an MCP resource server or authorization server is +deliberately hosted on your private network, add its **exact hostname or IP address** to +the trusted private-host list. + +For automated deployments, set the following environment variable on every +`turnstone-server` and `turnstone-console` process: + +```bash +TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS=gitlab.internal.example,auth.internal.example +``` + +The value is a comma- or newline-separated list. Entries must contain only an exact +hostname or IP address. URLs, ports, paths, credentials, and wildcards are rejected: + +```text +gitlab.internal.example # accepted +192.168.5.120 # accepted +https://gitlab.internal # rejected: URL +gitlab.internal:443 # rejected: port +*.internal.example # rejected: wildcard +``` + +Administrators with `admin.settings` and `admin.mcp` permissions can add +additional entries under **Admin → Settings → MCP** using the +`oauth_trusted_private_hosts` row. User-managed entries are stored in the database +and propagated to cluster nodes. Environment entries are merged first, shown with +an **environment** source label, and are read-only in the Web UI. If the same host +appears in both sources, the environment entry wins. The combined list is limited +to 100 hosts. + +This is a narrow private-address exception, not a general SSRF bypass: + +- matching is case-insensitive and exact; there is no suffix or wildcard matching; +- HTTPS is still required, apart from Turnstone's existing genuine-loopback development + exception; +- discovered endpoints must still satisfy issuer same-origin/trusted-host and port + checks; +- embedded credentials remain forbidden; and +- link-local, multicast, unspecified, reserved, and metadata-service addresses remain + refused even when their hostname is listed. + +Only list hosts whose DNS and services are controlled by the deployment operator. +Removing a user-managed entry takes effect without a restart. Environment entries must +be changed in the process configuration and the affected processes restarted. + ### Encryption key ```toml @@ -168,6 +217,7 @@ Every transition that changes what a stored row *means* deletes the rows outrigh | `mcp_consent_required` even after consenting | Token persistence failed, or refresh-token rejected by AS | Check audit log for `mcp_server.oauth.persist_failed` or `mcp_server.oauth.token_revoked`. Re-consent via settings modal. | | `mcp_token_undecryptable_key_unknown` | Encryption key rotated without keeping the previous key in the keyring | Add the previous key back to `mcp_token_encryption_keys` until all rows have been re-encrypted, then drop. | | `mcp_oauth_url_insecure` | MCP server URL is `http://` (not `https://`) on a non-loopback host | Use `https://`. Per-user bearers must not transit cleartext. | +| `PRM URL rejected: endpoint URL resolves to non-public address` | The MCP server or OAuth discovery endpoint deliberately resolves to a private address but is not trusted | Add its exact host under **Settings → MCP → oauth_trusted_private_hosts**, or set `TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` on every server and console process. Do not add a wildcard or full URL. | | Tools fail in scheduled / Discord / Slack runs | OAuth-MCP requires browser-based consent | Users must pre-consent via the web UI. Phase 9 dashboard badge surfaces deferred consents from these runs on next login. | | Circuit breaker open repeatedly | Transport-level errors on the MCP server (DNS, TLS, 5xx) | Check the per-server error pill; auth errors do not trip the breaker. | | **`oauth_obo`**: every tool call fails, log shows `obo_misconfigured` | Server row has no Audience, or `obo_grant_profile` is unset/unknown | Set the Audience on the server row; set `[oidc] obo_grant_profile` to `entra` or `rfc8693`. | diff --git a/docs/security.md b/docs/security.md index 8f1ef4c4..f2c63e07 100644 --- a/docs/security.md +++ b/docs/security.md @@ -297,6 +297,27 @@ API tokens are unaffected by this setting. --- +## MCP OAuth SSRF boundary + +MCP protected-resource and authorization-server metadata is untrusted network +input. Turnstone therefore resolves and classifies every discovery URL before +fetching it, requires HTTPS outside the genuine-loopback development case, +checks discovered endpoint origin and port, rejects embedded credentials, and +refuses link-local/metadata, multicast, unspecified, and reserved addresses. + +Private addresses are denied by default. Deployments with an intentionally +private MCP or authorization server can allow an **exact**, operator-controlled +hostname or IP through the MCP Servers admin page or +`TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS`. This opt-in relaxes only the +private-address classification for the listed host; it does not disable the +other checks above. Wildcards, URLs, and ports are not accepted. Environment +entries are read-only in the UI and merge with database-managed entries. + +See [MCP OAuth: Private-network OAuth hosts](mcp-oauth.md#private-network-oauth-hosts) +for configuration, precedence, and examples. + +--- + ## Token Detection Order The auth middleware inspects the `Authorization: Bearer ` header diff --git a/docs/settings.md b/docs/settings.md index 8b610823..eac536c3 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -271,7 +271,7 @@ initialization: | `tools` | timeout, approval_timeout_seconds, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results | | `server` | workstream_idle_timeout, max_workstreams | | `cluster` | node_fan_out_limit, mcp_max_servers | -| `mcp` | config_path, registry_url | +| `mcp` | config_path, registry_url, oauth_trusted_private_hosts | | `ratelimit` | enabled, requests_per_second, burst, trusted_proxies | | `health` | backend_probe_interval, backend_probe_timeout, circuit_breaker_threshold, circuit_breaker_cooldown | | `judge` | enabled, model, smart_approvals, confidence_threshold, max_context_ratio, timeout, parallel_evaluations, read_only_tools, output_guard, output_guard_budget_seconds, output_guard_llm, output_guard_model, output_guard_llm_timeout, redact_secrets, cancel_on_approval | @@ -283,6 +283,16 @@ Settings are addressed by dotted key (e.g. `memory.relevance_k`). Each has a declared type (`int`, `float`, `str`, `bool`), optional `min_value`/`max_value` range, optional `choices` list, and an `is_secret` flag. +`mcp.oauth_trusted_private_hosts` stores only the user-managed portion of the +MCP OAuth private-host allow-list. Open **Admin → Settings**, expand **MCP**, and +use the `oauth_trusted_private_hosts` row. Its specialized editor displays the +merged source of each entry. Administrators without `admin.mcp` permission see +the standard Settings text editor for the database-managed value. Unlike ordinary +ConfigStore environment seeding, +`TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS` remains a live, read-only deployment +source and is never copied into the database. See +[MCP OAuth: Private-network OAuth hosts](mcp-oauth.md#private-network-oauth-hosts). + --- ## Storage diff --git a/tests/test_app_js.py b/tests/test_app_js.py index 9b725c83..dc696124 100644 --- a/tests/test_app_js.py +++ b/tests/test_app_js.py @@ -29,6 +29,46 @@ ) _CONSOLE_APP_JS = Path(__file__).resolve().parent.parent / "turnstone/console/static/app.js" _CONSOLE_INDEX = Path(__file__).resolve().parent.parent / "turnstone/console/static/index.html" +_CONSOLE_ADMIN_JS = Path(__file__).resolve().parent.parent / "turnstone/console/static/admin.js" + + +def test_mcp_private_hosts_ui_lives_in_mcp_settings_and_uses_safe_dom_rendering() -> None: + html = _CONSOLE_INDEX.read_text(encoding="utf-8") + script = _CONSOLE_ADMIN_JS.read_text(encoding="utf-8") + + assert 'id="mcp-private-host-form"' not in html + assert 'id="mcp-private-host-list"' not in html + + settings_renderer_start = script.index("function _renderMcpTrustedPrivateHostsSetting(") + settings_renderer_end = script.index("function _renderSettingRow(", settings_renderer_start) + settings_renderer = script[settings_renderer_start:settings_renderer_end] + assert 'class="settings-row settings-row-mcp-private-hosts"' in settings_renderer + assert 'id="mcp-private-host-form"' in settings_renderer + assert 'id="mcp-private-host-list"' in settings_renderer + assert "TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS" in settings_renderer + assert "escapeHtml(item.help)" in settings_renderer + + mcp_loader_start = script.index("function loadAdminMcp()") + mcp_loader_end = script.index("function loadMcpTrustedPrivateHosts()", mcp_loader_start) + assert "loadMcpTrustedPrivateHosts()" not in script[mcp_loader_start:mcp_loader_end] + + settings_loader_start = script.index("function loadSettings()") + settings_loader_end = script.index("function _renderSettings(", settings_loader_start) + settings_loader = script[settings_loader_start:settings_loader_end] + assert '_consoleHasPermission("admin.mcp")' in settings_loader + assert "loadMcpTrustedPrivateHosts()" in settings_loader + assert 'items[j].key === "mcp.oauth_trusted_private_hosts" &&' in script, ( + "settings-only admins must retain the standard Settings editor" + ) + assert "/v1/api/admin/mcp-servers/trusted-private-hosts" in script + assert 'form.dataset.wired === "true"' in script + assert 'entry.source === "environment"' in script + assert "if (!entry.readonly)" in script + renderer_start = script.index("function _renderMcpTrustedPrivateHosts()") + renderer_end = script.index("function _setMcpPrivateHostStatus", renderer_start) + renderer = script[renderer_start:renderer_end] + assert ".textContent = entry.host" in renderer + assert "innerHTML" not in renderer def _pane_method_offset(body: str, name: str) -> int: @@ -752,7 +792,6 @@ def test_phase8_appendtooloutput_dispatches_mcp_error_before_renderer() -> None: _COORD_JS = ( Path(__file__).resolve().parent.parent / "turnstone/console/static/coordinator/coordinator.js" ) -_CONSOLE_ADMIN_JS = Path(__file__).resolve().parent.parent / "turnstone/console/static/admin.js" _CONSOLE_GOVERNANCE_JS = ( Path(__file__).resolve().parent.parent / "turnstone/console/static/governance.js" ) diff --git a/tests/test_mcp_admin_api.py b/tests/test_mcp_admin_api.py index e369dfe7..c81c1078 100644 --- a/tests/test_mcp_admin_api.py +++ b/tests/test_mcp_admin_api.py @@ -33,14 +33,17 @@ admin_create_mcp_server, admin_delete_mcp_server, admin_get_mcp_server, + admin_get_mcp_trusted_private_hosts, admin_import_mcp_config, admin_list_mcp_servers, admin_mcp_reconnect_one, admin_mcp_refresh_one, admin_mcp_reload, admin_update_mcp_server, + admin_update_mcp_trusted_private_hosts, ) from turnstone.core.auth import AuthResult +from turnstone.core.config_store import ConfigStore from turnstone.core.storage._sqlite import SQLiteBackend # --------------------------------------------------------------------------- @@ -126,6 +129,15 @@ async def dispatch(self, request: Request, call_next: Any) -> Response: admin_mcp_reload, methods=["POST"], ), + Route( + "/api/admin/mcp-servers/trusted-private-hosts", + admin_get_mcp_trusted_private_hosts, + ), + Route( + "/api/admin/mcp-servers/trusted-private-hosts", + admin_update_mcp_trusted_private_hosts, + methods=["PUT"], + ), Route( "/api/admin/mcp-servers/{name}/refresh", admin_mcp_refresh_one, @@ -240,6 +252,7 @@ def client(storage): middleware=[Middleware(_InjectAuthMiddleware)], ) app.state.auth_storage = storage + app.state.config_store = ConfigStore(storage) _install_token_store(app, storage) # Default: OIDC enabled under the entra profile so oauth_obo writes pass the # requirement gate. Per-test overrides install rfc8693 / disabled / bad @@ -256,6 +269,7 @@ def client_no_perm(storage): middleware=[Middleware(_InjectAuthNoMcpMiddleware)], ) app.state.auth_storage = storage + app.state.config_store = ConfigStore(storage) _install_token_store(app, storage) return TestClient(app) @@ -335,6 +349,69 @@ def test_list_returns_created_servers(self, client): assert "server-b" in names +class TestTrustedPrivateHosts: + endpoint = "/v1/api/admin/mcp-servers/trusted-private-hosts" + + def test_get_merges_environment_and_manual_sources( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv( + "TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS", + "gitlab.internal.example", + ) + saved = client.put(self.endpoint, json={"hosts": ["manual.internal.example"]}) + assert saved.status_code == 200 + + response = client.get(self.endpoint) + assert response.status_code == 200 + assert response.json()["hosts"] == [ + { + "host": "gitlab.internal.example", + "source": "environment", + "readonly": True, + }, + { + "host": "manual.internal.example", + "source": "manual", + "readonly": False, + }, + ] + + def test_put_cannot_replace_or_duplicate_environment_hosts( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv( + "TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS", + "gitlab.internal.example", + ) + response = client.put( + self.endpoint, + json={"hosts": ["gitlab.internal.example", "user.internal.example"]}, + ) + assert response.status_code == 200 + assert response.json()["hosts"] == [ + { + "host": "gitlab.internal.example", + "source": "environment", + "readonly": True, + }, + { + "host": "user.internal.example", + "source": "manual", + "readonly": False, + }, + ] + + def test_put_rejects_urls_and_wildcards(self, client: TestClient) -> None: + for invalid in ("https://gitlab.internal", "*.internal"): + response = client.put(self.endpoint, json={"hosts": [invalid]}) + assert response.status_code == 400 + + def test_requires_admin_mcp_permission(self, client_no_perm: TestClient) -> None: + assert client_no_perm.get(self.endpoint).status_code == 403 + assert client_no_perm.put(self.endpoint, json={"hosts": []}).status_code == 403 + + # --------------------------------------------------------------------------- # Create # --------------------------------------------------------------------------- diff --git a/tests/test_mcp_oauth_discovery.py b/tests/test_mcp_oauth_discovery.py index 1abe073c..935af142 100644 --- a/tests/test_mcp_oauth_discovery.py +++ b/tests/test_mcp_oauth_discovery.py @@ -68,6 +68,20 @@ def _public_addr_patch(): return patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]) +def _private_addr_patch(): + return patch("socket.getaddrinfo", return_value=[(2, 1, 6, "", ("192.168.5.120", 0))]) + + +def _private_as_metadata_doc() -> dict[str, Any]: + return { + "issuer": "https://gitlab.internal.example", + "authorization_endpoint": "https://gitlab.internal.example/oauth/authorize", + "token_endpoint": "https://gitlab.internal.example/oauth/token", + "code_challenge_methods_supported": ["S256"], + "token_endpoint_auth_methods_supported": ["none"], + } + + def _mk_storage_mock(server_id: str = "srv-id") -> MagicMock: storage = MagicMock() storage.update_mcp_server.return_value = True @@ -737,3 +751,84 @@ async def _run() -> ASMetadata: with pytest.raises(MCPOAuthDiscoveryError, match="revocation_endpoint"): asyncio.run(_run()) + + +class TestTrustedPrivateHosts: + def test_exact_trusted_private_resource_host_allows_prm_discovery(self) -> None: + async def _get(url: str, *args: Any, **kwargs: Any) -> MagicMock: + if url.endswith("/.well-known/oauth-protected-resource"): + return _mk_response( + 200, + {"authorization_servers": ["https://gitlab.internal.example"]}, + ) + if url.endswith("/.well-known/oauth-authorization-server"): + return _mk_response(200, _private_as_metadata_doc()) + raise AssertionError(f"unexpected URL: {url}") + + client = MagicMock(spec=httpx.AsyncClient) + client.get = AsyncMock(side_effect=_get) + storage = _mk_storage_mock() + + async def _run() -> ASMetadata: + with _private_addr_patch(): + return await discover_authorization_server( + server_name="private-gitlab", + server_url="https://gitlab.internal.example/mcp", + override_url=None, + cached_issuer=None, + http_client=client, + storage=storage, + server_id="srv-id", + trusted_hosts=frozenset(), + trusted_private_hosts=frozenset({"gitlab.internal.example"}), + ) + + metadata = asyncio.run(_run()) + assert metadata.issuer == "https://gitlab.internal.example" + + def test_exact_trusted_private_host_allows_discovery(self) -> None: + client = MagicMock(spec=httpx.AsyncClient) + client.get = AsyncMock(return_value=_mk_response(200, _private_as_metadata_doc())) + storage = _mk_storage_mock() + + async def _run() -> ASMetadata: + with _private_addr_patch(): + return await discover_authorization_server( + server_name="private-gitlab", + server_url="https://gitlab.internal.example/mcp", + override_url="https://gitlab.internal.example", + cached_issuer=None, + http_client=client, + storage=storage, + server_id="srv-id", + trusted_hosts=frozenset(), + trusted_private_hosts=frozenset({"gitlab.internal.example"}), + ) + + metadata = asyncio.run(_run()) + assert metadata.token_endpoint == "https://gitlab.internal.example/oauth/token" + + def test_unlisted_private_host_remains_rejected(self) -> None: + client = MagicMock(spec=httpx.AsyncClient) + client.get = AsyncMock(return_value=_mk_response(200, _private_as_metadata_doc())) + storage = _mk_storage_mock() + + async def _run() -> None: + with _private_addr_patch(): + await discover_authorization_server( + server_name="private-gitlab", + server_url="https://gitlab.internal.example/mcp", + override_url=None, + cached_issuer=None, + http_client=client, + storage=storage, + server_id="srv-id", + trusted_hosts=frozenset(), + trusted_private_hosts=frozenset({"other.internal.example"}), + ) + + with pytest.raises( + MCPOAuthDiscoveryError, + match=r"non-public address.*add 'gitlab\.internal\.example' to Settings → MCP → oauth_trusted_private_hosts", + ): + asyncio.run(_run()) diff --git a/tests/test_mcp_oauth_revoke.py b/tests/test_mcp_oauth_revoke.py index c6f4cffc..8a8982c7 100644 --- a/tests/test_mcp_oauth_revoke.py +++ b/tests/test_mcp_oauth_revoke.py @@ -343,6 +343,7 @@ def _build_args(self) -> dict[str, Any]: }, "server_id_for_audit": "srv-id-1", "refresh_token": "r-secret", + "trusted_private_hosts": frozenset(), } def test_attempt_upstream_revoke_swallows_unexpected_exception(self) -> None: diff --git a/tests/test_mcp_private_hosts.py b/tests/test_mcp_private_hosts.py new file mode 100644 index 00000000..9c3a4ceb --- /dev/null +++ b/tests/test_mcp_private_hosts.py @@ -0,0 +1,62 @@ +"""Trusted private-host configuration for MCP OAuth discovery.""" + +from __future__ import annotations + +import pytest + +from turnstone.core.mcp_private_hosts import ( + MAX_TRUSTED_PRIVATE_HOSTS, + merge_trusted_private_hosts, + normalize_trusted_private_host, + parse_trusted_private_hosts, +) + + +@pytest.mark.parametrize( + "raw", + [ + "https://gitlab.internal.example", + "gitlab.internal.example/path", + "*.internal.example", + "user@gitlab.internal.example", + "gitlab.internal.example:443", + "", + ], +) +def test_normalize_rejects_everything_except_an_exact_host(raw: str) -> None: + with pytest.raises(ValueError): + normalize_trusted_private_host(raw) + + +def test_normalize_is_case_insensitive_and_removes_dns_root_dot() -> None: + assert normalize_trusted_private_host("GitLab.Internal.Example.") == ("gitlab.internal.example") + + +def test_parse_accepts_comma_and_newline_separated_hosts() -> None: + assert parse_trusted_private_hosts("one.internal, TWO.internal.\n10.20.30.40") == ( + "one.internal", + "two.internal", + "10.20.30.40", + ) + + +def test_merge_marks_environment_entries_readonly_and_wins_duplicates() -> None: + merged = merge_trusted_private_hosts( + environment_value="gitlab.internal.example,env.internal", + manual_value="manual.internal\ngitlab.internal.example", + ) + + assert merged == [ + {"host": "gitlab.internal.example", "source": "environment", "readonly": True}, + {"host": "env.internal", "source": "environment", "readonly": True}, + {"host": "manual.internal", "source": "manual", "readonly": False}, + ] + + +def test_merge_caps_the_combined_environment_and_manual_list() -> None: + environment = ",".join(f"env-{i}.internal" for i in range(MAX_TRUSTED_PRIVATE_HOSTS)) + with pytest.raises(ValueError, match="in total"): + merge_trusted_private_hosts( + environment_value=environment, + manual_value="one-more.internal", + ) diff --git a/tests/test_sdk_stream_boundary.py b/tests/test_sdk_stream_boundary.py index 4494a48d..97b96cf6 100644 --- a/tests/test_sdk_stream_boundary.py +++ b/tests/test_sdk_stream_boundary.py @@ -8,7 +8,8 @@ Completions and Responses chunk iterators unwrapped. 2. OpenAI v3's runtime-only legacy-client path preserves the old ``httpx`` exception family when an application explicitly injects that client. -3. The Anthropic ``messages.stream()`` helper propagates the ``httpx`` shape. +3. The Anthropic ``messages.stream()`` helper propagates the transport-error + shape selected by that SDK release (``httpx`` or ``httpx2``). 4. Closing an OpenAI v3 default client from another thread while a read is blocked (the ``ModelRegistry.reload()`` shape) completes safely; a later wire release surfaces as an ``httpx2.TransportError`` on the blocked @@ -164,6 +165,27 @@ def handler(request: httpx2.Request) -> httpx2.Response: return httpx2.MockTransport(handler) +def _anthropic_dying_http_client(payload: str, requests: list): + """Build a mock client from the transport family Anthropic requires. + + Anthropic SDK 1.x moved its default transport from ``httpx`` to + ``httpx2``. ``DefaultHttpxClient`` is the SDK's public transport class, + so its base class is a stable way for this cross-version boundary probe + to select matching mock request, response, and exception types. + """ + if issubclass(anthropic.DefaultHttpxClient, httpx2.Client): + return ( + httpx2.Client(transport=_httpx2_dying_transport(payload, requests)), + httpx2.ReadError, + httpx2.TransportError, + ) + return ( + httpx.Client(transport=_dying_transport(payload, requests)), + httpx.ReadError, + httpx.TransportError, + ) + + @pytest.mark.parametrize("surface", ["chat", "responses"]) @pytest.mark.parametrize( ("status_code", "message", "error_type"), @@ -490,10 +512,13 @@ def test_openai_v3_legacy_httpx_midbody_death_keeps_legacy_error_family(): def test_anthropic_midbody_death_is_unwrapped_readerror_and_no_rerequest(): requests: list = [] + http_client, read_error, transport_error = _anthropic_dying_http_client( + ANTHROPIC_EVENTS, requests + ) client = anthropic.Anthropic( api_key="probe", base_url="http://probe.invalid", - http_client=httpx.Client(transport=_dying_transport(ANTHROPIC_EVENTS, requests)), + http_client=http_client, max_retries=2, ) texts = [] @@ -503,7 +528,7 @@ def test_anthropic_midbody_death_is_unwrapped_readerror_and_no_rerequest(): client.messages.stream( model="m", max_tokens=64, messages=[{"role": "user", "content": "hi"}] ) as stream, - pytest.raises(httpx.ReadError) as excinfo, + pytest.raises(read_error) as excinfo, ): for event in stream: if getattr(event, "type", "") == "content_block_delta": @@ -511,7 +536,7 @@ def test_anthropic_midbody_death_is_unwrapped_readerror_and_no_rerequest(): if getattr(delta, "type", "") == "text_delta": texts.append(delta.text) assert type(excinfo.value).__name__ == "ReadError" - assert isinstance(excinfo.value, httpx.TransportError) + assert isinstance(excinfo.value, transport_error) assert texts == ["hello"] assert len(requests) == 1 @@ -727,9 +752,10 @@ def test_anthropic_arms_eagerly(self): from turnstone.core.providers._anthropic import AnthropicProvider requests: list = [] + http_client, _, _ = _anthropic_dying_http_client(ANTHROPIC_EVENTS, requests) client = anthropic.Anthropic( api_key="probe", - http_client=httpx.Client(transport=_dying_transport(ANTHROPIC_EVENTS, requests)), + http_client=http_client, ) self._armed_at_return(AnthropicProvider(), client) assert len(requests) == 1 diff --git a/turnstone/console/server.py b/turnstone/console/server.py index 8c599518..0a6acd7b 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -10954,6 +10954,94 @@ async def admin_list_mcp_servers(request: Request) -> JSONResponse: return JSONResponse({"servers": result}) +def _trusted_private_hosts_response(config_store: Any) -> dict[str, Any]: + from turnstone.core.mcp_private_hosts import ( + MCP_TRUSTED_PRIVATE_HOSTS_ENV, + configured_trusted_private_hosts, + ) + + return { + "hosts": configured_trusted_private_hosts(config_store), + "environment_variable": MCP_TRUSTED_PRIVATE_HOSTS_ENV, + "help": ( + "Add only exact hostnames or IP addresses for operator-controlled MCP OAuth " + "services that deliberately resolve to a private network. This exception " + "allows private addresses for those hosts only; HTTPS, same-origin, port, " + "userinfo, and dangerous-address protections remain enabled." + ), + } + + +async def admin_get_mcp_trusted_private_hosts(request: Request) -> JSONResponse: + """GET the merged environment and user-managed private OAuth hosts.""" + from turnstone.core.auth import require_permission + + err = require_permission(request, "admin.mcp") + if err: + return err + config_store = getattr(request.app.state, "config_store", None) + if config_store is None: + return JSONResponse({"error": "Configuration store unavailable"}, status_code=503) + try: + return JSONResponse(_trusted_private_hosts_response(config_store)) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=500) + + +async def admin_update_mcp_trusted_private_hosts(request: Request) -> JSONResponse: + """PUT user-managed private OAuth hosts without mutating environment entries.""" + from turnstone.core.auth import require_permission + from turnstone.core.mcp_private_hosts import ( + MAX_TRUSTED_PRIVATE_HOSTS, + MCP_TRUSTED_PRIVATE_HOSTS_ENV, + MCP_TRUSTED_PRIVATE_HOSTS_SETTING, + merge_trusted_private_hosts, + normalize_trusted_private_host, + parse_trusted_private_hosts, + ) + from turnstone.core.web_helpers import read_json_or_400 + + err = require_permission(request, "admin.mcp") + if err: + return err + config_store = getattr(request.app.state, "config_store", None) + if config_store is None: + return JSONResponse({"error": "Configuration store unavailable"}, status_code=503) + body = await read_json_or_400(request) + if isinstance(body, JSONResponse): + return body + raw_hosts = body.get("hosts") + if not isinstance(raw_hosts, list) or any(not isinstance(host, str) for host in raw_hosts): + return JSONResponse({"error": "hosts must be a list of strings"}, status_code=400) + if len(raw_hosts) > MAX_TRUSTED_PRIVATE_HOSTS: + return JSONResponse( + {"error": f"at most {MAX_TRUSTED_PRIVATE_HOSTS} hosts are allowed"}, + status_code=400, + ) + try: + environment_hosts = set( + parse_trusted_private_hosts(os.environ.get(MCP_TRUSTED_PRIVATE_HOSTS_ENV)) + ) + manual_hosts = list(dict.fromkeys(normalize_trusted_private_host(h) for h in raw_hosts)) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + manual_hosts = [host for host in manual_hosts if host not in environment_hosts] + try: + merge_trusted_private_hosts( + environment_value=os.environ.get(MCP_TRUSTED_PRIVATE_HOSTS_ENV), + manual_value="\n".join(manual_hosts), + ) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + config_store.set( + MCP_TRUSTED_PRIVATE_HOSTS_SETTING, + "\n".join(manual_hosts), + changed_by=_auth_user_id(request), + ) + await _publish_config_change(request) + return JSONResponse(_trusted_private_hosts_response(config_store)) + + async def admin_create_mcp_server(request: Request) -> JSONResponse: """POST /v1/api/admin/mcp-servers — create an MCP server definition.""" import uuid @@ -15835,7 +15923,9 @@ def _seed_config_from_env(config_store: Any, storage: Any) -> None: """ from turnstone.core.settings_registry import SETTINGS - for key in SETTINGS: + for key, defn in SETTINGS.items(): + if not defn.seed_from_env: + continue env_name = "TURNSTONE_" + key.replace(".", "_").upper() env_val = os.environ.get(env_name) if env_val is None: @@ -16426,6 +16516,15 @@ def _coord_attachment_owner( admin_mcp_reload, methods=["POST"], ), + Route( + "/api/admin/mcp-servers/trusted-private-hosts", + admin_get_mcp_trusted_private_hosts, + ), + Route( + "/api/admin/mcp-servers/trusted-private-hosts", + admin_update_mcp_trusted_private_hosts, + methods=["PUT"], + ), Route( "/api/admin/mcp-servers/{name}/refresh", admin_mcp_refresh_one, diff --git a/turnstone/console/static/admin.js b/turnstone/console/static/admin.js index 80fb6019..5d23f14e 100644 --- a/turnstone/console/static/admin.js +++ b/turnstone/console/static/admin.js @@ -4086,6 +4086,7 @@ function loadSettings() { } _renderSettings(el, grouped); + if (_consoleHasPermission("admin.mcp")) loadMcpTrustedPrivateHosts(); }) .catch(function (err) { // NOTE: escapeHtml sanitises err.message before insertion. @@ -4120,7 +4121,11 @@ function _renderSettings(container, grouped) { '
'; for (let j = 0; j < items.length; j++) { - html += _renderSettingRow(items[j]); + html += + items[j].key === "mcp.oauth_trusted_private_hosts" && + _consoleHasPermission("admin.mcp") + ? _renderMcpTrustedPrivateHostsSetting(items[j]) + : _renderSettingRow(items[j]); } html += "
"; @@ -4201,6 +4206,52 @@ function _renderSettings(container, grouped) { } } +function _renderMcpTrustedPrivateHostsSetting(item) { + const escapedKey = escapeHtml(item.key); + const shortKey = "oauth_trusted_private_hosts"; + const helpId = escapedKey + "-help"; + + let html = + '
'; + html += '
'; + html += '
' + shortKey; + html += + ' '; + html += "
"; + html += '
' + escapeHtml(item.description) + "
"; + html += + '
"; + html += '
'; + html += + '
' + + '' + + '' + + '' + + "
"; + html += + '
' + + 'Loading trusted hosts...
'; + html += + '
'; + html += "
"; + return html; +} + function _renderSettingRow(item) { const shortKey = item.key.indexOf(".") !== -1 @@ -4696,6 +4747,7 @@ let _mcpCurrentView = "servers"; let _registryResults = []; let _registryCursor = null; let _registryQuery = ""; +let _mcpTrustedPrivateHosts = []; function loadAdminMcp() { authFetch("/v1/api/admin/mcp-servers") @@ -4724,6 +4776,132 @@ function loadAdminMcp() { }); } +function loadMcpTrustedPrivateHosts() { + _wireMcpPrivateHosts(); + authFetch("/v1/api/admin/mcp-servers/trusted-private-hosts") + .then(function (r) { + if (!r.ok) { + return r.json().then(function (data) { + throw new Error(data.error || "Failed to load trusted private hosts"); + }); + } + return r.json(); + }) + .then(function (data) { + _mcpTrustedPrivateHosts = data.hosts || []; + _renderMcpTrustedPrivateHosts(); + _setMcpPrivateHostStatus(""); + }) + .catch(function (error) { + _setMcpPrivateHostStatus(error.message || "Failed to load trusted private hosts", true); + }); +} + +function _wireMcpPrivateHosts() { + const form = document.getElementById("mcp-private-host-form"); + const input = document.getElementById("mcp-private-host-input"); + if (!form || !input) return; + if (form.dataset.wired === "true") return; + form.dataset.wired = "true"; + form.addEventListener("submit", function (event) { + event.preventDefault(); + const host = input.value.trim(); + if (!host) return; + const manual = _mcpTrustedPrivateHosts + .filter(function (entry) { + return entry.source === "manual"; + }) + .map(function (entry) { + return entry.host; + }); + manual.push(host); + _saveMcpTrustedPrivateHosts(manual, function () { + input.value = ""; + input.focus(); + }); + }); +} + +function _saveMcpTrustedPrivateHosts(manualHosts, onSuccess) { + _setMcpPrivateHostStatus("Saving..."); + authFetch("/v1/api/admin/mcp-servers/trusted-private-hosts", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hosts: manualHosts }), + }) + .then(function (r) { + return r.json().then(function (data) { + if (!r.ok) throw new Error(data.error || "Failed to save trusted private hosts"); + return data; + }); + }) + .then(function (data) { + _mcpTrustedPrivateHosts = data.hosts || []; + _renderMcpTrustedPrivateHosts(); + _setMcpPrivateHostStatus("Trusted private hosts saved."); + if (onSuccess) onSuccess(); + }) + .catch(function (error) { + _setMcpPrivateHostStatus(error.message || "Failed to save trusted private hosts", true); + }); +} + +function _renderMcpTrustedPrivateHosts() { + const list = document.getElementById("mcp-private-host-list"); + if (!list) return; + list.replaceChildren(); + if (!_mcpTrustedPrivateHosts.length) { + const empty = document.createElement("span"); + empty.className = "dashboard-empty"; + empty.textContent = "No private hosts are trusted."; + list.appendChild(empty); + return; + } + _mcpTrustedPrivateHosts.forEach(function (entry) { + const row = document.createElement("div"); + row.className = "mcp-private-host-row"; + row.setAttribute("role", "listitem"); + + const host = document.createElement("code"); + host.className = "mcp-private-host-name"; + host.textContent = entry.host; + row.appendChild(host); + + const source = document.createElement("span"); + source.className = "scope-badge " + + (entry.source === "environment" ? "mcp-host-source-environment" : "mcp-host-source-manual"); + source.textContent = entry.source === "environment" ? "environment" : "manual"; + row.appendChild(source); + + if (!entry.readonly) { + const remove = document.createElement("button"); + remove.type = "button"; + remove.className = "settings-reset-btn mcp-private-host-remove"; + remove.textContent = "Remove"; + remove.setAttribute("aria-label", "Remove trusted private host " + entry.host); + remove.addEventListener("click", function () { + const remaining = _mcpTrustedPrivateHosts + .filter(function (candidate) { + return candidate.source === "manual" && candidate.host !== entry.host; + }) + .map(function (candidate) { + return candidate.host; + }); + _saveMcpTrustedPrivateHosts(remaining); + }); + row.appendChild(remove); + } + list.appendChild(row); + }); +} + +function _setMcpPrivateHostStatus(message, isError) { + const status = document.getElementById("mcp-private-host-status"); + if (!status) return; + status.textContent = message; + status.classList.toggle("is-error", Boolean(isError)); +} + function _wireMcpTokenDropButtons(el, attr, opts) { // Shared binder for the two per-server token-drop list actions — // "Bulk-revoke" (oauth_user consents) and "Flush cache" (oauth_obo minted diff --git a/turnstone/console/static/style.css b/turnstone/console/static/style.css index 0be5166c..9b830624 100644 --- a/turnstone/console/static/style.css +++ b/turnstone/console/static/style.css @@ -3016,7 +3016,79 @@ h3.skill-spec-heading { box-shadow: 0 0 0 3px rgba(192, 132, 252, 0.15); } +/* -- Trusted private OAuth hosts setting --------------------------------- */ +.settings-row-mcp-private-hosts { + grid-template-columns: 200px minmax(0, 1fr); +} +.settings-mcp-private-hosts-input { + min-width: 0; +} +.mcp-private-host-name { + font-family: var(--font-mono); +} +.mcp-private-host-form { + display: flex; + gap: 8px; + max-width: 480px; + margin-bottom: 10px; +} +.mcp-private-host-form input { + flex: 1; + min-width: 180px; +} +.mcp-private-host-list { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.mcp-private-host-row { + display: inline-flex; + align-items: center; + gap: 6px; + min-width: 0; + padding: 5px 6px 5px 9px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--bg-highlight); +} +.mcp-private-host-name { + overflow-wrap: anywhere; + color: var(--fg-bright); + font-size: 11px; +} +.mcp-host-source-environment { + color: var(--cyan); + border-color: rgba(103, 232, 249, 0.25); +} +.mcp-host-source-manual { + color: var(--magenta); + border-color: rgba(192, 132, 252, 0.25); +} +.mcp-private-host-remove { + padding: 2px 6px; + font-size: 9px; +} +.mcp-private-host-status { + min-height: 16px; + margin-top: 6px; + color: var(--green); + font-size: 10px; +} +.mcp-private-host-status.is-error { + color: var(--red); +} + +@media (max-width: 700px) { + .settings-row-mcp-private-hosts { + grid-template-columns: 1fr; + } + .mcp-private-host-form { + max-width: none; + } +} + /* -- MCP Registry result cards -------------------------------------------- */ + .mcp-reg-card { display: grid; grid-template-columns: 1fr auto; diff --git a/turnstone/core/mcp_oauth.py b/turnstone/core/mcp_oauth.py index 28a812e5..b15f4973 100644 --- a/turnstone/core/mcp_oauth.py +++ b/turnstone/core/mcp_oauth.py @@ -48,6 +48,10 @@ is_valid_scope_token, parse_www_authenticate_bearer, ) +from turnstone.core.mcp_private_hosts import ( + trusted_private_host_set, + url_uses_trusted_private_host, +) from turnstone.core.model_registry import ( MODEL_AUTH_TEXT_MAX_LEN, sanitize_backend_auth_scopes, @@ -55,6 +59,7 @@ ) from turnstone.core.oauth_ssrf import ( OAuthSSRFError, + OAuthSSRFPrivateAddressError, sanitize_log_text, validate_discovered_endpoint_async, validate_url_no_ssrf_async, @@ -72,6 +77,39 @@ log = get_logger(__name__) +def _configured_trusted_private_hosts(app_state: Any) -> frozenset[str]: + """Read the live merged allow-list, failing closed on invalid config.""" + try: + return trusted_private_host_set(getattr(app_state, "config_store", None)) + except ValueError as exc: + log.error( + "mcp_server.oauth.trusted_private_hosts_invalid", + reason=sanitize_log_text(str(exc)), + ) + return frozenset() + + +def _private_host_remediation(url: str) -> str: + """Return the operator action for a private-address discovery rejection.""" + hostname = urllib.parse.urlparse(url).hostname + if not hostname: + return "" + return ( + " To allow this exact host, add " + f"'{hostname}' to Settings → MCP → oauth_trusted_private_hosts." + ) + + +def _discovery_url_rejection( + prefix: str, url: str, exc: OAuthSSRFError +) -> MCPOAuthDiscoveryError: + """Preserve strict validation errors and guide private-host remediation.""" + remediation = ( + _private_host_remediation(url) if isinstance(exc, OAuthSSRFPrivateAddressError) else "" + ) + return MCPOAuthDiscoveryError(f"{prefix}: {exc}{remediation}") + + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -203,6 +241,7 @@ async def _fetch_prm_issuer( server_url: str, *, http_client: httpx.AsyncClient, + trusted_private_hosts: frozenset[str], ) -> str: """Fetch the protected-resource metadata document and return its ``authorization_servers[0]``. @@ -224,9 +263,13 @@ async def _fetch_prm_issuer( prm_url = base.rstrip("/") + "/.well-known/oauth-protected-resource" try: - await validate_url_no_ssrf_async(prm_url, allow_http=True) + await validate_url_no_ssrf_async( + prm_url, + allow_http=True, + allow_private=url_uses_trusted_private_host(prm_url, trusted_private_hosts), + ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"PRM URL rejected: {exc}") from exc + raise _discovery_url_rejection("PRM URL rejected", prm_url, exc) from exc try: resp = await http_client.get(prm_url, timeout=_DEFAULT_HTTP_TIMEOUT) @@ -243,9 +286,13 @@ async def _fetch_prm_issuer( "server returned 401 without resource_metadata in WWW-Authenticate" ) try: - await validate_url_no_ssrf_async(challenge_url, allow_http=True) + await validate_url_no_ssrf_async( + challenge_url, + allow_http=True, + allow_private=url_uses_trusted_private_host(challenge_url, trusted_private_hosts), + ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"PRM challenge URL rejected: {exc}") from exc + raise _discovery_url_rejection("PRM challenge URL rejected", challenge_url, exc) from exc try: resp = await http_client.get(challenge_url, timeout=_DEFAULT_HTTP_TIMEOUT) except httpx.HTTPError as exc: @@ -276,9 +323,13 @@ async def _fetch_prm_issuer( # SSRF protection on the issuer URL itself — same-origin / trust-list # checks happen in ``_fetch_as_metadata``. try: - await validate_url_no_ssrf_async(issuer_url, allow_http=True) + await validate_url_no_ssrf_async( + issuer_url, + allow_http=True, + allow_private=url_uses_trusted_private_host(issuer_url, trusted_private_hosts), + ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"PRM issuer URL rejected: {exc}") from exc + raise _discovery_url_rejection("PRM issuer URL rejected", issuer_url, exc) from exc return issuer_url @@ -288,6 +339,7 @@ async def _fetch_as_metadata( *, http_client: httpx.AsyncClient, trusted_hosts: frozenset[str], + trusted_private_hosts: frozenset[str], ) -> ASMetadata: """Fetch ``.well-known/oauth-authorization-server`` for *issuer*. @@ -297,9 +349,13 @@ async def _fetch_as_metadata( :class:`MCPOAuthDiscoveryError` otherwise. """ try: - issuer_parsed = await validate_url_no_ssrf_async(issuer, allow_http=True) + issuer_parsed = await validate_url_no_ssrf_async( + issuer, + allow_http=True, + allow_private=url_uses_trusted_private_host(issuer, trusted_private_hosts), + ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"AS issuer URL rejected: {exc}") from exc + raise _discovery_url_rejection("AS issuer URL rejected", issuer, exc) from exc # MCP auth permits OpenID Connect discovery as a fallback to RFC 8414. # Major IdPs (notably Microsoft Entra) serve ONLY the OIDC document @@ -373,9 +429,12 @@ async def _fetch_as_metadata( issuer_parsed, allow_http=allow_http, trusted_endpoint_hosts=trusted_hosts, + allow_private=url_uses_trusted_private_host(endpoint_url, trusted_private_hosts), ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"AS {name} rejected (url={endpoint_url}): {exc}") from exc + raise _discovery_url_rejection( + f"AS {name} rejected (url={endpoint_url})", endpoint_url, exc + ) from exc for opt_name, opt_url in ( ("registration_endpoint", registration_endpoint), @@ -389,9 +448,12 @@ async def _fetch_as_metadata( issuer_parsed, allow_http=allow_http, trusted_endpoint_hosts=trusted_hosts, + allow_private=url_uses_trusted_private_host(opt_url, trusted_private_hosts), ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"AS {opt_name} rejected (url={opt_url}): {exc}") from exc + raise _discovery_url_rejection( + f"AS {opt_name} rejected (url={opt_url})", opt_url, exc + ) from exc code_methods_raw = doc.get("code_challenge_methods_supported", []) if not isinstance(code_methods_raw, list): @@ -453,6 +515,7 @@ async def discover_authorization_server( server_id: str, trusted_hosts: frozenset[str], metadata_cache: dict[str, tuple[ASMetadata, float]] | None = None, + trusted_private_hosts: frozenset[str] = frozenset(), ) -> ASMetadata: """Resolve the issuer URL and load AS metadata for an MCP server. @@ -472,9 +535,13 @@ async def discover_authorization_server( issuer: str if override_url: try: - await validate_url_no_ssrf_async(override_url, allow_http=True) + await validate_url_no_ssrf_async( + override_url, + allow_http=True, + allow_private=url_uses_trusted_private_host(override_url, trusted_private_hosts), + ) except OAuthSSRFError as exc: - raise MCPOAuthDiscoveryError(f"override AS URL rejected: {exc}") from exc + raise _discovery_url_rejection("override AS URL rejected", override_url, exc) from exc issuer = override_url elif cached_issuer: # Defense-in-depth: re-run SSRF validation on the cached value. @@ -482,7 +549,11 @@ async def discover_authorization_server( # we cached it (or the operator edited the row to point at a # private host), drop the cache and fall through to PRM. try: - await validate_url_no_ssrf_async(cached_issuer, allow_http=True) + await validate_url_no_ssrf_async( + cached_issuer, + allow_http=True, + allow_private=url_uses_trusted_private_host(cached_issuer, trusted_private_hosts), + ) except OAuthSSRFError as exc: log.warning( "mcp_server.oauth.cached_issuer_rejected", @@ -502,11 +573,19 @@ async def discover_authorization_server( server_name=server_name, exc_info=True, ) - issuer = await _fetch_prm_issuer(server_url, http_client=http_client) + issuer = await _fetch_prm_issuer( + server_url, + http_client=http_client, + trusted_private_hosts=trusted_private_hosts, + ) else: issuer = cached_issuer else: - issuer = await _fetch_prm_issuer(server_url, http_client=http_client) + issuer = await _fetch_prm_issuer( + server_url, + http_client=http_client, + trusted_private_hosts=trusted_private_hosts, + ) if metadata_cache is not None: cached = metadata_cache.get(issuer) @@ -516,7 +595,10 @@ async def discover_authorization_server( return metadata metadata = await _fetch_as_metadata( - issuer, http_client=http_client, trusted_hosts=trusted_hosts + issuer, + http_client=http_client, + trusted_hosts=trusted_hosts, + trusted_private_hosts=trusted_private_hosts, ) if metadata_cache is not None: @@ -3752,6 +3834,7 @@ async def _refresh_and_persist( server_id=server_id, trusted_hosts=frozenset(), metadata_cache=metadata_cache, + trusted_private_hosts=_configured_trusted_private_hosts(app_state), ) except MCPOAuthDiscoveryError as exc: raise MCPOAuthRefreshFailed(f"discovery failed during refresh: {exc}") from exc @@ -4290,6 +4373,7 @@ async def _handle_mcp_oauth_authorize_inner(request: Request) -> Response: server_id=server_id, trusted_hosts=frozenset(), metadata_cache=metadata_cache, + trusted_private_hosts=_configured_trusted_private_hosts(request.app.state), ) except MCPOAuthDiscoveryError as exc: log.warning("mcp_server.oauth.discovery_failed", server_name=server_name, exc_info=True) @@ -4534,6 +4618,7 @@ async def _handle_mcp_oauth_callback_inner(request: Request) -> Response: server_id=server_id, trusted_hosts=frozenset(), metadata_cache=metadata_cache, + trusted_private_hosts=_configured_trusted_private_hosts(request.app.state), ) except MCPOAuthDiscoveryError as exc: log.warning( @@ -4802,6 +4887,7 @@ async def _attempt_upstream_revoke( server_row: dict[str, Any], server_id_for_audit: str, refresh_token: str, + trusted_private_hosts: frozenset[str], ) -> None: """Best-effort RFC 7009 upstream revoke for ``user_revoked`` flow. @@ -4838,6 +4924,7 @@ async def _attempt_upstream_revoke( server_id=server_id_for_audit, trusted_hosts=frozenset(), metadata_cache=metadata_cache, + trusted_private_hosts=trusted_private_hosts, ) except MCPOAuthDiscoveryError as exc: log.info( @@ -5013,6 +5100,7 @@ async def _handle_mcp_oauth_revoke_connection_inner(request: Request) -> Respons server_row=server_row, server_id_for_audit=server_id_for_audit, refresh_token=refresh_token_for_revoke, + trusted_private_hosts=_configured_trusted_private_hosts(request.app.state), ), name="mcp-oauth-upstream-revoke", ) diff --git a/turnstone/core/mcp_private_hosts.py b/turnstone/core/mcp_private_hosts.py new file mode 100644 index 00000000..8d54d91e --- /dev/null +++ b/turnstone/core/mcp_private_hosts.py @@ -0,0 +1,130 @@ +"""Operator-managed private-network exceptions for MCP OAuth discovery. + +The allow-list is deliberately host-only and exact-match. It relaxes only +the private-address portion of OAuth SSRF validation; HTTPS, userinfo, +same-origin, port, and dangerous-address checks remain in force. +""" + +from __future__ import annotations + +import ipaddress +import os +import re +import urllib.parse +from typing import Any + +MCP_TRUSTED_PRIVATE_HOSTS_ENV = "TURNSTONE_MCP_OAUTH_TRUSTED_PRIVATE_HOSTS" +MCP_TRUSTED_PRIVATE_HOSTS_SETTING = "mcp.oauth_trusted_private_hosts" +MAX_TRUSTED_PRIVATE_HOSTS = 100 + +_DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") + + +def normalize_trusted_private_host(raw: str) -> str: + """Return a canonical exact hostname/IP or raise ``ValueError``.""" + value = str(raw).strip() + if not value: + raise ValueError("host must not be empty") + if any(marker in value for marker in ("://", "/", "?", "#", "@", "*")): + raise ValueError("use an exact hostname or IP address, without a URL or wildcard") + + # IP literals are accepted because urllib.parse exposes them as exact + # hostname strings during discovery. Brackets belong to URL syntax, not + # to the configured host value. + try: + return ipaddress.ip_address(value).compressed.lower() + except ValueError: + pass + + if ":" in value: + raise ValueError("ports are not allowed; enter only the exact hostname") + value = value.rstrip(".").lower() + try: + ascii_value = value.encode("idna").decode("ascii") + except UnicodeError as exc: + raise ValueError("host is not a valid DNS name") from exc + if len(ascii_value) > 253 or not ascii_value: + raise ValueError("host is not a valid DNS name") + if any(not _DNS_LABEL_RE.fullmatch(label) for label in ascii_value.split(".")): + raise ValueError("host is not a valid DNS name") + return ascii_value + + +def parse_trusted_private_hosts(raw: str | None) -> tuple[str, ...]: + """Parse a comma/newline-separated host list, preserving first-seen order.""" + if raw is None or not str(raw).strip(): + return () + result: list[str] = [] + seen: set[str] = set() + for item in re.split(r"[,\n]", str(raw)): + if not item.strip(): + continue + host = normalize_trusted_private_host(item) + if host not in seen: + seen.add(host) + result.append(host) + if len(result) > MAX_TRUSTED_PRIVATE_HOSTS: + raise ValueError( + f"at most {MAX_TRUSTED_PRIVATE_HOSTS} trusted private hosts are allowed" + ) + return tuple(result) + + +def merge_trusted_private_hosts( + *, environment_value: str | None, manual_value: str | None +) -> list[dict[str, object]]: + """Merge environment and user entries, with environment taking precedence.""" + environment_hosts = parse_trusted_private_hosts(environment_value) + manual_hosts = parse_trusted_private_hosts(manual_value) + merged: list[dict[str, object]] = [ + {"host": host, "source": "environment", "readonly": True} for host in environment_hosts + ] + environment_set = set(environment_hosts) + merged.extend( + {"host": host, "source": "manual", "readonly": False} + for host in manual_hosts + if host not in environment_set + ) + if len(merged) > MAX_TRUSTED_PRIVATE_HOSTS: + raise ValueError( + f"at most {MAX_TRUSTED_PRIVATE_HOSTS} trusted private hosts are allowed in total" + ) + return merged + + +def configured_trusted_private_hosts(config_store: Any) -> list[dict[str, object]]: + """Read and merge the live environment and database-backed user setting.""" + manual = "" + if config_store is not None: + manual = str(config_store.get(MCP_TRUSTED_PRIVATE_HOSTS_SETTING, "") or "") + return merge_trusted_private_hosts( + environment_value=os.environ.get(MCP_TRUSTED_PRIVATE_HOSTS_ENV), + manual_value=manual, + ) + + +def trusted_private_host_set(config_store: Any) -> frozenset[str]: + """Return the effective exact-match set used by OAuth discovery.""" + return frozenset(str(entry["host"]) for entry in configured_trusted_private_hosts(config_store)) + + +def url_uses_trusted_private_host(url: str, trusted_hosts: frozenset[str]) -> bool: + """Whether *url* names one of the exact, canonical trusted hosts.""" + try: + hostname = urllib.parse.urlparse(url).hostname + return bool(hostname and normalize_trusted_private_host(hostname) in trusted_hosts) + except ValueError: + return False + + +__all__ = [ + "MAX_TRUSTED_PRIVATE_HOSTS", + "MCP_TRUSTED_PRIVATE_HOSTS_ENV", + "MCP_TRUSTED_PRIVATE_HOSTS_SETTING", + "configured_trusted_private_hosts", + "merge_trusted_private_hosts", + "normalize_trusted_private_host", + "parse_trusted_private_hosts", + "trusted_private_host_set", + "url_uses_trusted_private_host", +] diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index c96ded2c..2389bd1f 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -31,6 +31,7 @@ class SettingDef: help: str = "" # plain-English explanation for non-experts reference_url: str = "" # link to arXiv, docs, or provider reference strict_int: bool = False # reject bool/float/non-canonical strings before int coercion + seed_from_env: bool = True # false when env values must remain live/read-only # Default auto-compaction trigger as a fraction of the context window. Shared @@ -488,6 +489,18 @@ def _build_registry() -> dict[str, SettingDef]: "Leave empty to use the official registry at registry.modelcontextprotocol.io.", reference_url="https://registry.modelcontextprotocol.io", ), + SettingDef( + "mcp.oauth_trusted_private_hosts", + "str", + "", + "Exact private-network hosts trusted for MCP OAuth discovery", + "mcp", + help="Comma- or newline-separated exact hostnames or IP addresses. This allows " + "OAuth discovery for operator-controlled MCP services that resolve to private " + "addresses. HTTPS and the remaining SSRF protections still apply. Environment " + "entries are merged separately and cannot be edited here.", + seed_from_env=False, + ), # -- ratelimit ------------------------------------------------------ SettingDef( "ratelimit.enabled", @@ -1054,6 +1067,11 @@ def validate_value(key: str, raw_value: Any) -> Any: if defn.choices is not None and typed not in defn.choices: raise ValueError(f"{key}: {typed!r} not in {defn.choices}") + if key == "mcp.oauth_trusted_private_hosts": + from turnstone.core.mcp_private_hosts import parse_trusted_private_hosts + + typed = "\n".join(parse_trusted_private_hosts(typed)) + return typed