Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 17 additions & 21 deletions scripts/capture_rpc_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,9 @@
* ABSENT — our id no longer appears in the bundle at all (rotation/stale — the alarm)
* PRESENT-UNPARSED— our id string is in the bundle but its registration form wasn't
parsed (not a rotation; a parser gap to widen, not an alert)
* LAZY-MODULE — a live-probed id whose registration is loaded only with a
feature module and is therefore intentionally absent from
the authenticated homepage's eager bundle set (not an alert;
the direct RPC health probe remains its rotation canary)
* LAZY-MODULE — a live-probed feature id that can be absent from the
authenticated homepage's eager bundle set. The direct RPC
health probe is its rotation canary.
* UNMAPPED — a live RPC the bundle declares that we don't expose, grouped by
service family: **current** (old `LabsTailwind*` consumer backend
— callable on our cohort now, just unexposed), **enterprise** (the
Expand All @@ -45,7 +44,7 @@
AudioFormat / app-variants), the ``Yp`` **quota-code** map (a feature-rollout
early-warning surface), and proto **required-field assertions** (schema-shape
drift). ``--check-enums`` extracts and diffs the switch enums against the int
enums in ``rpc/types.py`` with the same four-class spirit as the id diff, but a
enums in ``_types/enums.py`` with the same four-class spirit as the id diff, but a
distinct taxonomy — see :func:`diff_enums` for why ``NEW`` is report-only.

Auth: discovering the bundle URL needs **one authenticated homepage read** (an
Expand Down Expand Up @@ -127,13 +126,11 @@

_AUTH_FAILURE_EXIT = 2

# The authenticated homepage exposes only the eager JS bundle set. This RPC is
# registered by the source-search feature's lazy module, so it cannot be proven
# by the entry-bundle scrape even though both Web batchexecute and Android gRPC
# accepted it live on 2026-09-01. Keep it in a separate report bucket instead
# of falsely calling it CONFIRMED or ABSENT. ``check_rpc_health.py`` sends a
# direct read-only request for the id, so a rotation still has a live canary.
_LAZY_MODULE_RPC_IDS: frozenset[str] = frozenset({"ASU5Oe"})
# The authenticated homepage exposes only the eager JS bundle set. Source search
# (ASU5Oe) loads through a feature module. Connected Apps (EEr8Nb) was missing
# from the eager bundle on 2026-11-19 while that same CI run got its ID back from
# batchexecute (#2562). Both have direct read-only probes in check_rpc_health.py.
_LAZY_MODULE_RPC_IDS: frozenset[str] = frozenset({"ASU5Oe", "EEr8Nb"})


class _BundleAuthenticationError(RuntimeError):
Expand All @@ -155,7 +152,7 @@ def _write_outcome(path: Path | None, outcome: str) -> None:
_DEFAULT_TYPES = (
Path(__file__).resolve().parent.parent / "src" / "notebooklm" / "rpc" / "_identifiers.py"
)
_DEFAULT_ENUM_TYPES = _DEFAULT_TYPES.with_name("types.py")
_DEFAULT_ENUM_TYPES = _DEFAULT_TYPES.parent.parent / "_types" / "enums.py"

# --- Service-family classification: consumer backend vs enterprise (Discovery Engine) ---
# "Current" (the consumer backend serving our cohort now) is detected *empirically*:
Expand Down Expand Up @@ -359,7 +356,7 @@ def _in_bundle(rpc_id: str) -> bool:
# minified ``switch`` statements mapping an integer code to a display label,
# e.g. ``switch(a){case 1:return"Explainer";case 3:return"Cinematic";...}``.
# These are the human-facing labels for VideoFormat / AudioFormat / the
# app-variant picker — the same integers we hardcode in ``rpc/types.py`` and
# app-variant picker — the same integers we hardcode in ``_types/enums.py`` and
# send on the wire. If Google ever renumbers a *selectable* format (the #1597
# alarm: the VideoStyle/format code an existing label maps to changes) every
# generate-* call silently produces the wrong artifact, so we diff them.
Expand All @@ -380,7 +377,7 @@ def _in_bundle(rpc_id: str) -> bool:
# anchor set is attributed to that enum (so a block that gained an unreleased
# label still matches). Anchors are deliberately a handful of stable,
# distinctive labels — not the full set — so a NEW label never breaks
# attribution. The keys are our ``rpc/types.py`` enum class names.
# attribution. The keys are our studio enum class names.
_ENUM_LABEL_ANCHORS: dict[str, frozenset[str]] = {
"VideoFormat": frozenset({"Explainer", "Cinematic"}),
"AudioFormat": frozenset({"Deep Dive", "Critique", "Debate"}),
Expand Down Expand Up @@ -421,7 +418,7 @@ def extract_switch_enums(bundle: str) -> dict[str, dict[int, str]]:


def parse_enum_members_from_text(types_text: str, enum_name: str) -> dict[str, int]:
"""Return ``{MEMBER_NAME: int_value}`` for an ``(int, Enum)`` class in types.py.
"""Return ``{MEMBER_NAME: int_value}`` for an ``(int, Enum)`` class in source text.

Mirrors :func:`parse_ids_from_text` but for the integer studio enums. Scoped
to the named class body so members of other enums don't bleed in. Aliases
Expand Down Expand Up @@ -727,7 +724,7 @@ def _print_report(
print(f" {rpc_id:<8} {present[rpc_id]}")
if lazy_module:
print(
"\nLAZY-MODULE — registration is outside the eager bundle; "
"\nLAZY-MODULE — id absent from the eager bundle; "
"direct health probe owns drift detection:"
)
for rpc_id in sorted(lazy_module, key=lambda i: lazy_module[i]):
Expand Down Expand Up @@ -869,9 +866,8 @@ def main(argv: list[str] | None = None) -> int:
types_text = args.types.read_text(encoding="utf-8")
# ``--types`` historically accepted one synthetic file containing both
# RPCMethod and studio enums, which the offline tests and external scripts
# may still rely on. The repository default now splits those owners: IDs
# are dependency-bottom while the compatibility module still re-exports
# transport enums. Preserve custom-file behavior and split only our default.
# may still rely on. Preserve custom-file behavior and read the studio enum
# definitions from their owner when using the repository defaults.
enum_types_text = (
_DEFAULT_ENUM_TYPES.read_text(encoding="utf-8")
if args.types == _DEFAULT_TYPES
Expand Down Expand Up @@ -981,7 +977,7 @@ def main(argv: list[str] | None = None) -> int:
print(
f"\nFAIL: {len(enum_buckets['changed'])} CHANGED and "
f"{len(enum_buckets['stale'])} STALE studio enum value(s) — a selectable "
"format was renumbered or retired; re-capture rpc/types.py enums.",
"format was renumbered or retired; re-capture _types/enums.py values.",
file=sys.stderr,
)
exit_code = 1
Expand Down
54 changes: 51 additions & 3 deletions scripts/check_rpc_health.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,13 @@
)
from notebooklm._web.params.artifacts import build_retry_artifact_params
from notebooklm._web.params.chat_stream import build_streaming_chat_request
from notebooklm._web.params.notebooks import build_create_notebook_params
from notebooklm._web.params.notebooks import (
build_create_notebook_params,
build_toolset_preferences_params,
)
from notebooklm._web.rows.chat_stream import parse_streaming_chat_response
from notebooklm._web.rows.customization import unwrap_customization_choices
from notebooklm._web.rows.notebooks import decode_toolset_preferences
from notebooklm._web.wire.decoder import (
collect_rpc_ids,
decode_response,
Expand All @@ -134,7 +138,7 @@
get_batchexecute_url,
)
from notebooklm.rpc.types import _QUERY_ENDPOINT_PATH
from notebooklm.types import AudioFormat, SlideDeckFormat, VideoFormat
from notebooklm.types import AudioFormat, GrpcStatusCode, SlideDeckFormat, VideoFormat


class CheckStatus(str, Enum):
Expand Down Expand Up @@ -716,12 +720,14 @@ def get_test_params(method: RPCMethod, notebook_id: str | None) -> list[Any] | N
if not notebook_id:
return None

if method == RPCMethod.LIST_TOOLSET_PREFERENCES:
return build_toolset_preferences_params(notebook_id)

# Methods that take [notebook_id] as the only param
if method in (
RPCMethod.GET_NOTEBOOK,
RPCMethod.GET_SOURCE_GUIDE,
RPCMethod.GET_SHARE_STATUS,
RPCMethod.LIST_TOOLSET_PREFERENCES,
RPCMethod.REMOVE_RECENTLY_VIEWED,
):
return [notebook_id]
Expand Down Expand Up @@ -877,6 +883,45 @@ def get_test_params(method: RPCMethod, notebook_id: str | None) -> list[Any] | N
return None


async def _check_toolset_preferences(
client: httpx.AsyncClient,
auth: AuthTokens,
notebook_id: str,
params: list[Any],
) -> CheckResult:
"""Validate the notebook-scoped read that backs the eager-bundle exception."""
method = RPCMethod.LIST_TOOLSET_PREFERENCES
rpc_id = method.value
result = CheckResult(method, CheckStatus.ERROR, rpc_id, [])
response_text, error = await make_rpc_request(
client, auth, method, params, source_path=f"/notebook/{notebook_id}"
)
if error is not None or response_text is None:
result.error = error if error is not None else "Empty response from server"
return result

try:
data = decode_response(response_text, rpc_id, allow_null=True, strict_null_status=True)
decode_toolset_preferences(data)
except RPCError as exc:
result.found_ids = exc.found_ids or []
if (
isinstance(exc, UnknownRPCMethodError)
and result.found_ids
and rpc_id not in result.found_ids
):
result.status = CheckStatus.MISMATCH
result.error = f"Expected '{rpc_id}' not in response"
elif exc.rpc_code == GrpcStatusCode.RESOURCE_EXHAUSTED:
result.error = "RPC error: RESOURCE_EXHAUSTED"
else:
result.error = f"Parse error: {type(exc).__name__}"
else:
result.status = CheckStatus.OK
result.found_ids = [rpc_id]
return result


@trace_probe()
async def check_method(
client: httpx.AsyncClient,
Expand Down Expand Up @@ -942,6 +987,9 @@ async def check_method(
error="No test parameters available",
)

if method == RPCMethod.LIST_TOOLSET_PREFERENCES and notebook_id is not None:
return await _check_toolset_preferences(client, auth, notebook_id, params)

# Make the call
found_ids, error = await make_rpc_call(client, auth, method, params)

Expand Down
94 changes: 49 additions & 45 deletions src/notebooklm/_web/wire/decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from ...exceptions import (
AuthError,
ClientError,
DecodingError,
NetworkError,
RateLimitError,
RPCError,
Expand Down Expand Up @@ -548,17 +549,16 @@ def _extract_status_code(error_info: Any) -> tuple[int, str] | None:
return code, _GRPC_STATUS_MESSAGES[code]


def _null_result_payloads(chunks: list[Any], rpc_id: str) -> list[Any]:
def _null_result_payloads(
chunks: list[Any], rpc_id: str, *, include_encoded_null: bool = False
) -> list[Any]:
"""Every non-null index-5 payload on a null-result frame for ``rpc_id``.

Returns ALL of them rather than the first: in ``rt=c`` streamed mode the
backend can emit more than one frame per rpc id (``extract_rpc_result``
handles the same case), so stopping at the first would let a placeholder
frame's unclassifiable payload hide a real status on a later frame — which
is what the pre-#2188 loop avoided by continuing until a code parsed.

Shared walk behind :func:`_find_wrb_status` and
:func:`_has_null_result_payload` so the two agree on which frames they mean.
"""
source = "decoder._null_result_payloads"
payloads: list[Any] = []
Expand All @@ -578,41 +578,35 @@ def _null_result_payloads(chunks: list[Any], rpc_id: str) -> list[Any]:
continue
result_data = safe_index(item, 2, method_id=rpc_id, source=source)
error_info = safe_index(item, 5, method_id=rpc_id, source=source)
if result_data is not None or error_info is None:
if error_info is None:
continue
payloads.append(error_info)
if result_data is None or (
include_encoded_null
and isinstance(result_data, str)
and result_data.strip(" \t\n\r") == "null"
):
payloads.append(error_info)
return payloads


def _has_null_result_payload(chunks: list[Any], rpc_id: str) -> bool:
"""Whether the server attached anything at index 5 of a null-result frame.

Distinguishes "no payload at all" (a genuinely empty result — nothing to
report) from "a payload is there". The only caller consults this AFTER
:func:`_find_wrb_status` has already returned ``None``, so at that point
"a payload is there" means "there, and this decoder could not name it" —
which is drift worth logging. Re-checking classifiability here would be
dead logic, so it is not re-checked.
"""
return bool(_null_result_payloads(chunks, rpc_id))


def _find_wrb_status(chunks: list[Any], rpc_id: str) -> tuple[int, str, Any] | None:
"""Locate the status code at index 5 of a null-result frame for ``rpc_id``.

Used by ``decode_response`` to enrich the null-result error message when
the server explicitly flagged the RPC with a status code.
def _find_wrb_status(
payloads: list[Any], *, prefer_rejection: bool = False
) -> tuple[int, str, Any] | None:
"""Find the first recognized status, optionally preferring a rejection.

Returns ``(code, label, payload)`` where ``payload`` is the raw
``google.rpc.Status`` array, so the caller can also read the server's
``message`` field without re-walking the chunks.
"""
for payload in _null_result_payloads(chunks, rpc_id):
success = None
for payload in payloads:
status = _extract_status_code(payload)
if status is not None:
code, label = status
return code, label, payload
return None
if not prefer_rejection or code != GrpcStatusCode.OK:
return code, label, payload
success = code, label, payload
return success


def _contains_user_displayable_error(obj: Any, max_depth: int = 20) -> bool:
Expand Down Expand Up @@ -855,6 +849,7 @@ def decode_response(
allow_null: bool = False,
*,
raise_on_null_status: bool = False,
strict_null_status: bool = False,
) -> Any:
"""
Complete decode pipeline: strip prefix -> parse chunks -> extract result.
Expand All @@ -873,6 +868,10 @@ def decode_response(
opt-in per call site rather than a blanket change because three RPCs
are RECORDED doing exactly that on flows the client reports as
successful — see ``_RPCS_OBSERVED_SWALLOWING_A_STATUS`` (#2188).
strict_null_status: Require every status attached to a null frame for
this RPC to be recognized and OK, including JSON-encoded nulls and
null frames alongside a populated result. Implies ``raise_on_null_status``.
A status-free null still follows ``allow_null`` if there is no result.

Returns:
Decoded result data
Expand Down Expand Up @@ -912,7 +911,7 @@ def decode_response(
e.raw_response = _truncate_response_preview(response_preview)
raise

if result is None:
if result is None or strict_null_status:
# An *absent* RPC ID is categorically different from a *present-but-null*
# result. The drift detector — the strongest signal that Google changed a
# method ID or served an anti-bot/redirect wall instead of real RPC data —
Expand Down Expand Up @@ -941,10 +940,8 @@ def decode_response(
raw_response=response_preview,
)

# RPC ID was found but extract_rpc_result returned None.
# This means wrb.fr had null result_data without UserDisplayableError.
# Enrich the message if the server attached a bare status code at
# index 5 (issues #114 / #294 showed GET_NOTEBOOK returning [5]).
# Check statuses from matching null frames. In strict mode these can
# report a failure even when another frame supplied usable data.
#
# The obfuscated ``rpc_id`` (the #1 breakage class per CLAUDE.md — it
# changes whenever Google re-obfuscates a method) and the raw numeric
Expand All @@ -953,18 +950,17 @@ def decode_response(
# the exception ATTRIBUTES (``method_id`` / ``rpc_code`` / ``found_ids``)
# for logging and debugging, but keep the human-readable message free of
# them — only the stable gRPC status label is user-facing.
status = _find_wrb_status(chunks, rpc_id)

# ``rpc_id`` is present in ``found_ids`` but ``extract_rpc_result``
# returned None — the requested frame carried a genuinely null payload.
# Honor ``allow_null`` and return None for callers that opt in, EXCEPT
# when the caller asked for strictness and the server tagged that null
# with a non-OK status. A status-tagged null is the server saying "no",
# not an empty-but-fine payload; swallowing it made ``generate_*``
# report "<type> generation is unavailable" for a live INVALID_ARGUMENT
# rejection (#2188).
status_payloads = _null_result_payloads(
chunks, rpc_id, include_encoded_null=strict_null_status
)
status = _find_wrb_status(status_payloads, prefer_rejection=strict_null_status)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

# Preserve populated results and honor ``allow_null`` for empty results
# unless the caller opted in to checking a rejection status (#2188).
rejected = status is not None and status[0] != GrpcStatusCode.OK
if allow_null and not (raise_on_null_status and rejected):
if (allow_null or result is not None) and not (
(raise_on_null_status or strict_null_status) and rejected
):
if rejected and status is not None:
# Still swallowed, but no longer without trace. DEBUG, not
# WARNING: three RPCs are recorded doing this on flows the
Expand All @@ -980,7 +976,15 @@ def decode_response(
rpc_id,
status[1],
)
elif raise_on_null_status and _has_null_result_payload(chunks, rpc_id):
elif strict_null_status:
if any(_extract_status_code(payload) is None for payload in status_payloads):
raise DecodingError(
"Null RPC result carried an unrecognized status.",
method_id=rpc_id,
found_ids=found_ids,
raw_response=response_preview,
)
elif raise_on_null_status and status_payloads:
# The caller asked for strictness and the server DID attach
# something at index 5, but not a status this decoder can name —
# a code outside 0-16, a bool, a string, a nested ``[[3]]``. That
Expand All @@ -994,7 +998,7 @@ def decode_response(
rpc_id,
rpc_id,
)
return None
return result

if status is not None:
code, label, payload = status
Expand Down
Loading
Loading