From d97a2a489505e7f5b3b55dfecdd7026bbdc79e4d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 4 Sep 2026 21:10:47 +0200 Subject: [PATCH 1/2] feat(feed): load For You pages through one composite document query One composite `getDocuments` request now answers a For You page and everything a card needs to render it, under a single merged proof: the posts, their like/repost/reply/quote counts, the posts they quote, the authors' profiles and DPNS names, and (logged in) the viewer's own likes, reposts and bookmarks on the page. The SDK derives every sub-query from the proven page, so the responding node cannot steer any of it. About ten round trips per page become one. - lib/feed/composite-feed-page.ts builds the query from the contract topology, decodes the result into the page plus a `PreloadedEnrichment`, seeds the DPNS and profile caches, attaches quoted posts, and reports `null` when the surface is unavailable (an evo-sdk without `documents.composite`, a pre-v6 contract, or a recent failure with a one-minute backoff), so the legacy loaders keep working unchanged. - lib/feed/load-for-you-feed.ts tries the composite page first and translates the feed's id cursors into the range clause the composite surface paginates with. - useProgressiveEnrichment accepts the preloaded slices, merges them at once and only queries what they did not cover; PostCard and the per-card fallbacks are untouched. - dpnsService.seedUsernames and unifiedProfileService.seedProfileDocuments let proven lookups warm the batch resolvers (with a short negative cache for proven absences). Needs an evo-sdk release carrying the composite surface (dashpay/platform#4601 and its stack) and nodes serving it (dashpay/platform#4599); until then the capability check keeps this change inert. Co-Authored-By: Claude Fable 5.1 --- hooks/use-feed-data.ts | 8 +- hooks/use-progressive-enrichment.ts | 61 +++- lib/feed/composite-feed-page.ts | 448 ++++++++++++++++++++++++ lib/feed/load-for-you-feed.ts | 110 ++++-- lib/services/dpns-service.ts | 21 ++ lib/services/unified-profile-service.ts | 42 +++ 6 files changed, 651 insertions(+), 39 deletions(-) create mode 100644 lib/feed/composite-feed-page.ts diff --git a/hooks/use-feed-data.ts b/hooks/use-feed-data.ts index bb204e4e..bd7eeee6 100644 --- a/hooks/use-feed-data.ts +++ b/hooks/use-feed-data.ts @@ -4,7 +4,7 @@ import { useAuth } from '@/contexts/auth-context'; import { useAsyncState } from '@/components/ui/loading-state'; import { Post } from '@/lib/types'; import { cacheManager } from '@/lib/cache-manager'; -import { useProgressiveEnrichment } from '@/hooks/use-progressive-enrichment'; +import { useProgressiveEnrichment, type PreloadedEnrichment } from '@/hooks/use-progressive-enrichment'; import { enrichPostsWithRepostsAndQuotes } from '@/lib/feed/enrich-posts'; import { loadFollowingFeed, type FollowingFeedWindow } from '@/lib/feed/load-following-feed'; import { loadForYouFeed } from '@/lib/feed/load-for-you-feed'; @@ -287,6 +287,8 @@ export function useFeedData({ activeTab, feedLanguage }: UseFeedDataOptions): Us } let posts: Post[] = []; + // Enrichment that arrived with the posts (composite For You pages). + let forYouPreloaded: PreloadedEnrichment | undefined; if (activeTab === 'following' && user?.identityId) { let followingPosts: Post[] = []; @@ -321,6 +323,7 @@ export function useFeedData({ activeTab, feedLanguage }: UseFeedDataOptions): Us const forYouResult = await loadForYouFeed({ startAfter: pagination?.startAfter, feedLanguage, + currentUserId: user?.identityId, setData, setHasMore, setLastPostId, @@ -328,6 +331,7 @@ export function useFeedData({ activeTab, feedLanguage }: UseFeedDataOptions): Us }); posts = forYouResult.posts; + forYouPreloaded = forYouResult.preloaded; if (posts.length === 0) { logger.debug('Feed: No posts found on platform'); @@ -367,7 +371,7 @@ export function useFeedData({ activeTab, feedLanguage }: UseFeedDataOptions): Us } if (activeTab !== 'following') { - enrichProgressively(sortedPosts); + enrichProgressively(sortedPosts, forYouPreloaded); } if (!isPaginating && sortedPosts.length > 0) { diff --git a/hooks/use-progressive-enrichment.ts b/hooks/use-progressive-enrichment.ts index 1ff44e7a..c21d6ae1 100644 --- a/hooks/use-progressive-enrichment.ts +++ b/hooks/use-progressive-enrichment.ts @@ -34,6 +34,21 @@ export interface ReplyToData { authorUsername: string | null // null = no DPNS } +/** + * Enrichment slices that arrived with the posts themselves (a composite + * feed page proves the counts, profiles, names and the viewer's marks + * under the same root as the posts). Keys present here are merged into + * the state at once and excluded from the follow-up queries; keys absent + * are fetched as before, so partial preloads degrade gracefully. + */ +export interface PreloadedEnrichment { + usernames?: Map + profiles?: Map + avatars?: Map + stats?: Map + interactions?: Map +} + export interface EnrichmentState { // Author data keyed by authorId usernames: Map // authorId → DPNS username (null = no DPNS) @@ -70,7 +85,7 @@ interface UseProgressiveEnrichmentOptions { } interface UseProgressiveEnrichmentResult { - enrichProgressively: (posts: Post[]) => void + enrichProgressively: (posts: Post[], preloaded?: PreloadedEnrichment) => void enrichmentState: EnrichmentState reset: () => void getPostEnrichment: (post: Post) => { @@ -119,7 +134,7 @@ export function useProgressiveEnrichment( * Start progressive enrichment for the given posts. * Non-blocking - returns immediately and updates state as data loads. */ - const enrichProgressively = useCallback((posts: Post[]) => { + const enrichProgressively = useCallback((posts: Post[], preloaded?: PreloadedEnrichment) => { if (posts.length === 0) return // Increment request ID to invalidate any in-flight requests @@ -150,9 +165,6 @@ export function useProgressiveEnrichment( new Set([...authorIds, ...posts.map(p => p.quotedPost?.author.id).filter((id): id is string => !!id)]) ) - // Set loading phase - setEnrichmentState(prev => ({ ...prev, phase: 'loading' })) - // Helper to merge Maps (TypeScript-compatible without downlevelIteration) const mergeMaps = (prev: Map, next: Map): Map => { const merged = new Map(prev) @@ -160,11 +172,32 @@ export function useProgressiveEnrichment( return merged } + // Set loading phase, merging whatever arrived with the posts + setEnrichmentState(prev => ({ + ...prev, + phase: 'loading', + usernames: preloaded?.usernames ? mergeMaps(prev.usernames, preloaded.usernames) : prev.usernames, + profiles: preloaded?.profiles ? mergeMaps(prev.profiles, preloaded.profiles) : prev.profiles, + avatars: preloaded?.avatars ? mergeMaps(prev.avatars, preloaded.avatars) : prev.avatars, + stats: preloaded?.stats ? mergeMaps(prev.stats, preloaded.stats) : prev.stats, + interactions: preloaded?.interactions ? mergeMaps(prev.interactions, preloaded.interactions) : prev.interactions, + })) + + // Only what the preload did not cover goes to the network + const notIn = (map: Map | undefined) => (id: string) => !map?.has(id) + const usernameAuthorIds = authorIds.filter(notIn(preloaded?.usernames)) + const profileAuthorIds = authorIds.filter(notIn(preloaded?.profiles)) + const avatarAuthorIds = authorIds.filter(notIn(preloaded?.avatars)) + const statsTargets = targets.filter(target => notIn(preloaded?.stats)(target.id)) + const interactionTargets = targets.filter(target => notIn(preloaded?.interactions)(target.id)) + // Store promises so we can reuse them for completion tracking // This prevents duplicate queries that were happening before // Priority 1: DPNS usernames (most visible - author identity) - const usernamePromise = dpnsService.resolveUsernamesBatch(authorIds) + const usernamePromise = usernameAuthorIds.length > 0 + ? dpnsService.resolveUsernamesBatch(usernameAuthorIds) + : Promise.resolve(new Map()) usernamePromise.then(usernames => { if (!isValid()) return setEnrichmentState(prev => ({ @@ -174,7 +207,9 @@ export function useProgressiveEnrichment( }).catch(err => logger.error('Progressive enrichment: usernames failed', err)) // Priority 1: Profiles (display names) - const profilePromise = unifiedProfileService.getProfilesByIdentityIds(authorIds) + const profilePromise = profileAuthorIds.length > 0 + ? unifiedProfileService.getProfilesByIdentityIds(profileAuthorIds) + : Promise.resolve([] as Awaited>) // Seed queried authors that have no entry yet, so consumers can // distinguish "loaded, no profile" (empty entry) from "still loading" // (no entry) without clobbering profiles from earlier batches @@ -215,7 +250,9 @@ export function useProgressiveEnrichment( }) // Priority 2: Avatars - const avatarPromise = unifiedProfileService.getAvatarUrlsBatch(authorIds) + const avatarPromise = avatarAuthorIds.length > 0 + ? unifiedProfileService.getAvatarUrlsBatch(avatarAuthorIds) + : Promise.resolve(new Map()) avatarPromise.then(avatars => { if (!isValid()) return setEnrichmentState(prev => ({ @@ -225,7 +262,9 @@ export function useProgressiveEnrichment( }).catch(err => logger.error('Progressive enrichment: avatars failed', err)) // Priority 3: Stats - const statsPromise = postService.getBatchPostStats(targets) + const statsPromise = statsTargets.length > 0 + ? postService.getBatchPostStats(statsTargets) + : Promise.resolve(new Map()) statsPromise.then(stats => { if (!isValid()) return setEnrichmentState(prev => ({ @@ -235,8 +274,8 @@ export function useProgressiveEnrichment( }).catch(err => logger.error('Progressive enrichment: stats failed', err)) // Priority 4: User interactions (only if logged in) - const interactionsPromise = currentUserId - ? postService.getBatchUserInteractions(targets) + const interactionsPromise = currentUserId && interactionTargets.length > 0 + ? postService.getBatchUserInteractions(interactionTargets) : Promise.resolve(new Map()) if (currentUserId) { diff --git a/lib/feed/composite-feed-page.ts b/lib/feed/composite-feed-page.ts new file mode 100644 index 00000000..af4c92a4 --- /dev/null +++ b/lib/feed/composite-feed-page.ts @@ -0,0 +1,448 @@ +import type { EvoSDK } from '@dashevo/evo-sdk'; +import { logger } from '@/lib/logger'; +import { + DPNS_CONTRACT_ID, + DPNS_DOCUMENT_TYPE, + YAPPR_CONTRACT_ID, + YAPPR_PROFILE_CONTRACT_ID, +} from '@/lib/constants'; +import { + bookmarkIndexFor, + likeIndexFor, + quoteFieldFor, + referencesAreEnforced, + replyCountFieldFor, + repostIndexFor, +} from '@/lib/contract-topology'; +import type { + PostStats, + PreloadedEnrichment, + ProfileData, + UserInteractions, +} from '@/hooks/use-progressive-enrichment'; +import { Post } from '@/lib/types'; +import { getEvoSdk } from '@/lib/services/evo-sdk-service'; +import { dpnsService } from '@/lib/services/dpns-service'; +import { resolvePostAuthorsBatch } from '@/lib/services/post-enrichment-helpers'; +import { documentToPlainObject, identifierToBase58 } from '@/lib/services/sdk-helpers'; +import { unifiedProfileService } from '@/lib/services/unified-profile-service'; +import { getPrimaryUsername } from '@/lib/utils/username'; +import { transformRawPost } from './transform-raw-post'; + +/** + * One For You feed page as ONE composite document query. + * + * The node answers the page and everything a card needs to render it + * under a single merged proof: the four engagement counts, the posts the + * page quotes, the authors' profiles and DPNS names, and (logged in) the + * viewer's own likes, reposts and bookmarks on the page. The SDK derives + * every sub-query from the PROVEN page, so nothing here can be steered by + * the responding node. Compared with the legacy loaders this replaces + * about ten round trips per page with one. + * + * The composite surface needs an evo-sdk that exposes + * `documents.composite` and a v6 (refersTo-enforced) contract; where either + * is missing this module reports `null` and the caller keeps the legacy + * path. Nothing downstream changes shape: the result is the same raw page + * plus a `PreloadedEnrichment` the progressive-enrichment hook merges and + * then skips over, so `PostCard` and the per-card fallbacks are untouched. + */ + +// ---- Wire types (mirror wasm-sdk's `CompositeDocumentsQuery` / `Result`) ---- + +type WhereClause = [string, string, unknown]; +type OrderByClause = [string, 'asc' | 'desc']; + +interface CompositeBind { + source?: 'page' | number; + sourceProperty: string; + field: string; +} + +interface CompositeSubQuery { + dataContractId?: string; + documentType: string; + kind?: 'documents' | 'counts'; + where?: WhereClause[]; + orderBy?: OrderByClause[]; + limit?: number; + bind?: CompositeBind; +} + +interface CompositeDocumentsQuery { + dataContractId: string; + documentType: string; + where?: WhereClause[]; + orderBy?: OrderByClause[]; + limit: number; + subQueries: CompositeSubQuery[]; +} + +type CompositeSubResult = + | { kind: 'documents'; documents: unknown[] } + | { kind: 'counts'; counts: Map }; + +interface CompositeDocumentsResult { + pageDocuments: unknown[]; + subResults: CompositeSubResult[]; +} + +interface CompositeDocumentsFacade { + composite(query: CompositeDocumentsQuery): Promise; +} + +/** The evo-sdk build in use may predate the composite surface. */ +function compositeFacade(sdk: EvoSDK): CompositeDocumentsFacade | null { + const documents = sdk.documents as unknown as Partial; + return typeof documents.composite === 'function' + ? (documents as CompositeDocumentsFacade) + : null; +} + +// ---- Availability ---- + +/** At most this many sub-queries per request (the platform's `MAX_SUB_QUERIES`). */ +const MAX_SUB_QUERIES = 10; +/** DPNS `records.identity` is a non-unique index, so the lookup needs a per-identity cap. */ +const DPNS_NAMES_PER_IDENTITY = 3; +/** After a composite failure, use the legacy loaders for this long before retrying. */ +const RETRY_BACKOFF_MS = 60_000; + +let unsupportedLogged = false; +let retryAfter = 0; + +export interface CompositeFeedPageOptions { + language: string; + limit: number; + /** Continue past this `$createdAt` (exclusive); omit for the first page. */ + beforeCreatedAt?: number; + currentUserId?: string; +} + +export interface CompositeFeedPage { + /** The normalized page records, newest first, tombstones included. */ + rawPosts: Record[]; + /** The page as feed posts, tombstones dropped, quoted posts attached. */ + posts: Post[]; + /** Everything the progressive enrichment would otherwise query for this page. */ + preloaded: PreloadedEnrichment; + hasMore: boolean; +} + +/** + * Load one feed page through the composite surface, or `null` when the + * surface is unavailable (older SDK, pre-v6 contract, recent failure), in + * which case the caller falls back to the legacy per-query loaders. + */ +export async function loadCompositeFeedPage( + options: CompositeFeedPageOptions +): Promise { + if (!referencesAreEnforced()) return null; + if (Date.now() < retryAfter) return null; + + const sdk = await getEvoSdk(); + const facade = compositeFacade(sdk); + if (!facade) { + if (!unsupportedLogged) { + unsupportedLogged = true; + logger.info('Feed: this evo-sdk has no composite documents surface; using the legacy loaders'); + } + return null; + } + + const { query, slots } = buildFeedPageQuery(options); + + let result: CompositeDocumentsResult; + try { + result = await facade.composite(query); + } catch (error) { + retryAfter = Date.now() + RETRY_BACKOFF_MS; + logger.warn('Feed: composite page failed, falling back to the legacy loaders', error); + return null; + } + + return decodeFeedPage(result, slots, options); +} + +// ---- Query ---- + +interface SubQuerySlots { + likeCounts: number; + repostCounts: number; + replyCounts: number; + quoteCounts: number; + quotedPosts: number; + profiles: number; + usernames: number; + /** Anonymous only: the quoted posts' authors' profiles (bound to the join). */ + quotedAuthorProfiles: number; + /** Logged in only. */ + myLikes: number; + myReposts: number; + myBookmarks: number; +} + +function buildFeedPageQuery(options: CompositeFeedPageOptions): { + query: CompositeDocumentsQuery; + slots: SubQuerySlots; +} { + const subQueries: CompositeSubQuery[] = []; + const slot = (sub: CompositeSubQuery): number => subQueries.push(sub) - 1; + const fromPage = (sourceProperty: string, field: string): CompositeBind => ({ + source: 'page', + sourceProperty, + field, + }); + + const like = likeIndexFor('post'); + const repost = repostIndexFor('post'); + const bookmark = bookmarkIndexFor('post'); + const quoteField = quoteFieldFor('post'); + const replyCountField = replyCountFieldFor('post'); + + // Engagement counts: one grouped count per page id, each from the + // `countable` index keyed by the target id alone. + const likeCounts = slot({ documentType: like.docType, kind: 'counts', bind: fromPage('$id', like.field) }); + const repostCounts = repost + ? slot({ documentType: repost.docType, kind: 'counts', bind: fromPage('$id', repost.field) }) + : -1; + const replyCounts = slot({ documentType: 'reply', kind: 'counts', bind: fromPage('$id', replyCountField) }); + const quoteCounts = quoteField + ? slot({ documentType: 'post', kind: 'counts', bind: fromPage('$id', quoteField) }) + : -1; + + // The posts this page quotes: a by-id JOIN through `refersTo`, so a + // missing quoted post is a verification error rather than a hole. + const quotedPosts = quoteField + ? slot({ documentType: 'post', bind: fromPage(quoteField, '$id') }) + : -1; + + // Author identity, cross-contract: profiles sit on a unique `$ownerId` + // index (value-bounded, no limit), DPNS names on a non-unique one. + const profiles = slot({ + dataContractId: YAPPR_PROFILE_CONTRACT_ID, + documentType: 'profile', + bind: fromPage('$ownerId', '$ownerId'), + }); + const usernames = slot({ + dataContractId: DPNS_CONTRACT_ID, + documentType: DPNS_DOCUMENT_TYPE, + bind: fromPage('$ownerId', 'records.identity'), + limit: DPNS_NAMES_PER_IDENTITY, + }); + + let quotedAuthorProfiles = -1; + let myLikes = -1; + let myReposts = -1; + let myBookmarks = -1; + if (options.currentUserId) { + // The viewer's marks on the page: `$ownerId == me` pins the owner-first + // index, the bound post id is its terminal, so these are value-bounded. + const mine: WhereClause[] = [['$ownerId', '==', options.currentUserId]]; + myLikes = slot({ documentType: like.docType, where: mine, bind: fromPage('$id', like.field) }); + if (repost) { + myReposts = slot({ documentType: repost.docType, where: mine, bind: fromPage('$id', repost.field) }); + } + if (bookmark) { + myBookmarks = slot({ documentType: bookmark.docType, where: mine, bind: fromPage('$id', bookmark.field) }); + } + } else if (quotedPosts >= 0) { + // With the request budget free, chain the quoted posts' authors' + // profiles off the join so embedded cards need no straggler hop. + quotedAuthorProfiles = slot({ + dataContractId: YAPPR_PROFILE_CONTRACT_ID, + documentType: 'profile', + bind: { source: quotedPosts, sourceProperty: '$ownerId', field: '$ownerId' }, + }); + } + + if (subQueries.length > MAX_SUB_QUERIES) { + throw new Error(`Feed: composite page needs ${subQueries.length} sub-queries, the limit is ${MAX_SUB_QUERIES}`); + } + + const before = options.beforeCreatedAt; + const query: CompositeDocumentsQuery = { + dataContractId: YAPPR_CONTRACT_ID, + documentType: 'post', + where: [ + ['language', '==', options.language], + before !== undefined ? ['$createdAt', '<', before] : ['$createdAt', '>', 0], + ], + orderBy: [['language', 'asc'], ['$createdAt', 'desc']], + limit: options.limit, + subQueries, + }; + + return { + query, + slots: { + likeCounts, + repostCounts, + replyCounts, + quoteCounts, + quotedPosts, + profiles, + usernames, + quotedAuthorProfiles, + myLikes, + myReposts, + myBookmarks, + }, + }; +} + +// ---- Result ---- + +function documentsAt(result: CompositeDocumentsResult, index: number): Record[] { + if (index < 0) return []; + const sub = result.subResults[index]; + if (!sub || sub.kind !== 'documents') return []; + return sub.documents.map((doc) => documentToPlainObject(doc)); +} + +function countsAt(result: CompositeDocumentsResult, index: number): Map { + const counts = new Map(); + if (index < 0) return counts; + const sub = result.subResults[index]; + if (!sub || sub.kind !== 'counts') return counts; + sub.counts.forEach((count, key) => counts.set(key, Number(count))); + return counts; +} + +/** The post ids named by a set of owned documents (likes, reposts, bookmarks). */ +function targetIdsOf(records: Record[], field: string): Set { + const ids = new Set(); + for (const record of records) { + const id = identifierToBase58(record[field]); + if (id) ids.add(id); + } + return ids; +} + +function usernamesByIdentity(records: Record[], identityIds: readonly string[]): Map { + const names = new Map(); + for (const doc of records) { + const data = (doc.data || doc) as Record; + const domainRecords = data.records as Record | undefined; + const identityId = identifierToBase58(domainRecords?.identity || domainRecords?.dashUniqueIdentityId); + const label = data.label || data.normalizedLabel; + if (!identityId || !label) continue; + const parentDomain = data.normalizedParentDomainName || 'dash'; + const existing = names.get(identityId) || []; + existing.push(`${label}.${parentDomain}`); + names.set(identityId, existing); + } + const usernames = new Map(); + for (const id of identityIds) { + const candidates = names.get(id); + usernames.set(id, candidates ? getPrimaryUsername(candidates) : null); + } + return usernames; +} + +async function decodeFeedPage( + result: CompositeDocumentsResult, + slots: SubQuerySlots, + options: CompositeFeedPageOptions +): Promise { + const rawPosts = result.pageDocuments.map((doc) => documentToPlainObject(doc)); + const posts = rawPosts + .map((doc) => transformRawPost(doc)) + .filter((post) => !post.deleted); + const pageIds = rawPosts + .map((doc) => doc.$id) + .filter((id): id is string => typeof id === 'string'); + const authorIds = Array.from(new Set(posts.map((post) => post.author.id).filter(Boolean))); + + // Stats, seeded to zero for every page id: a value without a count entry + // is a proven zero. + const likes = countsAt(result, slots.likeCounts); + const reposts = countsAt(result, slots.repostCounts); + const replies = countsAt(result, slots.replyCounts); + const quotes = countsAt(result, slots.quoteCounts); + const stats = new Map(); + for (const id of pageIds) { + stats.set(id, { + likes: likes.get(id) ?? 0, + reposts: reposts.get(id) ?? 0, + replies: replies.get(id) ?? 0, + quotes: quotes.get(id) ?? 0, + views: 0, + }); + } + + // Author identity, and seed the service caches so any later lookup for + // these authors (reposter names, quoted authors, profile pages) is a hit. + const foundProfiles = unifiedProfileService.seedProfileDocuments(documentsAt(result, slots.profiles), authorIds); + const profiles = new Map(); + const avatars = new Map(); + for (const id of authorIds) { + const doc = foundProfiles.get(id); + profiles.set(id, doc ? { displayName: doc.displayName, bio: doc.bio } : {}); + avatars.set( + id, + doc ? unifiedProfileService.parseAvatarField(doc.avatar, id) : unifiedProfileService.getDefaultAvatarUrl(id) + ); + } + const usernames = usernamesByIdentity(documentsAt(result, slots.usernames), authorIds); + dpnsService.seedUsernames(usernames); + + // The viewer's marks; only meaningful when logged in. + const preloaded: PreloadedEnrichment = { usernames, profiles, avatars, stats }; + if (options.currentUserId) { + const liked = targetIdsOf(documentsAt(result, slots.myLikes), likeIndexFor('post').field); + const reposted = targetIdsOf(documentsAt(result, slots.myReposts), repostIndexFor('post')?.field ?? 'postId'); + const bookmarked = targetIdsOf(documentsAt(result, slots.myBookmarks), bookmarkIndexFor('post')?.field ?? 'postId'); + const interactions = new Map(); + for (const id of pageIds) { + interactions.set(id, { liked: liked.has(id), reposted: reposted.has(id), bookmarked: bookmarked.has(id) }); + } + preloaded.interactions = interactions; + } + + // Quoted posts, attached in place; their authors resolve through the + // (now seeded) batch resolvers, so anonymous pages take no extra hop and + // logged-in pages take at most one for the names. + const quotedPosts = documentsAt(result, slots.quotedPosts) + .map((doc) => transformRawPost(doc)) + .filter((post) => !post.deleted); + if (quotedPosts.length > 0) { + const quotedAuthorIds = Array.from(new Set(quotedPosts.map((post) => post.author.id).filter(Boolean))); + if (slots.quotedAuthorProfiles >= 0) { + unifiedProfileService.seedProfileDocuments(documentsAt(result, slots.quotedAuthorProfiles), quotedAuthorIds); + } + try { + await resolvePostAuthorsBatch(quotedPosts); + } catch (error) { + logger.warn('Feed: quoted post authors did not resolve', error); + } + const quotedById = new Map(quotedPosts.map((post) => [post.id, post])); + for (const post of posts) { + const quoted = post.quotedPostId ? quotedById.get(post.quotedPostId) : undefined; + if (quoted) post.quotedPost = quoted; + } + } + + for (const post of posts) { + const postStats = stats.get(post.id); + if (postStats) { + post.likes = postStats.likes; + post.reposts = postStats.reposts; + post.replies = postStats.replies; + post.quotes = postStats.quotes; + } + const mine = preloaded.interactions?.get(post.id); + if (mine) { + post.liked = mine.liked; + post.reposted = mine.reposted; + post.bookmarked = mine.bookmarked; + } + } + + return { + rawPosts, + posts, + preloaded, + hasMore: rawPosts.length === options.limit, + }; +} diff --git a/lib/feed/load-for-you-feed.ts b/lib/feed/load-for-you-feed.ts index 5f51501f..b63b1dbc 100644 --- a/lib/feed/load-for-you-feed.ts +++ b/lib/feed/load-for-you-feed.ts @@ -1,6 +1,8 @@ import { logger } from '@/lib/logger'; import { postService } from '@/lib/services/post-service'; import { Post } from '@/lib/types'; +import type { PreloadedEnrichment } from '@/hooks/use-progressive-enrichment'; +import { loadCompositeFeedPage } from './composite-feed-page'; import { enrichPostsWithRepostsAndQuotes } from './enrich-posts'; import { sortFeedByTimestamp } from './transform-raw-post'; @@ -18,17 +20,71 @@ function withLoadingAuthor(post: Post): Post { }; } +const PAGE_SIZE = 20; + +interface FeedPage { + posts: Post[]; + cursor: string | null; + hasMore: boolean; + preloaded?: PreloadedEnrichment; +} + +async function fetchFeedPage(options: { + startAfter?: string; + language?: string; + currentUserId?: string; +}): Promise { + const compositeOptions = { + language: options.language || 'en', + limit: PAGE_SIZE, + currentUserId: options.currentUserId, + }; + + if (!options.startAfter) { + const page = await loadCompositeFeedPage(compositeOptions); + if (page) { + const last = page.rawPosts[page.rawPosts.length - 1]; + const cursor = last ? String(last.$id) : null; + return { posts: page.posts, cursor, hasMore: page.hasMore, preloaded: page.preloaded }; + } + } + + // Composite queries have no document cursor. Use the timeline's real + // startAfter to select subsequent pages, including timestamp ties, then + // batch enrichment for those exact ids. Keep the raw cursor even when + // tombstones leave no visible cards or a document changes between reads. + const raw = (await postService.getTimeline({ + limit: PAGE_SIZE, + startAfter: options.startAfter, + language: options.language, + })).documents; + const cursor = raw.length ? raw[raw.length - 1].id : null; + const hasMore = raw.length === PAGE_SIZE; + if (options.startAfter && raw.length) { + const page = await loadCompositeFeedPage({ + ...compositeOptions, + documentIds: raw.map(post => post.id), + }); + if (page) { + return { posts: page.posts, cursor, hasMore, preloaded: page.preloaded }; + } + } + + const posts = raw.filter(post => !post.deleted).map(withLoadingAuthor); + return { posts, cursor, hasMore }; +} + export async function loadForYouFeed(options: { startAfter?: string; feedLanguage?: string; + currentUserId?: string; setData: (updater: (prev: Post[] | null) => Post[] | null) => void; setHasMore: (value: boolean) => void; setLastPostId: (id: string) => void; - enrichProgressively: (posts: Post[]) => void; -}): Promise<{ posts: Post[]; cursor: string | null; hasMore: boolean }> { + enrichProgressively: (posts: Post[], preloaded?: PreloadedEnrichment) => void; +}): Promise<{ posts: Post[]; cursor: string | null; hasMore: boolean; preloaded?: PreloadedEnrichment }> { const MIN_NON_REPLY_POSTS = 20; const MAX_FETCH_ITERATIONS = 5; - const PAGE_SIZE = 20; const currentStartAfter = options.startAfter; @@ -38,30 +94,28 @@ export async function loadForYouFeed(options: { '(iteration 1)' ); - const firstBatchRaw = (await postService.getTimeline({ - limit: PAGE_SIZE, + const firstPage = await fetchFeedPage({ startAfter: currentStartAfter, language: options.feedLanguage, - })).documents; + currentUserId: options.currentUserId, + }); - if (firstBatchRaw.length === 0) { + if (!firstPage.cursor) { logger.debug('Feed: No posts available'); options.setHasMore(false); return { posts: [], cursor: null, hasMore: false }; } - // Tombstones are dropped here, before the raw batch renders — the async - // enrichment merge falls back to the ORIGINAL post for ids missing from the - // enriched result, so filtering only inside enrichPostsWithRepostsAndQuotes - // would let deleted posts reappear. `deleted` is never set on v2. - const firstBatchPosts = firstBatchRaw.filter((post) => !post.deleted).map(withLoadingAuthor); - const firstBatchCursor = firstBatchRaw[firstBatchRaw.length - 1].id; + const firstBatchPosts = firstPage.posts; + const firstBatchCursor = firstPage.cursor; logger.debug(`Feed: First batch has ${firstBatchPosts.length} posts`); const forYouNextCursor: string | null = firstBatchCursor; - const forYouHasMore = firstBatchRaw.length === PAGE_SIZE; + const forYouHasMore = firstPage.hasMore; + // Repost attribution ("X reposted") and whatever quotes the composite page + // did not already attach (quoted replies, blog quotes). enrichPostsWithRepostsAndQuotes(firstBatchPosts) .then((enrichedPosts) => { options.setData((current) => { @@ -83,31 +137,32 @@ export async function loadForYouFeed(options: { let bgCurrentStartAfter = firstBatchCursor; let bgFetchIteration = 1; let allPostCount = firstBatchPosts.length; - let bgLastBatchSize = firstBatchRaw.length; + let bgHasMore: boolean = forYouHasMore; while ( allPostCount < MIN_NON_REPLY_POSTS && bgFetchIteration < MAX_FETCH_ITERATIONS && - bgLastBatchSize === PAGE_SIZE + bgHasMore && + bgCurrentStartAfter ) { bgFetchIteration++; logger.debug(`Feed: Loading posts starting after ${bgCurrentStartAfter} (iteration ${bgFetchIteration})`); - const bgRawPosts = (await postService.getTimeline({ - limit: PAGE_SIZE, + const bgPage = await fetchFeedPage({ startAfter: bgCurrentStartAfter, language: options.feedLanguage, - })).documents; + currentUserId: options.currentUserId, + }); - bgLastBatchSize = bgRawPosts.length; + bgHasMore = bgPage.hasMore; - if (bgRawPosts.length === 0) { + if (!bgPage.cursor) { logger.debug('Feed: No more posts available (background)'); options.setHasMore(false); break; } - const bgPosts = bgRawPosts.filter((post) => !post.deleted).map(withLoadingAuthor); + const bgPosts = bgPage.posts; enrichPostsWithRepostsAndQuotes(bgPosts) .then((enrichedPosts) => { @@ -123,7 +178,7 @@ export async function loadForYouFeed(options: { allPostCount += bgPosts.length; - bgCurrentStartAfter = bgRawPosts[bgRawPosts.length - 1].id; + bgCurrentStartAfter = bgPage.cursor; options.setData((currentItems) => { if (!currentItems) return bgPosts; @@ -136,15 +191,17 @@ export async function loadForYouFeed(options: { return allItems; }); - options.enrichProgressively(bgPosts); - options.setLastPostId(bgCurrentStartAfter); + options.enrichProgressively(bgPosts, bgPage.preloaded); + if (bgCurrentStartAfter) { + options.setLastPostId(bgCurrentStartAfter); + } if (allPostCount < MIN_NON_REPLY_POSTS && bgFetchIteration < MAX_FETCH_ITERATIONS) { logger.debug(`Feed: Only ${allPostCount} posts, fetching more... (need ${MIN_NON_REPLY_POSTS})`); } } - options.setHasMore(bgLastBatchSize === PAGE_SIZE); + options.setHasMore(bgHasMore); logger.debug(`Feed: Background fetch complete. Total posts: ${allPostCount}`); }; @@ -164,5 +221,6 @@ export async function loadForYouFeed(options: { posts: sortedPosts, cursor: forYouNextCursor, hasMore: forYouHasMore, + preloaded: firstPage.preloaded, }; } diff --git a/lib/services/dpns-service.ts b/lib/services/dpns-service.ts index 6d556c1c..a79150ee 100644 --- a/lib/services/dpns-service.ts +++ b/lib/services/dpns-service.ts @@ -45,12 +45,27 @@ class DpnsService { /** identity id -> primary username */ private reverseCache = new TtlMap(DpnsService.CACHE_TTL_MS); + /** Cache only complete DPNS lookup results; null records a proven absence. */ + private reverseMissCache = new TtlMap(5 * 60 * 1000); + + seedUsernames(usernames: ReadonlyMap): void { + usernames.forEach((username, identityId) => { + if (username) { + this._cacheEntry(username, identityId); + } else { + this.reverseCache.delete(identityId); + this.reverseMissCache.set(identityId, true); + } + }); + } + /** * Helper method to cache entries in both directions */ private _cacheEntry(username: string, identityId: string): void { this.cache.set(username.toLowerCase(), identityId); this.reverseCache.set(identityId, username); + this.reverseMissCache.delete(identityId); } /** @@ -122,6 +137,8 @@ class DpnsService { const cached = this.reverseCache.get(id); if (cached !== undefined) { results.set(id, cached); + } else if (this.reverseMissCache.has(id)) { + results.set(id, null); } else { uncachedIds.push(id); } @@ -187,6 +204,7 @@ class DpnsService { // Check cache const cached = this.reverseCache.get(identityId); if (cached !== undefined) return cached; + if (this.reverseMissCache.has(identityId)) return null; // Get all usernames for this identity and pick the primary one const allUsernames = await this.getAllUsernames(identityId); @@ -582,10 +600,12 @@ class DpnsService { } if (identityId) { this.reverseCache.delete(identityId); + this.reverseMissCache.delete(identityId); } if (!username && !identityId) { this.cache.clear(); this.reverseCache.clear(); + this.reverseMissCache.clear(); } } @@ -595,6 +615,7 @@ class DpnsService { cleanupCache(): void { this.cache.prune(); this.reverseCache.prune(); + this.reverseMissCache.prune(); } } diff --git a/lib/services/unified-profile-service.ts b/lib/services/unified-profile-service.ts index 39f2af4c..54965c6d 100644 --- a/lib/services/unified-profile-service.ts +++ b/lib/services/unified-profile-service.ts @@ -217,6 +217,48 @@ class UnifiedProfileService extends BaseDocumentService { return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); } + // ==================== Seeding from external lookups ==================== + + /** + * Seed the profile caches from documents fetched elsewhere (a composite + * feed page carries the authors' profiles under the same proof as the + * posts). Every id in `queriedOwnerIds` without a document is a PROVEN + * absence and is negative-cached exactly as a batch miss would be, so + * the DataLoader answers later lookups from cache. Returns the found + * documents keyed by owner, with their avatar URLs already cached. + */ + seedProfileDocuments( + records: readonly Record[], + queriedOwnerIds: readonly string[] + ): Map { + const found = new Map(); + for (const record of records) { + const profileDoc = this.extractDocumentData(record); + if (!profileDoc.$ownerId) continue; + found.set(profileDoc.$ownerId, profileDoc); + cacheManager.set(this.RAW_PROFILE_CACHE, profileDoc.$ownerId, profileDoc, { + ttl: 300000, + tags: ['profile', `user:${profileDoc.$ownerId}`] + }); + cacheManager.set(this.AVATAR_CACHE, profileDoc.$ownerId, this.parseAvatarField(profileDoc.avatar, profileDoc.$ownerId), { + ttl: 300000, + tags: ['avatar', `user:${profileDoc.$ownerId}`] + }); + } + for (const ownerId of queriedOwnerIds) { + if (found.has(ownerId)) continue; + cacheManager.set(this.MISSING_PROFILE_CACHE, ownerId, true, { + ttl: 60000, + tags: ['profile', `user:${ownerId}`] + }); + cacheManager.set(this.AVATAR_CACHE, ownerId, this.getDefaultAvatarUrl(ownerId), { + ttl: 300000, + tags: ['avatar', `user:${ownerId}`] + }); + } + return found; + } + // ==================== Batching for Profile Documents ==================== /** From f8ff5c2ff5ef0033e324a2d6d6be188a93ad58ae Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 8 Sep 2026 16:04:08 +0700 Subject: [PATCH 2/2] fix(feed): preserve composite pagination and complete DPNS results --- lib/feed/composite-feed-page.test.ts | 145 ++++++++++++++++++++++++ lib/feed/composite-feed-page.ts | 79 +++++++------ lib/feed/load-for-you-feed.test.ts | 59 ++++++++++ lib/services/dpns-service.test.ts | 58 ++++++++++ lib/services/dpns-service.ts | 35 +++++- lib/services/unified-profile-service.ts | 2 + 6 files changed, 337 insertions(+), 41 deletions(-) create mode 100644 lib/feed/composite-feed-page.test.ts create mode 100644 lib/feed/load-for-you-feed.test.ts create mode 100644 lib/services/dpns-service.test.ts diff --git a/lib/feed/composite-feed-page.test.ts b/lib/feed/composite-feed-page.test.ts new file mode 100644 index 00000000..345473a5 --- /dev/null +++ b/lib/feed/composite-feed-page.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Exercise the loader and decoders with an in-memory SDK boundary. No SDK +// initialization, browser storage or network calls are made by these tests. +const mocks = vi.hoisted(() => ({ + composite: vi.fn(), + getEvoSdk: vi.fn(), + seedUsernames: vi.fn(), + seedProfiles: vi.fn(() => new Map()), + resolveAuthors: vi.fn(), +})); +vi.mock('@/lib/services/evo-sdk-service', () => ({ getEvoSdk: mocks.getEvoSdk })); +vi.mock('@/lib/services/dpns-service', () => ({ dpnsService: { seedUsernames: mocks.seedUsernames } })); +vi.mock('@/lib/services/unified-profile-service', () => ({ + unifiedProfileService: { + seedProfileDocuments: mocks.seedProfiles, + getDefaultAvatarUrl: (id: string) => `avatar:${id}`, + }, +})); +vi.mock('@/lib/services/post-enrichment-helpers', () => ({ resolvePostAuthorsBatch: mocks.resolveAuthors })); + +const ownerIds = ['111111111', '222222222', '333333333', '444444444']; +const docs = ownerIds.map((ownerId, i) => ({ + $id: `post0000${i}`, $ownerId: ownerId, $createdAt: 1000, content: 'test', language: 'en', +})); +const names = ownerIds.map((id, i) => ({ records: { identity: id }, label: `name${i}` })); +function result(dpns = names, page = docs) { + return { + pageDocuments: page, + subResults: [ + ...Array.from({ length: 4 }, () => ({ kind: 'counts', counts: new Map() })), + { kind: 'documents', documents: [] }, + { kind: 'documents', documents: [] }, + { kind: 'documents', documents: dpns }, + { kind: 'documents', documents: [] }, + ], + }; +} + +beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + vi.stubEnv('NEXT_PUBLIC_CONTRACT_TOPOLOGY', 'v6'); + mocks.getEvoSdk.mockResolvedValue({ documents: { composite: mocks.composite } }); + mocks.composite.mockResolvedValue(result()); +}); +afterEach(() => vi.unstubAllEnvs()); + +describe('composite feed page', () => { + it('should preload all four named authors using a total budget of 100', async () => { + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20 }); + expect(page?.preloaded.usernames?.size).toBe(4); + expect(page?.preloaded.usernames?.get(ownerIds[3])).toBe('name3.dash'); + const query = mocks.composite.mock.calls[0][0]; + expect(query.subQueries).toHaveLength(8); + expect(query.subQueries[6].limit).toBe(100); + }); + + it('should not preload partial primary names or negative cache entries at the cap', async () => { + mocks.composite.mockResolvedValue(result(Array.from({ length: 100 }, (_, i) => ({ + records: { identity: ownerIds[0] }, label: `alias${i}`, + })))); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20 }); + expect(page?.preloaded.usernames?.size).toBe(0); + expect(mocks.seedUsernames).toHaveBeenCalledWith(new Map()); + }); + + it('should seed absence only when the DPNS result leaves capacity for every author', async () => { + mocks.composite.mockResolvedValue(result(names.slice(0, 3))); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20 }); + expect(page?.preloaded.usernames?.get(ownerIds[3])).toBeNull(); + }); + + it('should leave capacity for empty identity branches before trusting completeness', async () => { + mocks.composite.mockResolvedValue(result(Array.from({ length: 97 }, (_, i) => ({ + records: { identity: ownerIds[0] }, label: `alias${i}`, + })))); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20 }); + expect(page?.preloaded.usernames?.size).toBe(0); + }); + + it('should query exact cursor-selected ids and restore their timeline order', async () => { + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const ids = docs.map(doc => doc.$id).reverse(); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20, documentIds: ids }); + expect(mocks.composite.mock.calls[0][0].where).toEqual([['$id', 'in', ids]]); + expect(mocks.composite.mock.calls[0][0].orderBy).toBeUndefined(); + expect(page?.posts.map(post => post.id)).toEqual(ids); + }); + + it('should keep tombstones in the raw cursor page and remove them from cards', async () => { + mocks.composite.mockResolvedValue(result(names, docs.map((doc, i) => ({ ...doc, deleted: i === 3 })))); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 4 }); + expect(page?.rawPosts).toHaveLength(4); + expect(page?.posts).toHaveLength(3); + expect(page?.hasMore).toBe(true); + }); + + it('should fit viewer interactions within ten subqueries', async () => { + const response = result(); + response.subResults.push({ kind: 'documents', documents: [] }, { kind: 'documents', documents: [] }); + mocks.composite.mockResolvedValue(response); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + const page = await loadCompositeFeedPage({ language: 'en', limit: 20, currentUserId: ownerIds[0] }); + expect(mocks.composite.mock.calls[0][0].subQueries).toHaveLength(10); + expect(page?.preloaded.interactions?.size).toBe(4); + }); + + it('should fall back without requesting composite documents on an old SDK', async () => { + mocks.getEvoSdk.mockResolvedValue({ documents: {} }); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + expect(await loadCompositeFeedPage({ language: 'en', limit: 20 })).toBeNull(); + expect(mocks.composite).not.toHaveBeenCalled(); + }); + + it('should fall back on unsupported nodes and back off for a minute', async () => { + mocks.composite.mockRejectedValue(new Error('unsupported sub_queries')); + const now = Date.now(); + const time = vi.spyOn(Date, 'now').mockReturnValue(now); + try { + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + expect(await loadCompositeFeedPage({ language: 'en', limit: 20 })).toBeNull(); + expect(await loadCompositeFeedPage({ language: 'en', limit: 20 })).toBeNull(); + expect(mocks.composite).toHaveBeenCalledTimes(1); + time.mockReturnValue(now + 60_001); + mocks.composite.mockResolvedValue(result()); + expect(await loadCompositeFeedPage({ language: 'en', limit: 20 })).not.toBeNull(); + } finally { + time.mockRestore(); + } + }); + + it('should fall back without seeding false absences on an incomplete response', async () => { + mocks.composite.mockResolvedValue({ pageDocuments: docs, subResults: [] }); + const { loadCompositeFeedPage } = await import('./composite-feed-page'); + expect(await loadCompositeFeedPage({ language: 'en', limit: 20 })).toBeNull(); + expect(mocks.seedUsernames).not.toHaveBeenCalled(); + expect(mocks.seedProfiles).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/feed/composite-feed-page.ts b/lib/feed/composite-feed-page.ts index af4c92a4..a1d5a74d 100644 --- a/lib/feed/composite-feed-page.ts +++ b/lib/feed/composite-feed-page.ts @@ -30,22 +30,15 @@ import { getPrimaryUsername } from '@/lib/utils/username'; import { transformRawPost } from './transform-raw-post'; /** - * One For You feed page as ONE composite document query. + * Batch a feed page, engagement counts, quoted posts, author profiles/names + * and viewer interactions through the composite documents surface. Initial + * pages take one document request; subsequent pages first use the timeline's + * cursor query, then fetch those exact ids with their enrichment here. * - * The node answers the page and everything a card needs to render it - * under a single merged proof: the four engagement counts, the posts the - * page quotes, the authors' profiles and DPNS names, and (logged in) the - * viewer's own likes, reposts and bookmarks on the page. The SDK derives - * every sub-query from the PROVEN page, so nothing here can be steered by - * the responding node. Compared with the legacy loaders this replaces - * about ten round trips per page with one. - * - * The composite surface needs an evo-sdk that exposes - * `documents.composite` and a v6 (refersTo-enforced) contract; where either - * is missing this module reports `null` and the caller keeps the legacy - * path. Nothing downstream changes shape: the result is the same raw page - * plus a `PreloadedEnrichment` the progressive-enrichment hook merges and - * then skips over, so `PostCard` and the per-card fallbacks are untouched. + * Requires an SDK exposing documents.composite, compatible nodes and an + * enforced-reference contract. Older deployments use the ordinary loaders. + * Repost attribution, block/follow status and unseeded quoted authors still + * need separate lookups; this is not a fixed total request count for the UI. */ // ---- Wire types (mirror wasm-sdk's `CompositeDocumentsQuery` / `Result`) ---- @@ -103,8 +96,8 @@ function compositeFacade(sdk: EvoSDK): CompositeDocumentsFacade | null { /** At most this many sub-queries per request (the platform's `MAX_SUB_QUERIES`). */ const MAX_SUB_QUERIES = 10; -/** DPNS `records.identity` is a non-unique index, so the lookup needs a per-identity cap. */ -const DPNS_NAMES_PER_IDENTITY = 3; +/** Total DPNS document budget across ALL page authors, not per identity. */ +const DPNS_QUERY_LIMIT = 100; /** After a composite failure, use the legacy loaders for this long before retrying. */ const RETRY_BACKOFF_MS = 60_000; @@ -114,8 +107,8 @@ let retryAfter = 0; export interface CompositeFeedPageOptions { language: string; limit: number; - /** Continue past this `$createdAt` (exclusive); omit for the first page. */ - beforeCreatedAt?: number; + /** Exact next-page ids selected by a timeline query using startAfter. */ + documentIds?: string[]; currentUserId?: string; } @@ -152,16 +145,18 @@ export async function loadCompositeFeedPage( const { query, slots } = buildFeedPageQuery(options); - let result: CompositeDocumentsResult; try { - result = await facade.composite(query); + const result = await facade.composite(query); + if (result.subResults.length !== query.subQueries.length) { + throw new Error('Feed: incomplete composite result'); + } + return await decodeFeedPage(result, slots, options); } catch (error) { retryAfter = Date.now() + RETRY_BACKOFF_MS; logger.warn('Feed: composite page failed, falling back to the legacy loaders', error); return null; } - return decodeFeedPage(result, slots, options); } // ---- Query ---- @@ -228,7 +223,7 @@ function buildFeedPageQuery(options: CompositeFeedPageOptions): { dataContractId: DPNS_CONTRACT_ID, documentType: DPNS_DOCUMENT_TYPE, bind: fromPage('$ownerId', 'records.identity'), - limit: DPNS_NAMES_PER_IDENTITY, + limit: DPNS_QUERY_LIMIT, }); let quotedAuthorProfiles = -1; @@ -260,15 +255,13 @@ function buildFeedPageQuery(options: CompositeFeedPageOptions): { throw new Error(`Feed: composite page needs ${subQueries.length} sub-queries, the limit is ${MAX_SUB_QUERIES}`); } - const before = options.beforeCreatedAt; const query: CompositeDocumentsQuery = { dataContractId: YAPPR_CONTRACT_ID, documentType: 'post', - where: [ - ['language', '==', options.language], - before !== undefined ? ['$createdAt', '<', before] : ['$createdAt', '>', 0], - ], - orderBy: [['language', 'asc'], ['$createdAt', 'desc']], + where: options.documentIds + ? [['$id', 'in', options.documentIds]] + : [['language', '==', options.language], ['$createdAt', '>', 0]], + orderBy: options.documentIds ? undefined : [['language', 'asc'], ['$createdAt', 'desc']], limit: options.limit, subQueries, }; @@ -296,7 +289,7 @@ function buildFeedPageQuery(options: CompositeFeedPageOptions): { function documentsAt(result: CompositeDocumentsResult, index: number): Record[] { if (index < 0) return []; const sub = result.subResults[index]; - if (!sub || sub.kind !== 'documents') return []; + if (!sub || sub.kind !== 'documents') throw new Error('Feed: missing composite documents result'); return sub.documents.map((doc) => documentToPlainObject(doc)); } @@ -304,7 +297,7 @@ function countsAt(result: CompositeDocumentsResult, index: number): Map(); if (index < 0) return counts; const sub = result.subResults[index]; - if (!sub || sub.kind !== 'counts') return counts; + if (!sub || sub.kind !== 'counts') throw new Error('Feed: missing composite counts result'); sub.counts.forEach((count, key) => counts.set(key, Number(count))); return counts; } @@ -319,7 +312,13 @@ function targetIdsOf(records: Record[], field: string): Set[], identityIds: readonly string[]): Map { +function usernamesByIdentity(records: Record[], identityIds: readonly string[], pageSize: number): Map { + // At the cap, even returned authors may have unseen aliases that would + // change their primary name. Leave the entire slice to normal enrichment; + // never cache a missing name (or a partial primary name) as a proven result. + // Empty bound identity branches can also consume a slot. Reserve one per + // page document (including tombstones), conservatively covering every author. + if (records.length + pageSize >= DPNS_QUERY_LIMIT) return new Map(); const names = new Map(); for (const doc of records) { const data = (doc.data || doc) as Record; @@ -345,7 +344,15 @@ async function decodeFeedPage( slots: SubQuerySlots, options: CompositeFeedPageOptions ): Promise { - const rawPosts = result.pageDocuments.map((doc) => documentToPlainObject(doc)); + let rawPosts = result.pageDocuments.map((doc) => documentToPlainObject(doc)); + if (options.documentIds) { + // By-id queries return key order, which is different from timeline order. + const byId = new Map(rawPosts.map(doc => [doc.$id, doc])); + rawPosts = options.documentIds.flatMap(id => { + const doc = byId.get(id); + return doc ? [doc] : []; + }); + } const posts = rawPosts .map((doc) => transformRawPost(doc)) .filter((post) => !post.deleted); @@ -384,7 +391,7 @@ async function decodeFeedPage( doc ? unifiedProfileService.parseAvatarField(doc.avatar, id) : unifiedProfileService.getDefaultAvatarUrl(id) ); } - const usernames = usernamesByIdentity(documentsAt(result, slots.usernames), authorIds); + const usernames = usernamesByIdentity(documentsAt(result, slots.usernames), authorIds, rawPosts.length); dpnsService.seedUsernames(usernames); // The viewer's marks; only meaningful when logged in. @@ -401,8 +408,8 @@ async function decodeFeedPage( } // Quoted posts, attached in place; their authors resolve through the - // (now seeded) batch resolvers, so anonymous pages take no extra hop and - // logged-in pages take at most one for the names. + // (now seeded) batch resolvers. Distinct quoted authors still need DPNS + // lookups, and logged-in pages also need their profiles. const quotedPosts = documentsAt(result, slots.quotedPosts) .map((doc) => transformRawPost(doc)) .filter((post) => !post.deleted); diff --git a/lib/feed/load-for-you-feed.test.ts b/lib/feed/load-for-you-feed.test.ts new file mode 100644 index 00000000..fe21588f --- /dev/null +++ b/lib/feed/load-for-you-feed.test.ts @@ -0,0 +1,59 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { transformRawPost } from './transform-raw-post'; +import type { CompositeFeedPage } from './composite-feed-page'; +import type { Post } from '@/lib/types'; + +const mocks = vi.hoisted(() => ({ composite: vi.fn(), timeline: vi.fn() })); +vi.mock('./composite-feed-page', () => ({ loadCompositeFeedPage: mocks.composite })); +vi.mock('@/lib/services/post-service', () => ({ postService: { getTimeline: mocks.timeline } })); +vi.mock('./enrich-posts', () => ({ enrichPostsWithRepostsAndQuotes: async (posts: Post[]) => posts })); +import { loadForYouFeed } from './load-for-you-feed'; + +const raw = Array.from({ length: 41 }, (_, i) => ({ + $id: `post${String(i).padStart(8, '0')}`, $ownerId: '111111111', $createdAt: 1000, content: 'test', +})); +const posts = raw.map(transformRawPost); +const callbacks = () => ({ + setData: vi.fn(), setHasMore: vi.fn(), setLastPostId: vi.fn(), enrichProgressively: vi.fn(), +}); +function compositePage(start: number, end: number): CompositeFeedPage { + return { rawPosts: raw.slice(start, end), posts: posts.slice(start, end), hasMore: end - start === 20, preloaded: {} }; +} +beforeEach(() => vi.resetAllMocks()); + +describe('For You pagination', () => { + it('should return every timestamp tie over three pages using document cursors', async () => { + mocks.composite.mockResolvedValueOnce(compositePage(0, 20)) + .mockResolvedValueOnce(compositePage(20, 40)).mockResolvedValueOnce(compositePage(40, 41)); + mocks.timeline.mockImplementation(async ({ startAfter, limit }: { startAfter: string; limit: number }) => { + const start = posts.findIndex(post => post.id === startAfter) + 1; + return { documents: posts.slice(start, start + limit) }; + }); + const first = await loadForYouFeed(callbacks()); + expect(mocks.timeline).not.toHaveBeenCalled(); + const second = await loadForYouFeed({ ...callbacks(), startAfter: first.cursor ?? undefined }); + const third = await loadForYouFeed({ ...callbacks(), startAfter: second.cursor ?? undefined }); + expect([...first.posts, ...second.posts, ...third.posts].map(post => post.id)).toEqual(posts.map(post => post.id)); + expect(mocks.timeline.mock.calls.map(call => call[0].startAfter)).toEqual([posts[19].id, posts[39].id]); + expect(mocks.composite.mock.calls[1][0].documentIds).toEqual(posts.slice(20, 40).map(post => post.id)); + expect(third.hasMore).toBe(false); + }); + + it('should reuse the cursor query when composite is unavailable', async () => { + mocks.composite.mockResolvedValue(null); + mocks.timeline.mockResolvedValue({ documents: posts.slice(20) }); + const page = await loadForYouFeed({ ...callbacks(), startAfter: posts[19].id }); + expect(mocks.timeline).toHaveBeenCalledTimes(1); + expect(page.posts[0].author.hasDpns).toBeUndefined(); + expect(page.posts.map(post => post.id)).toEqual(posts.slice(20).map(post => post.id)); + }); + + it('should preserve the raw timeline cursor when the last card is a tombstone', async () => { + mocks.timeline.mockResolvedValue({ documents: posts.slice(20, 23) }); + mocks.composite.mockResolvedValue({ ...compositePage(20, 23), posts: posts.slice(20, 22) }); + const page = await loadForYouFeed({ ...callbacks(), startAfter: posts[19].id }); + expect(page.cursor).toBe(posts[22].id); + expect(page.posts).toHaveLength(2); + expect(page.hasMore).toBe(false); + }); +}); diff --git a/lib/services/dpns-service.test.ts b/lib/services/dpns-service.test.ts new file mode 100644 index 00000000..bf33cef8 --- /dev/null +++ b/lib/services/dpns-service.test.ts @@ -0,0 +1,58 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const query = vi.hoisted(() => vi.fn()); +vi.mock('./evo-sdk-service', () => ({ getEvoSdk: async () => ({ documents: { query } }) })); +vi.mock('./signer-service', () => ({ signerService: {} })); +vi.mock('@/lib/crypto/keys', () => ({ matchIdentityKey: vi.fn() })); +import { dpnsService } from './dpns-service'; + +beforeEach(() => { + vi.useFakeTimers(); + dpnsService.clearCache(); + query.mockReset().mockResolvedValue([]); +}); +afterEach(() => vi.useRealTimers()); + +describe('DPNS composite cache seeds', () => { + it('should expire proven absences after five minutes', async () => { + dpnsService.seedUsernames(new Map([['111111111', null]])); + expect((await dpnsService.resolveUsernamesBatch(['111111111'])).get('111111111')).toBeNull(); + expect(query).not.toHaveBeenCalled(); + vi.advanceTimersByTime(300_001); + await dpnsService.resolveUsernamesBatch(['111111111']); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('should invalidate an absence when a username is registered or cache is cleared', async () => { + dpnsService.seedUsernames(new Map([['111111111', null]])); + dpnsService.clearCache(undefined, '111111111'); + await dpnsService.resolveUsernamesBatch(['111111111']); + expect(query).toHaveBeenCalledTimes(1); + }); + + it('should retry crowded batches per identity and paginate aliases before choosing primary names', async () => { + const aliases = Array.from({ length: 100 }, (_, i) => ({ + $id: String(i + 1).replace(/0/g, '1'), + records: { identity: '111111111' }, label: `longalias${i}`, + })); + query.mockResolvedValueOnce(aliases.slice(0, 99)) + .mockResolvedValueOnce(aliases) + .mockResolvedValueOnce([{ records: { identity: '111111111' }, label: 'abc' }]) + .mockResolvedValueOnce([{ records: { identity: '222222222' }, label: 'def' }]); + const names = await dpnsService.resolveUsernamesBatch(['111111111', '222222222']); + expect(names.get('111111111')).toBe('abc.dash'); + expect(names.get('222222222')).toBe('def.dash'); + expect(query.mock.calls[1][0].where).toEqual([['records.identity', '==', '111111111']]); + expect(query.mock.calls[2][0].startAfter).toBe(aliases[99].$id); + expect(query.mock.calls[3][0].where).toEqual([['records.identity', '==', '222222222']]); + }); + + it('should replace stale names with absences and absences with new names', async () => { + dpnsService.seedUsernames(new Map([['111111111', 'old.dash']])); + dpnsService.seedUsernames(new Map([['111111111', null]])); + expect(await dpnsService.resolveUsername('111111111')).toBeNull(); + dpnsService.seedUsernames(new Map([['111111111', 'new.dash']])); + expect(await dpnsService.resolveUsername('111111111')).toBe('new.dash'); + expect(query).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/services/dpns-service.ts b/lib/services/dpns-service.ts index a79150ee..4d917e2e 100644 --- a/lib/services/dpns-service.ts +++ b/lib/services/dpns-service.ts @@ -118,10 +118,9 @@ class DpnsService { * Uses 'in' operator for efficient single-query resolution * Selects the "best" username for identities with multiple names (contested first, then shortest, then alphabetically) * - * TODO: This query uses 'in' clause which doesn't support reliable pagination. - * The SDK returns incomplete results when subtrees are empty but still count against the limit. - * Once SDK provides better 'in' query support (e.g., a flag indicating result completeness), - * implement pagination here to handle cases where results exceed the limit. + * Near the shared limit, retry per identity with document cursors: empty + * identity branches can consume IN-query capacity too, so fewer than 100 + * documents does not by itself prove that the batch is complete. */ async resolveUsernamesBatch(identityIds: string[]): Promise> { const results = new Map(); @@ -160,7 +159,33 @@ class DpnsService { limit: 100 }); - const documents = extractDocuments(response); + let documents = extractDocuments(response); + if (documents.length + uncachedIds.length >= 100) { + // Discard the partial batch, including potentially incomplete alias + // sets, before choosing primary names or reporting missing authors. + documents = []; + for (const identityId of uncachedIds) { + let startAfter: string | undefined; + while (true) { + const page = extractDocuments(await sdk.documents.query({ + dataContractId: DPNS_CONTRACT_ID, + documentTypeName: DPNS_DOCUMENT_TYPE, + where: [['records.identity', '==', identityId]], + orderBy: [['records.identity', 'asc']], + limit: 100, + ...(startAfter ? { startAfter } : {}), + })); + documents.push(...page); + if (page.length < 100) break; + const last = page[page.length - 1]; + const next = identifierToBase58(last.$id || last.id); + if (!next || next === startAfter) { + throw new Error('DPNS: username pagination did not advance'); + } + startAfter = next; + } + } + } // Collect ALL usernames per identity (some users have multiple) const usernamesByIdentity = new Map(); diff --git a/lib/services/unified-profile-service.ts b/lib/services/unified-profile-service.ts index 54965c6d..1b24ee85 100644 --- a/lib/services/unified-profile-service.ts +++ b/lib/services/unified-profile-service.ts @@ -236,6 +236,7 @@ class UnifiedProfileService extends BaseDocumentService { const profileDoc = this.extractDocumentData(record); if (!profileDoc.$ownerId) continue; found.set(profileDoc.$ownerId, profileDoc); + cacheManager.delete(this.MISSING_PROFILE_CACHE, profileDoc.$ownerId); cacheManager.set(this.RAW_PROFILE_CACHE, profileDoc.$ownerId, profileDoc, { ttl: 300000, tags: ['profile', `user:${profileDoc.$ownerId}`] @@ -247,6 +248,7 @@ class UnifiedProfileService extends BaseDocumentService { } for (const ownerId of queriedOwnerIds) { if (found.has(ownerId)) continue; + cacheManager.delete(this.RAW_PROFILE_CACHE, ownerId); cacheManager.set(this.MISSING_PROFILE_CACHE, ownerId, true, { ttl: 60000, tags: ['profile', `user:${ownerId}`]