diff --git a/.gitignore b/.gitignore index 3b23dcd..733adf4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ build/ coverage/ apps/pwa/cypress/screenshots/ apps/pwa/cypress/videos/ +apps/pwa/cypress/downloads/ diff --git a/apps/pwa/cypress/downloads/studyos.ics b/apps/pwa/cypress/downloads/studyos.ics deleted file mode 100644 index 6f33fed..0000000 --- a/apps/pwa/cypress/downloads/studyos.ics +++ /dev/null @@ -1,11 +0,0 @@ -BEGIN:VCALENDAR -VERSION:2.0 -PRODID:-//StudyOS//planner//PT-BR -BEGIN:VEVENT -UID:019f4bee-5415-7f79-a3db-f0a66b4f7be1@studyos -SUMMARY:rotina e2e 1783685336185 -DTSTART:20260710T090000 -DURATION:PT90M -RRULE:FREQ=WEEKLY;BYDAY=FR -END:VEVENT -END:VCALENDAR diff --git a/apps/pwa/cypress/e2e/library-loop.cy.ts b/apps/pwa/cypress/e2e/library-loop.cy.ts new file mode 100644 index 0000000..51fc3a2 --- /dev/null +++ b/apps/pwa/cypress/e2e/library-loop.cy.ts @@ -0,0 +1,103 @@ +// M4 acceptance: search (stubbed sources), attach a video to a topic, see it on the +// topic, and find things via the global FTS search. Network is fully intercepted. +describe('library and content loop', () => { + const stamp = Date.now(); + const trackTitle = `trilha conteúdo ${stamp}`; + const topicTitle = `Controle difuso ${stamp}`; + const videoTitle = `Aula de controle difuso ${stamp}`; + + function stubSources(): void { + cy.intercept('GET', 'https://pt.wikipedia.org/w/api.php*', { + body: { + query: { + search: [ + { + pageid: 101, + title: 'Controle de constitucionalidade', + snippet: 'exame', + wordcount: 900, + }, + ], + }, + }, + }).as('wiki'); + cy.intercept('GET', 'https://api.stackexchange.com/**', { body: { items: [] } }).as('se'); + cy.intercept('GET', '/proxy/youtube/search*', { + body: { + items: [ + { + id: 'dQw4w9WgXcQ', + title: videoTitle, + channel: 'prof e2e', + thumbnail: null, + duration: null, + }, + ], + }, + }).as('yt'); + } + + it('sets up a track with one topic', () => { + cy.visit('/tracks'); + cy.get('[data-testid="track-title-input"]').type(trackTitle); + cy.get('[data-testid="track-submit"]').click(); + cy.get('[data-testid="track-item"]').contains(trackTitle).click(); + cy.get('[data-testid="topic-form"] [data-testid="topic-title-input"]').type(topicTitle); + cy.get('[data-testid="topic-submit"]').click(); + cy.get('[data-testid="topic-title"]').contains(topicTitle); + }); + + it('searches the library and attaches a video to the topic', () => { + stubSources(); + cy.visit('/library'); + cy.get('[data-testid="library-search-input"]').type('controle difuso'); + cy.get('[data-testid="library-search-submit"]').click(); + cy.wait(['@wiki', '@yt']); + + cy.get('[data-testid="library-result"]').contains(videoTitle); + cy.get('[data-testid="library-result"]') + .contains(videoTitle) + .closest('[data-testid="library-result"]') + .find('[data-testid="library-attach"]') + .click(); + cy.get('[data-testid="attach-track-select"]').select(trackTitle); + cy.get('[data-testid="attach-topic-select"]').select(topicTitle); + cy.get('[data-testid="attach-confirm"]').click(); + cy.contains('anexado ·'); + }); + + it('shows the attached video on the topic and plays it with transcript', () => { + cy.visit('/tracks'); + cy.get('[data-testid="track-item"]').contains(trackTitle).click(); + cy.get('[data-testid="topic-title"]').contains(topicTitle).click(); + cy.get('[data-testid="topic-content-list"] [data-testid="topic-content-item"]') + .contains(videoTitle) + .should('have.attr', 'href') + .and('include', '/library/watch/dQw4w9WgXcQ'); + + cy.intercept('GET', '/proxy/youtube/transcript*', { + headers: { 'content-type': 'text/xml' }, + body: 'primeira falasegunda fala', + }).as('transcript'); + cy.visit('/library/watch/dQw4w9WgXcQ'); + cy.wait('@transcript'); + cy.get('[data-testid="video-player"]').should('be.visible'); + cy.get('[data-testid="transcript-cue"]') + .should('have.length', 2) + .first() + .contains('primeira fala'); + }); + + it('finds the topic and the content via global search', () => { + cy.visit('/'); + cy.get('[data-testid="global-search-input"]').type('Controle difuso'); + cy.get('[data-testid="global-search-results"] [data-testid="global-search-result"]').contains( + topicTitle, + ); + cy.get('[data-testid="global-search-input"]').clear(); + cy.get('[data-testid="global-search-input"]').type('Aula de controle'); + cy.get('[data-testid="global-search-results"] [data-testid="global-search-result"]').contains( + videoTitle, + ); + }); +}); diff --git a/apps/pwa/package.json b/apps/pwa/package.json index f656bcc..a92c876 100644 --- a/apps/pwa/package.json +++ b/apps/pwa/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@journeyapps/wa-sqlite": "^1.7.0", + "@studyos/connectors": "workspace:*", "@studyos/core": "workspace:*", "@studyos/db": "workspace:*", "@studyos/shared": "workspace:*", diff --git a/apps/pwa/src/lib/components/GlobalSearch.svelte b/apps/pwa/src/lib/components/GlobalSearch.svelte new file mode 100644 index 0000000..5869f0b --- /dev/null +++ b/apps/pwa/src/lib/components/GlobalSearch.svelte @@ -0,0 +1,148 @@ + + +
+ + = 0 ? `global-search-option-${activeIndex}` : undefined} + placeholder="buscar" + autocomplete="off" + value={store.query} + {oninput} + {onkeydown} + {onblur} + {onfocus} + class="type-meta h-8 w-36 rounded-micro border border-border bg-surface px-3 text-text-body placeholder:text-text-low" + /> + + {#if store.open} + + {/if} +
diff --git a/apps/pwa/src/lib/db/client.ts b/apps/pwa/src/lib/db/client.ts index bac417a..eedde83 100644 --- a/apps/pwa/src/lib/db/client.ts +++ b/apps/pwa/src/lib/db/client.ts @@ -1,5 +1,12 @@ import { browser } from '$app/environment'; -import type { DbDriver, Row, SqlValue, Stmt } from '@studyos/db'; +import { + ensureSearchIndex, + reindexAll, + type DbDriver, + type Row, + type SqlValue, + type Stmt, +} from '@studyos/db'; import type { DbReady, DbRequest, DbResponse } from './rpc'; let instance: Promise | null = null; @@ -64,7 +71,7 @@ async function createDriver(): Promise { }); } - return { + const driver: DbDriver = { exec(sql: string, params?: SqlValue[]): Promise { const id = nextId++; return send( @@ -76,4 +83,10 @@ async function createDriver(): Promise { await send({ id, kind: 'batch', stmts, mutates: mutatedTables(stmts) }); }, }; + + // Local-only FTS index (see packages/db/src/search.ts): create after migrate, + // then rebuild in the background so global search reflects the current data. + await ensureSearchIndex(driver); + void reindexAll(driver); + return driver; } diff --git a/apps/pwa/src/lib/stores/library.svelte.ts b/apps/pwa/src/lib/stores/library.svelte.ts new file mode 100644 index 0000000..1153e4a --- /dev/null +++ b/apps/pwa/src/lib/stores/library.svelte.ts @@ -0,0 +1,156 @@ +import { dev } from '$app/environment'; +import { getConnector, type ContentResult, type FetchLike } from '@studyos/connectors'; +import { + attachContent, + getOrCreateDeviceId, + getSetting, + listTopics, + listTracks, +} from '@studyos/db'; +import { SETTINGS_KEYS, type TopicRow, type TrackRow } from '@studyos/shared'; +import { getDb } from '$lib/db/client'; +import { liveQuery } from '$lib/db/live.svelte'; + +export const SOURCES = ['wikipedia', 'stackexchange', 'youtube'] as const; +export type Source = (typeof SOURCES)[number]; +export type SourceFilter = 'all' | Source; + +export const SOURCE_LABELS: Record = { + wikipedia: 'wikipédia', + stackexchange: 'stack exchange', + youtube: 'youtube', +}; + +export const KIND_LABELS: Record = { + video: 'vídeo', + article: 'artigo', + qa: 'pergunta', + doc: 'documento', +}; + +// Same token policy as lib/sync: stored sync token, dev fallback in dev builds. +async function getToken(): Promise { + const db = await getDb(); + const stored = await getSetting(db, SETTINGS_KEYS.syncToken); + if (stored) return stored; + return dev ? 'dev-token' : null; +} + +/** fetch with the sync bearer token — required by the worker's /proxy/* routes. */ +export const authedFetch: FetchLike = async (url, init) => { + const token = await getToken(); + const headers = new Headers(init?.headers); + if (token !== null) headers.set('authorization', `Bearer ${token}`); + return fetch(url, { ...init, headers }); +}; + +export interface SourceGroup { + source: Source; + results: ContentResult[]; +} + +const plainFetch: FetchLike = (url, init) => fetch(url, init); + +async function loadTopics(trackId: string): Promise { + const db = await getDb(); + return listTopics(db, trackId); +} + +async function attach(result: ContentResult, topicId: string): Promise { + const db = await getDb(); + const deviceId = await getOrCreateDeviceId(db); + await attachContent(db, deviceId, { + topic_id: topicId, + source: result.source, + external_id: result.external_id, + url: result.url, + title: result.title, + kind: result.kind, + meta_json: JSON.stringify(result.meta), + }); +} + +export interface LibraryStore { + get status(): 'idle' | 'loading' | 'done'; + get groups(): SourceGroup[]; + get youtubeUnavailable(): boolean; + get filter(): SourceFilter; + get tracks(): TrackRow[]; + setFilter(next: SourceFilter): void; + search(q: string): Promise; + loadTopics(trackId: string): Promise; + attach(result: ContentResult, topicId: string): Promise; + destroy(): void; +} + +export function createLibraryStore(): LibraryStore { + let status = $state<'idle' | 'loading' | 'done'>('idle'); + let groups = $state([]); + let youtubeUnavailable = $state(false); + let filter = $state('all'); + // Guards against a slow earlier search overwriting a newer one. + let searchSeq = 0; + + const tracksLive = liveQuery((db) => listTracks(db), ['tracks'], [] as TrackRow[]); + + async function search(q: string): Promise { + const query = q.trim(); + if (query === '') return; + const seq = ++searchSeq; + status = 'loading'; + + const enabled = SOURCES.filter((source) => filter === 'all' || filter === source); + let youtubeStatus: number | null = null; + // The youtube connector goes through the worker proxy: it needs the bearer + // token, and recording the response status lets us tell "no results" apart + // from "proxy not configured" (503). + const youtubeFetch: FetchLike = async (url, init) => { + const res = await authedFetch(url, init); + youtubeStatus = res.status; + return res; + }; + + const settled = await Promise.allSettled( + enabled.map((source) => { + const connector = getConnector(source); + if (connector === null) return Promise.resolve([]); + return connector.search(query, source === 'youtube' ? youtubeFetch : plainFetch); + }), + ); + if (seq !== searchSeq) return; + + groups = enabled.map((source, i) => { + const outcome = settled[i]; + return { source, results: outcome?.status === 'fulfilled' ? outcome.value : [] }; + }); + youtubeUnavailable = youtubeStatus === 503; + status = 'done'; + } + + return { + get status() { + return status; + }, + get groups() { + return groups; + }, + get youtubeUnavailable() { + return youtubeUnavailable; + }, + get filter() { + return filter; + }, + get tracks() { + return tracksLive.value; + }, + setFilter(next: SourceFilter) { + filter = next; + }, + search, + loadTopics, + attach, + destroy() { + tracksLive.destroy(); + }, + }; +} diff --git a/apps/pwa/src/lib/stores/search.svelte.ts b/apps/pwa/src/lib/stores/search.svelte.ts new file mode 100644 index 0000000..0a8f62f --- /dev/null +++ b/apps/pwa/src/lib/stores/search.svelte.ts @@ -0,0 +1,123 @@ +import { browser } from '$app/environment'; +import { + getCard, + getContent, + getTopic, + reindexAll, + searchLocal, + type SearchHit, +} from '@studyos/db'; +import { DB_CHANNEL } from '@studyos/shared'; +import { getDb } from '$lib/db/client'; +import type { DbBroadcast } from '$lib/db/rpc'; + +const REINDEX_TABLES = new Set(['topics', 'cards', 'content_items']); +const REINDEX_DEBOUNCE_MS = 1000; +const SEARCH_DEBOUNCE_MS = 200; +const SEARCH_LIMIT = 8; + +// Module singleton: one channel per tab keeps the fts index fresh after +// writes to the indexed tables (repos don't maintain it incrementally in M4). +let reindexWatcherStarted = false; + +function startReindexWatcher(): void { + if (!browser || reindexWatcherStarted) return; + reindexWatcherStarted = true; + const channel = new BroadcastChannel(DB_CHANNEL); + let timer: ReturnType | null = null; + channel.addEventListener('message', (event: MessageEvent) => { + const message = event.data; + if (message.kind !== 'tables-changed') return; + if (!message.tables.some((t) => REINDEX_TABLES.has(t))) return; + if (timer !== null) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + void getDb().then((db) => reindexAll(db)); + }, REINDEX_DEBOUNCE_MS); + }); +} + +async function resolveHref(hit: SearchHit): Promise { + const db = await getDb(); + if (hit.kind === 'topic') { + const topic = await getTopic(db, hit.ref_id); + return topic === null ? null : `/tracks/${topic.track_id}`; + } + if (hit.kind === 'card') { + const card = await getCard(db, hit.ref_id); + if (card === null) return null; + const topic = await getTopic(db, card.topic_id); + return topic === null ? null : `/tracks/${topic.track_id}`; + } + const content = await getContent(db, hit.ref_id); + if (content === null) return null; + if (content.source === 'youtube' && content.external_id !== null) { + return `/library/watch/${content.external_id}`; + } + return content.url; +} + +export interface SearchStore { + get query(): string; + set query(value: string); + get results(): SearchHit[]; + get open(): boolean; + run(): void; + resolveHref(hit: SearchHit): Promise; + close(): void; +} + +export function createSearchStore(): SearchStore { + startReindexWatcher(); + + let query = $state(''); + let results = $state([]); + let open = $state(false); + let timer: ReturnType | null = null; + let runId = 0; + + function run(): void { + if (timer !== null) { + clearTimeout(timer); + timer = null; + } + const q = query.trim(); + if (q === '') { + runId += 1; // invalidate any in-flight search + results = []; + open = false; + return; + } + timer = setTimeout(() => { + timer = null; + const id = ++runId; + void (async () => { + const db = await getDb(); + const hits = await searchLocal(db, q, SEARCH_LIMIT); + if (id !== runId) return; + results = hits; + open = true; + })(); + }, SEARCH_DEBOUNCE_MS); + } + + return { + get query() { + return query; + }, + set query(value: string) { + query = value; + }, + get results() { + return results; + }, + get open() { + return open; + }, + run, + resolveHref, + close() { + open = false; + }, + }; +} diff --git a/apps/pwa/src/routes/+layout.svelte b/apps/pwa/src/routes/+layout.svelte index e6e5520..16a0a66 100644 --- a/apps/pwa/src/routes/+layout.svelte +++ b/apps/pwa/src/routes/+layout.svelte @@ -2,6 +2,7 @@ import '../app.css'; import { onMount } from 'svelte'; import { page } from '$app/state'; + import GlobalSearch from '$lib/components/GlobalSearch.svelte'; import { requestPersistence } from '$lib/db/client'; import { registerServiceWorker } from '$lib/push/register'; import { startSyncLifecycle } from '$lib/sync/index.svelte'; @@ -14,6 +15,7 @@ { href: '/tracks', label: 'trilhas' }, { href: '/routines', label: 'rotina' }, { href: '/study', label: 'estudar' }, + { href: '/library', label: 'biblioteca' }, { href: '/reminders', label: 'lembretes' }, { href: '/stats', label: 'stats' }, ] as const; @@ -80,6 +82,7 @@ + {#if !online} diff --git a/apps/pwa/src/routes/library/+page.svelte b/apps/pwa/src/routes/library/+page.svelte new file mode 100644 index 0000000..ee29376 --- /dev/null +++ b/apps/pwa/src/routes/library/+page.svelte @@ -0,0 +1,177 @@ + + + + StudyOS — biblioteca + + +
+

conteúdo

+

biblioteca

+ +
+ +
+ + +
+
+ +
+ {#each FILTERS as option (option.value)} + + {/each} +
+ +
+ {#if store.status === 'idle'} +

busque um assunto — vídeo, artigo ou pergunta.

+ {:else if store.status === 'loading'} +

buscando…

+ {:else if total === 0 && !store.youtubeUnavailable} +

nada encontrado — tente outros termos.

+ {/if} +
+ + {#if store.status === 'done' && visibleGroups.length > 0} +
+ {#each visibleGroups as group (group.source)} +
+

{SOURCE_LABELS[group.source]}

+ {#if group.source === 'youtube' && store.youtubeUnavailable} +

busca do youtube não configurada.

+ {/if} +
    + {#each group.results as result (keyOf(result))} +
  • + {#if result.source === 'youtube'} + + {result.title} + + {:else} + + {result.title} + + {/if} +

    + {KIND_LABELS[result.kind] ?? result.kind} · {SOURCE_LABELS[group.source]} +

    + {#if result.description !== null && result.description !== ''} +

    {result.description}

    + {/if} + + {#if attached[keyOf(result)] !== undefined} +

    anexado · {attached[keyOf(result)]}

    + {:else} + + {/if} + + {#if openPickerKey === keyOf(result)} + store.loadTopics(trackId)} + onconfirm={(topic) => void confirmAttach(result, topic)} + /> + {/if} +
  • + {/each} +
+
+ {/each} +
+ {/if} +
diff --git a/apps/pwa/src/routes/library/AttachPicker.svelte b/apps/pwa/src/routes/library/AttachPicker.svelte new file mode 100644 index 0000000..30745de --- /dev/null +++ b/apps/pwa/src/routes/library/AttachPicker.svelte @@ -0,0 +1,76 @@ + + +
+
+ + +
+ +
+ + +
+ + +
diff --git a/apps/pwa/src/routes/library/watch/[videoId]/+page.svelte b/apps/pwa/src/routes/library/watch/[videoId]/+page.svelte new file mode 100644 index 0000000..c9072bb --- /dev/null +++ b/apps/pwa/src/routes/library/watch/[videoId]/+page.svelte @@ -0,0 +1,187 @@ + + + + StudyOS — vídeo + + +
+ + ← biblioteca + + + {#if !validId} +

vídeo não encontrado.

+ {:else} + + +
+

transcrição

+ +
+ +
+ {#if transcriptState === 'loading'} +

carregando transcrição…

+ {:else if transcriptState === 'missing'} +

sem transcrição disponível.

+ {:else} +
    + {#each cues as cue, i (i)} +
  • + +
  • + {/each} +
+ {/if} +
+ {/if} +
+ + diff --git a/apps/pwa/src/routes/tracks/[id]/+page.svelte b/apps/pwa/src/routes/tracks/[id]/+page.svelte index 1fa1796..e9e498e 100644 --- a/apps/pwa/src/routes/tracks/[id]/+page.svelte +++ b/apps/pwa/src/routes/tracks/[id]/+page.svelte @@ -11,6 +11,7 @@ import OutlineImport from './OutlineImport.svelte'; import CardsPanel from './CardsPanel.svelte'; import CycleEditor from './CycleEditor.svelte'; + import TopicContent from './TopicContent.svelte'; const trackId = $derived(page.params.id ?? ''); @@ -161,6 +162,7 @@ {cards} onadd={(front, back) => store?.addCard(front, back) ?? Promise.resolve()} /> + {:else}

cards

selecione um tópico para ver e criar cards.

diff --git a/apps/pwa/src/routes/tracks/[id]/TopicContent.svelte b/apps/pwa/src/routes/tracks/[id]/TopicContent.svelte new file mode 100644 index 0000000..cc11b2a --- /dev/null +++ b/apps/pwa/src/routes/tracks/[id]/TopicContent.svelte @@ -0,0 +1,88 @@ + + +{#if items.length > 0} +
+

conteúdo

+ +
    + {#each items as item (item.id)} + {@const href = hrefFor(item)} + {@const external = href !== null && href.startsWith('http')} +
  • + {#if href !== null} + + {item.title} + + {:else} + {item.title} + {/if} + {badge(item)} + +
  • + {/each} +
+
+{/if} diff --git a/apps/worker/.dev.vars.example b/apps/worker/.dev.vars.example index 353ee5d..e15875d 100644 --- a/apps/worker/.dev.vars.example +++ b/apps/worker/.dev.vars.example @@ -3,3 +3,5 @@ SYNC_TOKEN=dev-token VAPID_PUBLIC_KEY= VAPID_PRIVATE_KEY= VAPID_SUBJECT=mailto:dev@example.com +# Optional: YouTube Data API v3 key — without it /proxy/youtube/search answers 503 +YOUTUBE_API_KEY= diff --git a/apps/worker/README.md b/apps/worker/README.md index 708f31c..95dbd09 100644 --- a/apps/worker/README.md +++ b/apps/worker/README.md @@ -14,6 +14,34 @@ See `docs/SYNC.md` for the frozen wire contract. - `POST /push/subscribe` - same auth, body `{ device_id, endpoint, p256dh, auth }`, upserts into `push_subscriptions` keyed by `device_id` - `GET /push/vapid` - same auth, returns `{ publicKey }` for `pushManager.subscribe` +- `GET /proxy/youtube/search?q=` - same auth, see "Content proxy" +- `GET /proxy/youtube/transcript?id=` - same auth, see "Content proxy" +- `GET /proxy/rss?url=` - same auth, see "Content proxy" + +## Content proxy + +Three bearer-auth'd `GET` endpoints back the M4 content features (`docs/M4-CONTRACTS.md`), +all served from `src/proxy.ts` and cached with the workerd Cache API (`caches.default`, +behind the `src/cache.ts` seam so bun tests can inject a fake): + +- `/proxy/youtube/search?q=` - calls the YouTube Data API v3 + (`search?part=snippet&type=video&maxResults=10`) with the `YOUTUBE_API_KEY` secret and + maps the result to the frozen wire format + `{ items: { id, title, channel, thumbnail, duration }[] }`. `duration` is always `null` + in M4: the search endpoint does not return it and fetching it would cost an extra + `videos.list` call. Missing secret answers `503 { "error": "youtube api not configured" }` + so the PWA can hide the source. Cached 6h, keyed by the trimmed lowercase query. +- `/proxy/youtube/transcript?id=` - fetches `https://www.youtube.com/api/timedtext?v=` + with `lang=pt` then falls back to `lang=en` (YouTube answers 200 with an empty body when + a track is missing); returns the raw timedtext XML as `text/xml` for the client-side + `parseTimedText`, or 404 when neither language exists. Cached 24h. +- `/proxy/rss?url=` - fetches the given feed (https only; localhost, private ranges and + `*.internal` hosts are rejected with 400) with a 5s timeout and passes the body through + with its original content-type. Upstream failures answer 502. Cached 1h. + +`YOUTUBE_API_KEY` is optional: set it in `.dev.vars` locally and with +`bun x wrangler secret put YOUTUBE_API_KEY` in production (an API key restricted to the +YouTube Data API v3 is enough — no OAuth). ## Web push diff --git a/apps/worker/src/cache.ts b/apps/worker/src/cache.ts new file mode 100644 index 0000000..1386e80 --- /dev/null +++ b/apps/worker/src/cache.ts @@ -0,0 +1,39 @@ +/** + * Seam around the workerd Cache API: bun (tests) has no `caches` global, so + * handlers go through getCache() and tests inject a fake via + * setCacheForTesting(). Structural on purpose — only what the proxy needs. + */ +export interface CacheLike { + match(req: Request): Promise; + put(req: Request, res: Response): Promise; +} + +// Minimal local view of the workerd global (no @cloudflare/workers-types here). +interface WorkerdCaches { + default: CacheLike; +} + +/** + * Bun types Response.clone() as the undici base Response, which is not + * assignable back to the global Response; at runtime it is the same object. + * Recast here so callers can cache a clone and return the original. + */ +export function cloneResponse(res: Response): Response { + return res.clone() as Response; +} + +let testCache: CacheLike | null = null; + +/** Test-only: inject a fake cache; pass null to restore the workerd default. */ +export function setCacheForTesting(cache: CacheLike | null): void { + testCache = cache; +} + +export async function getCache(): Promise { + if (testCache) return testCache; + const caches = (globalThis as { caches?: WorkerdCaches }).caches; + if (!caches) { + throw new Error('Cache API unavailable: outside workerd, inject one with setCacheForTesting'); + } + return caches.default; +} diff --git a/apps/worker/src/env.ts b/apps/worker/src/env.ts index 75d7599..a2310ec 100644 --- a/apps/worker/src/env.ts +++ b/apps/worker/src/env.ts @@ -26,4 +26,6 @@ export interface Env { VAPID_PRIVATE_KEY: string; /** mailto: or https: contact, RFC 8292 `sub` claim. */ VAPID_SUBJECT: string; + /** YouTube Data API v3 key; optional — /proxy/youtube/search answers 503 without it. */ + YOUTUBE_API_KEY?: string; } diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 9ae7e6d..78c61c9 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import { bearerAuth } from './auth'; import { handleCron } from './cron'; import type { Env } from './env'; +import { handleRss, handleYoutubeSearch, handleYoutubeTranscript } from './proxy'; import { handleSubscribe, handleVapidKey } from './push'; import { handlePull, handlePush } from './sync'; @@ -14,6 +15,10 @@ export function createApp(): Hono<{ Bindings: Env }> { app.use('/push/*', bearerAuth); app.post('/push/subscribe', handleSubscribe); app.get('/push/vapid', handleVapidKey); + app.use('/proxy/*', bearerAuth); + app.get('/proxy/youtube/search', handleYoutubeSearch); + app.get('/proxy/youtube/transcript', handleYoutubeTranscript); + app.get('/proxy/rss', handleRss); return app; } diff --git a/apps/worker/src/proxy.ts b/apps/worker/src/proxy.ts new file mode 100644 index 0000000..795fe7c --- /dev/null +++ b/apps/worker/src/proxy.ts @@ -0,0 +1,158 @@ +import type { Handler } from 'hono'; +import { cloneResponse, getCache } from './cache'; +import type { Env } from './env'; + +// Synthetic origin for Cache API keys — never fetched, just a stable namespace. +const CACHE_ORIGIN = 'https://cache.studyos'; + +const VIDEO_ID_RE = /^[A-Za-z0-9_-]{5,20}$/; + +interface YoutubeSearchUpstream { + items?: { + id?: { videoId?: string }; + snippet?: { + title?: string; + channelTitle?: string; + thumbnails?: { medium?: { url?: string }; default?: { url?: string } }; + }; + }[]; +} + +/** Frozen wire format (docs/M4-CONTRACTS.md). */ +interface YoutubeSearchItem { + id: string; + title: string; + channel: string; + thumbnail: string | null; + duration: string | null; +} + +export const handleYoutubeSearch: Handler<{ Bindings: Env }> = async (c) => { + const q = c.req.query('q')?.trim() ?? ''; + if (q === '') return c.json({ error: 'q query param is required' }, 400); + const key = c.env.YOUTUBE_API_KEY; + if (key === undefined || key === '') { + return c.json({ error: 'youtube api not configured' }, 503); + } + + const cache = await getCache(); + const cacheKey = new Request( + `${CACHE_ORIGIN}/yt-search?q=${encodeURIComponent(q.toLowerCase())}`, + ); + const hit = await cache.match(cacheKey); + if (hit) return hit; + + const url = new URL('https://www.googleapis.com/youtube/v3/search'); + url.searchParams.set('part', 'snippet'); + url.searchParams.set('type', 'video'); + url.searchParams.set('maxResults', '10'); + url.searchParams.set('q', q); + url.searchParams.set('key', key); + const upstream = await fetch(url.toString()); + if (!upstream.ok) return c.json({ error: 'youtube upstream error' }, 502); + + const data = (await upstream.json()) as YoutubeSearchUpstream; + const items: YoutubeSearchItem[] = []; + for (const item of data.items ?? []) { + const id = item.id?.videoId; + if (!id) continue; + items.push({ + id, + title: item.snippet?.title ?? '', + channel: item.snippet?.channelTitle ?? '', + thumbnail: + item.snippet?.thumbnails?.medium?.url ?? item.snippet?.thumbnails?.default?.url ?? null, + // the search endpoint has no contentDetails; duration would need an extra + // videos.list call per page — deferred, the wire format allows null. + duration: null, + }); + } + + const res = new Response(JSON.stringify({ items }), { + headers: { + 'content-type': 'application/json', + 'cache-control': 'public, max-age=21600', // 6h + }, + }); + await cache.put(cacheKey, cloneResponse(res)); + return res; +}; + +export const handleYoutubeTranscript: Handler<{ Bindings: Env }> = async (c) => { + const id = c.req.query('id') ?? ''; + if (!VIDEO_ID_RE.test(id)) return c.json({ error: 'invalid video id' }, 400); + + const cache = await getCache(); + const cacheKey = new Request(`${CACHE_ORIGIN}/yt-transcript?id=${id}`); + const hit = await cache.match(cacheKey); + if (hit) return hit; + + let xml: string | null = null; + for (const lang of ['pt', 'en']) { + const upstream = await fetch(`https://www.youtube.com/api/timedtext?v=${id}&lang=${lang}`); + if (!upstream.ok) continue; + const body = await upstream.text(); + if (body.trim() === '') continue; // youtube answers 200 with an empty body + xml = body; + break; + } + if (xml === null) return c.json({ error: 'transcript not found' }, 404); + + const res = new Response(xml, { + headers: { + 'content-type': 'text/xml; charset=utf-8', + 'cache-control': 'public, max-age=86400', // 24h + }, + }); + await cache.put(cacheKey, cloneResponse(res)); + return res; +}; + +function isBlockedHost(hostname: string): boolean { + const host = hostname.toLowerCase(); + return ( + host === 'localhost' || + host === '::1' || + host === '[::1]' || + host.startsWith('127.') || + host.startsWith('10.') || + host.startsWith('192.168.') || + host.endsWith('.internal') + ); +} + +export const handleRss: Handler<{ Bindings: Env }> = async (c) => { + const raw = c.req.query('url') ?? ''; + let target: URL; + try { + target = new URL(raw); + } catch { + return c.json({ error: 'invalid url' }, 400); + } + if (target.protocol !== 'https:' || isBlockedHost(target.hostname)) { + return c.json({ error: 'url not allowed' }, 400); + } + + const cache = await getCache(); + const cacheKey = new Request(`${CACHE_ORIGIN}/rss?url=${encodeURIComponent(target.toString())}`); + const hit = await cache.match(cacheKey); + if (hit) return hit; + + let upstream: Response; + try { + upstream = await fetch(target.toString(), { signal: AbortSignal.timeout(5000) }); + } catch { + return c.json({ error: 'upstream unreachable' }, 502); + } + if (!upstream.ok) return c.json({ error: 'upstream error' }, 502); + + const body = await upstream.arrayBuffer(); + const res = new Response(body, { + headers: { + 'content-type': upstream.headers.get('content-type') ?? 'application/octet-stream', + 'cache-control': 'public, max-age=3600', // 1h + }, + }); + await cache.put(cacheKey, cloneResponse(res)); + return res; +}; diff --git a/apps/worker/test/proxy.test.ts b/apps/worker/test/proxy.test.ts new file mode 100644 index 0000000..3ea7810 --- /dev/null +++ b/apps/worker/test/proxy.test.ts @@ -0,0 +1,286 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { cloneResponse, setCacheForTesting, type CacheLike } from '../src/cache'; +import { createApp } from '../src/index'; +import type { Env } from '../src/env'; +import { createFakeD1 } from './fake-d1'; + +const TOKEN = 'test'; +const app = createApp(); +const AUTH = { headers: { authorization: `Bearer ${TOKEN}` } }; + +// recorded-shape fixture: YouTube Data API v3 search response (trimmed) +const YT_SEARCH_FIXTURE = { + kind: 'youtube#searchListResponse', + items: [ + { + id: { kind: 'youtube#video', videoId: 'vid-11' }, + snippet: { + title: 'Direito constitucional — aula 1', + channelTitle: 'Canal Estudos', + thumbnails: { + default: { url: 'https://i.ytimg.com/vi/vid-11/default.jpg' }, + medium: { url: 'https://i.ytimg.com/vi/vid-11/mqdefault.jpg' }, + }, + }, + }, + { + id: { kind: 'youtube#video', videoId: 'vid-22' }, + snippet: { + title: 'Aula 2', + channelTitle: 'Outro Canal', + thumbnails: { default: { url: 'https://i.ytimg.com/vi/vid-22/default.jpg' } }, + }, + }, + // no videoId (channel result): must be skipped by the mapper + { id: { kind: 'youtube#channel' }, snippet: { title: 'Canal', channelTitle: 'Canal' } }, + ], +}; + +class FakeCache implements CacheLike { + private readonly store = new Map(); + + async match(req: Request): Promise { + const res = this.store.get(req.url); + return res === undefined ? undefined : cloneResponse(res); + } + + async put(req: Request, res: Response): Promise { + this.store.set(req.url, res); + } + + get size(): number { + return this.store.size; + } + + keys(): string[] { + return [...this.store.keys()]; + } +} + +const realFetch = globalThis.fetch; + +let env: Env; +let cache: FakeCache; +let fetched: string[]; +let responder: (url: string) => Response; + +beforeEach(async () => { + env = { + DB: await createFakeD1(), + SYNC_TOKEN: TOKEN, + ASSETS: { fetch: async () => new Response(null, { status: 404 }) }, + VAPID_PUBLIC_KEY: 'test-public-key', + VAPID_PRIVATE_KEY: '{}', + VAPID_SUBJECT: 'mailto:test@example.com', + YOUTUBE_API_KEY: 'yt-key', + }; + cache = new FakeCache(); + setCacheForTesting(cache); + + fetched = []; + responder = () => new Response(null, { status: 500 }); + globalThis.fetch = (async (input: string | URL | Request) => { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url; + fetched.push(url); + if (url.includes('boom')) throw new Error('network down'); + return responder(url); + }) as typeof fetch; +}); + +afterEach(() => { + globalThis.fetch = realFetch; + setCacheForTesting(null); +}); + +describe('proxy auth', () => { + test('all proxy routes require the bearer token', async () => { + for (const path of [ + '/proxy/youtube/search?q=x', + '/proxy/youtube/transcript?id=vid-11', + '/proxy/rss?url=https%3A%2F%2Fexample.com%2Ffeed.xml', + ]) { + const res = await app.request(path, {}, env); + expect(res.status).toBe(401); + expect(fetched).toHaveLength(0); + } + }); +}); + +describe('/proxy/youtube/search', () => { + test('missing q returns 400', async () => { + expect((await app.request('/proxy/youtube/search', AUTH, env)).status).toBe(400); + expect((await app.request('/proxy/youtube/search?q=%20%20', AUTH, env)).status).toBe(400); + }); + + test('missing YOUTUBE_API_KEY returns 503', async () => { + const { YOUTUBE_API_KEY: _unused, ...rest } = env; + const res = await app.request('/proxy/youtube/search?q=direito', AUTH, rest as Env); + expect(res.status).toBe(503); + expect(await res.json()).toEqual({ error: 'youtube api not configured' }); + expect(fetched).toHaveLength(0); + }); + + test('maps the Data API response to the frozen wire format', async () => { + responder = () => Response.json(YT_SEARCH_FIXTURE); + + const res = await app.request('/proxy/youtube/search?q=Direito', AUTH, env); + expect(res.status).toBe(200); + expect(res.headers.get('cache-control')).toBe('public, max-age=21600'); + + const upstream = new URL(fetched[0] ?? ''); + expect(upstream.origin + upstream.pathname).toBe( + 'https://www.googleapis.com/youtube/v3/search', + ); + expect(upstream.searchParams.get('part')).toBe('snippet'); + expect(upstream.searchParams.get('type')).toBe('video'); + expect(upstream.searchParams.get('maxResults')).toBe('10'); + expect(upstream.searchParams.get('key')).toBe('yt-key'); + + expect(await res.json()).toEqual({ + items: [ + { + id: 'vid-11', + title: 'Direito constitucional — aula 1', + channel: 'Canal Estudos', + thumbnail: 'https://i.ytimg.com/vi/vid-11/mqdefault.jpg', + duration: null, + }, + { + id: 'vid-22', + title: 'Aula 2', + channel: 'Outro Canal', + thumbnail: 'https://i.ytimg.com/vi/vid-22/default.jpg', + duration: null, + }, + ], + }); + }); + + test('second call is served from cache, normalized on the query', async () => { + responder = () => Response.json(YT_SEARCH_FIXTURE); + + const first = await app.request('/proxy/youtube/search?q=direito', AUTH, env); + // different case/whitespace, same normalized cache key + const second = await app.request('/proxy/youtube/search?q=%20DiReiTo%20', AUTH, env); + + expect(fetched).toHaveLength(1); + expect(cache.size).toBe(1); + expect(cache.keys()[0]).toBe('https://cache.studyos/yt-search?q=direito'); + expect(await second.json()).toEqual(await first.json()); + }); + + test('upstream failure returns 502 and is not cached', async () => { + const res = await app.request('/proxy/youtube/search?q=direito', AUTH, env); + expect(res.status).toBe(502); + expect(cache.size).toBe(0); + }); +}); + +describe('/proxy/youtube/transcript', () => { + const PT_XML = 'olá'; + const EN_XML = 'hello'; + + test('invalid id returns 400', async () => { + for (const id of ['', 'abc', 'a'.repeat(21), 'bad%20id', 'in%2Fvalid']) { + const res = await app.request(`/proxy/youtube/transcript?id=${id}`, AUTH, env); + expect(res.status).toBe(400); + } + expect(fetched).toHaveLength(0); + }); + + test('returns the pt transcript as text/xml when available', async () => { + responder = () => new Response(PT_XML); + + const res = await app.request('/proxy/youtube/transcript?id=vid-11', AUTH, env); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('text/xml; charset=utf-8'); + expect(res.headers.get('cache-control')).toBe('public, max-age=86400'); + expect(await res.text()).toBe(PT_XML); + expect(fetched).toEqual(['https://www.youtube.com/api/timedtext?v=vid-11&lang=pt']); + }); + + test('falls back to en when the pt track is an empty 200 body', async () => { + responder = (url) => new Response(url.includes('lang=pt') ? '' : EN_XML); + + const res = await app.request('/proxy/youtube/transcript?id=vid-11', AUTH, env); + expect(res.status).toBe(200); + expect(await res.text()).toBe(EN_XML); + expect(fetched).toEqual([ + 'https://www.youtube.com/api/timedtext?v=vid-11&lang=pt', + 'https://www.youtube.com/api/timedtext?v=vid-11&lang=en', + ]); + }); + + test('404 when neither language has a track', async () => { + responder = () => new Response(''); + + const res = await app.request('/proxy/youtube/transcript?id=vid-11', AUTH, env); + expect(res.status).toBe(404); + expect(fetched).toHaveLength(2); + expect(cache.size).toBe(0); + }); + + test('second call is served from cache', async () => { + responder = () => new Response(PT_XML); + + await app.request('/proxy/youtube/transcript?id=vid-11', AUTH, env); + const second = await app.request('/proxy/youtube/transcript?id=vid-11', AUTH, env); + + expect(fetched).toHaveLength(1); + expect(await second.text()).toBe(PT_XML); + }); +}); + +describe('/proxy/rss', () => { + const FEED = ''; + + function rssPath(url: string): string { + return `/proxy/rss?url=${encodeURIComponent(url)}`; + } + + test('rejects non-https and private hosts with 400', async () => { + for (const url of [ + 'not a url', + 'http://example.com/feed.xml', + 'ftp://example.com/feed.xml', + 'https://localhost/feed.xml', + 'https://127.0.0.1/feed.xml', + 'https://10.1.2.3/feed.xml', + 'https://192.168.1.30/feed.xml', + 'https://[::1]/feed.xml', + 'https://sync.internal/feed.xml', + ]) { + const res = await app.request(rssPath(url), AUTH, env); + expect(res.status).toBe(400); + } + expect(fetched).toHaveLength(0); + }); + + test('passes through body and content-type, then serves from cache', async () => { + responder = () => + new Response(FEED, { headers: { 'content-type': 'application/rss+xml; charset=utf-8' } }); + + const res = await app.request(rssPath('https://example.com/feed.xml'), AUTH, env); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/rss+xml; charset=utf-8'); + expect(res.headers.get('cache-control')).toBe('public, max-age=3600'); + expect(await res.text()).toBe(FEED); + + const second = await app.request(rssPath('https://example.com/feed.xml'), AUTH, env); + expect(fetched).toHaveLength(1); + expect(await second.text()).toBe(FEED); + }); + + test('upstream non-2xx returns 502', async () => { + responder = () => new Response('nope', { status: 404 }); + const res = await app.request(rssPath('https://example.com/feed.xml'), AUTH, env); + expect(res.status).toBe(502); + expect(cache.size).toBe(0); + }); + + test('upstream network failure returns 502', async () => { + const res = await app.request(rssPath('https://boom.example.com/feed.xml'), AUTH, env); + expect(res.status).toBe(502); + }); +}); diff --git a/bun.lock b/bun.lock index 34b9069..c3f1fba 100644 --- a/bun.lock +++ b/bun.lock @@ -18,6 +18,7 @@ "version": "0.0.1", "dependencies": { "@journeyapps/wa-sqlite": "^1.7.0", + "@studyos/connectors": "workspace:*", "@studyos/core": "workspace:*", "@studyos/db": "workspace:*", "@studyos/shared": "workspace:*", diff --git a/docs/M4-CONTRACTS.md b/docs/M4-CONTRACTS.md new file mode 100644 index 0000000..f006f6b --- /dev/null +++ b/docs/M4-CONTRACTS.md @@ -0,0 +1,126 @@ +# M4 contracts (frozen for parallel workstreams) + +Content: `Connector` interface + Wikipedia/Stack Exchange (client-side) + YouTube via +Worker proxy, Library screen with unified search, attach content to topics, video player +with transcript, local FTS5 search. Done when: search a subject → attach a video to a +topic → open it with transcript from the topic. + +## packages/connectors (stream A) + +```ts +export type ContentKind = 'video' | 'article' | 'qa' | 'doc'; +export interface ContentResult { + source: string; // 'youtube' | 'wikipedia' | 'stackexchange' + external_id: string; + url: string; + title: string; + kind: ContentKind; + description: string | null; + meta: Record; // thumbnails, score, duration... source-specific +} +export type FetchLike = (url: string, init?: RequestInit) => Promise; +export interface Connector { + source: string; + kind: ContentKind; + search(q: string, fetchFn: FetchLike): Promise; // <= 10 results +} +export const connectors: Connector[]; // registry, stable order: wikipedia, stackexchange, youtube +export function getConnector(source: string): Connector | null; +``` + +- **wikipedia**: `https://pt.wikipedia.org/w/api.php?action=query&list=search&format=json&origin=*&srsearch=` → + kind 'article', url `https://pt.wikipedia.org/wiki/`, external_id = pageid. +- **stackexchange**: `https://api.stackexchange.com/2.3/search/advanced?site=stackoverflow&order=desc&sort=relevance&q=` → + kind 'qa', title html-entity-decoded, meta { score, answer_count, is_answered }. +- **youtube**: calls the app-relative proxy `/proxy/youtube/search?q=` (Worker adds the API + key). Response wire format (frozen, worker implements): + `{ items: { id, title, channel, thumbnail, duration: string | null }[] }` → + kind 'video', url `https://www.youtube.com/watch?v=<id>`, external_id = id. +- All connectors: inject `fetchFn`, no globals; non-2xx → return `[]` (never throw); + unit-tested in bun with stubbed fetch + recorded-shape fixtures. + +### Transcript (youtube only) + +```ts +export interface TranscriptCue { + start: number; + dur: number; + text: string; +} +export function parseTimedText(xml: string): TranscriptCue[]; // youtube timedtext XML -> cues +``` + +## apps/worker proxy (stream B) + +- `GET /proxy/youtube/search?q=` (bearer): YouTube Data API v3 `search?part=snippet&type=video&maxResults=10&key=env.YOUTUBE_API_KEY`, + mapped to the frozen wire format above. 503 `{ error: 'youtube api not configured' }` + when the secret is missing. Cache API: `caches.default` keyed by normalized query, TTL + 6h (Cache-Control on the synthetic response). +- `GET /proxy/youtube/transcript?id=` (bearer): fetch + `https://www.youtube.com/api/timedtext?v=<id>&lang=pt` then fallback `&lang=en`; return + the raw XML as `text/xml` (client parses with `parseTimedText`); 404 when neither + exists. Cached 24h via Cache API. +- `GET /proxy/rss?url=` (bearer): fetch the url (https only, block private IPs/localhost), + return body with original content-type, cached 1h. +- Secrets: `YOUTUBE_API_KEY` (optional). Update env.ts, .dev.vars.example, README. +- Tests: bun + app.request with stubbed `globalThis.fetch` and a FakeCache injected — + Cache API isn't in bun: wrap cache access in a small `getCache()` seam module that + returns `caches.default` in workerd and an injectable stub in tests. + +## packages/db (stream B) + +```ts +// repo/content.ts +attachContent(db, deviceId, { topic_id, source, external_id?, url?, title, kind, meta_json? }): Promise<ContentItemRow> +listContentByTopic(db, topicId): Promise<ContentItemRow[]> +deleteContent(db, deviceId, id): Promise<void> + +// search.ts (local-only FTS5 — NOT in shared migrations: D1 lacks FTS5) +ensureSearchIndex(db): Promise<void> // CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(title, body, kind UNINDEXED, ref_id UNINDEXED) +reindexAll(db): Promise<void> // rebuild from topics (title+notes_md), cards (front/back), content_items (title) +searchLocal(db, q, limit?): Promise<{ kind: 'topic' | 'card' | 'content'; ref_id: string; title: string; snippet: string }[]> +// maintenance: repos do NOT write fts on every write in M4; the PWA calls reindexAll +// on app open + after imports/attach (cheap at this scale); document as M6 candidate. +// searchLocal uses fts5 MATCH with prefix (`q*`), bm25 order, snippet(). +``` + +PWA db worker calls `ensureSearchIndex` after migrate (stream D wires it). + +## UI (streams C and D) + +Stream C owns `/library/**` + `lib/stores/library*.svelte.ts`. Stream D owns the topic +content list inside `/tracks/[id]/` (new colocated component TopicContent.svelte mounted +by ONE line in the cards panel area), global search in the header (+layout.svelte — +max 15-line diff), and the FTS wiring in `lib/db/worker.ts` (one call) + +`lib/stores/search*.svelte.ts`. + +### /library (C) + +- `library-search-form`, `library-search-input`, `library-search-submit` +- Source filter chips `library-filter-all|youtube|wikipedia|stackexchange` (aria-pressed) +- Results `library-results`, `library-result` (title, source · kind badge, description), + per result `library-attach` (opens topic picker: `attach-picker`, `attach-track-select`, + `attach-topic-select`, `attach-confirm`) → attachContent +- Player: `/library/watch/[videoId]` — youtube-nocookie iframe embed (`video-player`), + transcript panel `transcript-panel` with `transcript-cue` rows (time + text); fetch + `/proxy/youtube/transcript?id=` (bearer token same as sync), parse with parseTimedText; + clicking a cue seeks via the YouTube iframe postMessage API (`seekTo`); transcript + missing → calm 'sem transcrição disponível.' `transcript-follow` toggle (auto-scroll). +- Search runs all enabled connectors in parallel (Promise.allSettled), tagged sections or + merged list grouped by source; empty: 'nada encontrado — tente outros termos.' + +### Topic content + global search (D) + +- TopicContent.svelte: `topic-content-list`, `topic-content-item` (title, source badge, + link — youtube items link to `/library/watch/<external_id>`, others open url in new tab + rel=noopener), `topic-content-remove`. Mounted below CardsPanel for the selected topic. +- Header global search: `global-search-input` (compact, right side of nav) with dropdown + `global-search-results` / `global-search-result` (kind badge + title; topic/card results + navigate to their track page, content to its link). Debounced 200ms over searchLocal. + Esc closes; keyboard navigable (arrow keys optional — at minimum tab-reachable). +- Wire `ensureSearchIndex` + `reindexAll` on db ready (in lib/db/client.ts getDb flow or + layout onMount — keep it one call site), and refresh index after outline import/attach + via reindexAll debounced on tables-changed ['topics','cards','content_items']. + +Copy pt-BR sentence case; badges text-only (`vídeo · youtube`); no icons/emoji; tokens only. +Token for auth'd proxy/transcript calls: reuse the sync token pattern (see lib/push/subscribe.ts getToken usage). diff --git a/packages/connectors/src/index.ts b/packages/connectors/src/index.ts index 2483ddb..f34c62c 100644 --- a/packages/connectors/src/index.ts +++ b/packages/connectors/src/index.ts @@ -1,7 +1,21 @@ -export interface Connector { - source: string; - search(q: string): Promise<unknown[]>; - resolve(externalId: string): Promise<unknown | null>; -} +import type { Connector } from './types'; +import { wikipediaConnector } from './wikipedia'; +import { stackexchangeConnector } from './stackexchange'; +import { youtubeConnector } from './youtube'; + +export type { Connector, ContentKind, ContentResult, FetchLike, TranscriptCue } from './types'; +export { parseTimedText, decodeEntities } from './timedtext'; +export { wikipediaConnector } from './wikipedia'; +export { stackexchangeConnector } from './stackexchange'; +export { youtubeConnector } from './youtube'; -export const registry: Map<string, Connector> = new Map(); +// Registry, stable order: wikipedia, stackexchange, youtube. +export const connectors: Connector[] = [ + wikipediaConnector, + stackexchangeConnector, + youtubeConnector, +]; + +export function getConnector(source: string): Connector | null { + return connectors.find((c) => c.source === source) ?? null; +} diff --git a/packages/connectors/src/platform.d.ts b/packages/connectors/src/platform.d.ts new file mode 100644 index 0000000..b84ed82 --- /dev/null +++ b/packages/connectors/src/platform.d.ts @@ -0,0 +1,27 @@ +// Minimal ambient declarations for the web platform globals this package +// relies on: tsconfig.base uses lib es2023 (no DOM) and this package has no +// @types/bun. At runtime these globals exist in browsers, workers and bun. +// Structural on purpose — consumers compiling with lib.dom (or @types/bun) +// resolve the real types instead and never load this file. + +interface RequestInit { + method?: string; + headers?: Record<string, string>; + body?: string; + signal?: unknown; +} + +interface Response { + readonly ok: boolean; + readonly status: number; + json(): Promise<unknown>; + text(): Promise<string>; +} + +declare var Response: { + prototype: Response; + new ( + body?: string | null, + init?: { status?: number; headers?: Record<string, string> }, + ): Response; +}; diff --git a/packages/connectors/src/stackexchange.ts b/packages/connectors/src/stackexchange.ts new file mode 100644 index 0000000..e209365 --- /dev/null +++ b/packages/connectors/src/stackexchange.ts @@ -0,0 +1,46 @@ +import { decodeEntities } from './timedtext'; +import { MAX_RESULTS, isRecord, type Connector, type ContentResult, type FetchLike } from './types'; + +const SEARCH_URL = + 'https://api.stackexchange.com/2.3/search/advanced?site=stackoverflow&order=desc&sort=relevance&q='; + +export const stackexchangeConnector: Connector = { + source: 'stackexchange', + kind: 'qa', + async search(q: string, fetchFn: FetchLike): Promise<ContentResult[]> { + try { + const res = await fetchFn(`${SEARCH_URL}${encodeURIComponent(q)}`); + if (!res.ok) return []; + const body: unknown = await res.json(); + if (!isRecord(body) || !Array.isArray(body.items)) return []; + const results: ContentResult[] = []; + for (const item of body.items) { + if (results.length >= MAX_RESULTS) break; + if ( + !isRecord(item) || + typeof item.question_id !== 'number' || + typeof item.title !== 'string' || + typeof item.link !== 'string' + ) { + continue; + } + results.push({ + source: 'stackexchange', + external_id: String(item.question_id), + url: item.link, + title: decodeEntities(item.title), + kind: 'qa', + description: null, + meta: { + score: typeof item.score === 'number' ? item.score : null, + answer_count: typeof item.answer_count === 'number' ? item.answer_count : null, + is_answered: typeof item.is_answered === 'boolean' ? item.is_answered : null, + }, + }); + } + return results; + } catch { + return []; + } + }, +}; diff --git a/packages/connectors/src/timedtext.ts b/packages/connectors/src/timedtext.ts new file mode 100644 index 0000000..711a326 --- /dev/null +++ b/packages/connectors/src/timedtext.ts @@ -0,0 +1,50 @@ +import type { TranscriptCue } from './types'; + +const NAMED_ENTITIES: Record<string, string> = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", +}; + +/** Decodes & < > " ' and numeric (' / ') html entities. */ +export function decodeEntities(input: string): string { + return input.replace( + /&(?:#(\d+)|#x([0-9a-fA-F]+)|([a-zA-Z]+));/g, + (match, dec: string | undefined, hex: string | undefined, name: string | undefined) => { + const codePoint = + dec !== undefined + ? Number.parseInt(dec, 10) + : hex !== undefined + ? Number.parseInt(hex, 16) + : null; + if (codePoint !== null) { + return Number.isFinite(codePoint) && codePoint <= 0x10ffff + ? String.fromCodePoint(codePoint) + : match; + } + return (name !== undefined ? NAMED_ENTITIES[name] : undefined) ?? match; + }, + ); +} + +const TEXT_ELEMENT_RE = /<text\b([^>]*)>([\s\S]*?)<\/text>/g; +const START_ATTR_RE = /\bstart="([^"]*)"/; +const DUR_ATTR_RE = /\bdur="([^"]*)"/; + +/** Parses youtube timedtext XML (`<text start="..." dur="...">escaped</text>`) into cues. */ +export function parseTimedText(xml: string): TranscriptCue[] { + const cues: TranscriptCue[] = []; + for (const match of xml.matchAll(TEXT_ELEMENT_RE)) { + const attrs = match[1] ?? ''; + const body = match[2] ?? ''; + const start = Number.parseFloat(START_ATTR_RE.exec(attrs)?.[1] ?? ''); + if (!Number.isFinite(start)) continue; + const dur = Number.parseFloat(DUR_ATTR_RE.exec(attrs)?.[1] ?? ''); + const text = decodeEntities(body).trim(); + if (text === '') continue; + cues.push({ start, dur: Number.isFinite(dur) ? dur : 0, text }); + } + return cues; +} diff --git a/packages/connectors/src/types.ts b/packages/connectors/src/types.ts new file mode 100644 index 0000000..6797e1d --- /dev/null +++ b/packages/connectors/src/types.ts @@ -0,0 +1,31 @@ +export type ContentKind = 'video' | 'article' | 'qa' | 'doc'; + +export interface ContentResult { + source: string; // 'youtube' | 'wikipedia' | 'stackexchange' + external_id: string; + url: string; + title: string; + kind: ContentKind; + description: string | null; + meta: Record<string, unknown>; // thumbnails, score, duration... source-specific +} + +export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>; + +export interface Connector { + source: string; + kind: ContentKind; + search(q: string, fetchFn: FetchLike): Promise<ContentResult[]>; // <= 10 results +} + +export interface TranscriptCue { + start: number; + dur: number; + text: string; +} + +export const MAX_RESULTS = 10; + +export function isRecord(value: unknown): value is Record<string, unknown> { + return typeof value === 'object' && value !== null; +} diff --git a/packages/connectors/src/wikipedia.ts b/packages/connectors/src/wikipedia.ts new file mode 100644 index 0000000..285f265 --- /dev/null +++ b/packages/connectors/src/wikipedia.ts @@ -0,0 +1,48 @@ +import { decodeEntities } from './timedtext'; +import { MAX_RESULTS, isRecord, type Connector, type ContentResult, type FetchLike } from './types'; + +const SEARCH_URL = + 'https://pt.wikipedia.org/w/api.php?action=query&list=search&format=json&origin=*&srsearch='; + +function stripTags(html: string): string { + return html.replace(/<[^>]*>/g, ''); +} + +export const wikipediaConnector: Connector = { + source: 'wikipedia', + kind: 'article', + async search(q: string, fetchFn: FetchLike): Promise<ContentResult[]> { + try { + const res = await fetchFn(`${SEARCH_URL}${encodeURIComponent(q)}`); + if (!res.ok) return []; + const body: unknown = await res.json(); + if (!isRecord(body) || !isRecord(body.query) || !Array.isArray(body.query.search)) { + return []; + } + const results: ContentResult[] = []; + for (const item of body.query.search) { + if (results.length >= MAX_RESULTS) break; + if (!isRecord(item) || typeof item.pageid !== 'number' || typeof item.title !== 'string') { + continue; + } + const snippet = typeof item.snippet === 'string' ? item.snippet : ''; + const description = decodeEntities(stripTags(snippet)).trim(); + results.push({ + source: 'wikipedia', + external_id: String(item.pageid), + url: `https://pt.wikipedia.org/wiki/${encodeURIComponent(item.title.replaceAll(' ', '_'))}`, + title: item.title, + kind: 'article', + description: description === '' ? null : description, + meta: { + snippet, + wordcount: typeof item.wordcount === 'number' ? item.wordcount : null, + }, + }); + } + return results; + } catch { + return []; + } + }, +}; diff --git a/packages/connectors/src/youtube.ts b/packages/connectors/src/youtube.ts new file mode 100644 index 0000000..27410ca --- /dev/null +++ b/packages/connectors/src/youtube.ts @@ -0,0 +1,40 @@ +import { MAX_RESULTS, isRecord, type Connector, type ContentResult, type FetchLike } from './types'; + +// App-relative proxy: the worker adds the API key (see docs/M4-CONTRACTS.md). +const SEARCH_URL = '/proxy/youtube/search?q='; + +export const youtubeConnector: Connector = { + source: 'youtube', + kind: 'video', + async search(q: string, fetchFn: FetchLike): Promise<ContentResult[]> { + try { + const res = await fetchFn(`${SEARCH_URL}${encodeURIComponent(q)}`); + if (!res.ok) return []; + const body: unknown = await res.json(); + if (!isRecord(body) || !Array.isArray(body.items)) return []; + const results: ContentResult[] = []; + for (const item of body.items) { + if (results.length >= MAX_RESULTS) break; + if (!isRecord(item) || typeof item.id !== 'string' || typeof item.title !== 'string') { + continue; + } + results.push({ + source: 'youtube', + external_id: item.id, + url: `https://www.youtube.com/watch?v=${item.id}`, + title: item.title, + kind: 'video', + description: null, + meta: { + channel: typeof item.channel === 'string' ? item.channel : null, + thumbnail: typeof item.thumbnail === 'string' ? item.thumbnail : null, + duration: typeof item.duration === 'string' ? item.duration : null, + }, + }); + } + return results; + } catch { + return []; + } + }, +}; diff --git a/packages/connectors/test/bun-test.d.ts b/packages/connectors/test/bun-test.d.ts index 6b5f4d5..0eef489 100644 --- a/packages/connectors/test/bun-test.d.ts +++ b/packages/connectors/test/bun-test.d.ts @@ -7,6 +7,12 @@ declare module 'bun:test' { toBe(expected: unknown): void; toEqual(expected: unknown): void; toThrow(expected?: string | RegExp): void; + toBeCloseTo(expected: number, numDigits?: number): void; + toBeGreaterThan(expected: number): void; + toBeGreaterThanOrEqual(expected: number): void; + toBeLessThan(expected: number): void; + toBeLessThanOrEqual(expected: number): void; + toBeNull(): void; } export function expect(actual: unknown): Expectation; } diff --git a/packages/connectors/test/registry.test.ts b/packages/connectors/test/registry.test.ts index 998248f..e19c2f6 100644 --- a/packages/connectors/test/registry.test.ts +++ b/packages/connectors/test/registry.test.ts @@ -1,18 +1,24 @@ import { expect, test } from 'bun:test'; -import { registry, type Connector } from '../src'; +import { + connectors, + getConnector, + stackexchangeConnector, + wikipediaConnector, + youtubeConnector, +} from '../src'; -test('registry starts empty and accepts connectors', async () => { - expect(registry.size).toBe(0); +test('registry has the stable order wikipedia, stackexchange, youtube', () => { + expect(connectors.map((c) => c.source)).toEqual(['wikipedia', 'stackexchange', 'youtube']); + expect(connectors.map((c) => c.kind)).toEqual(['article', 'qa', 'video']); +}); - const fake: Connector = { - source: 'fake', - search: () => Promise.resolve([]), - resolve: () => Promise.resolve(null), - }; - registry.set(fake.source, fake); - expect(registry.size).toBe(1); - expect(await registry.get('fake')?.search('query')).toEqual([]); +test('getConnector resolves each source to its connector', () => { + expect(getConnector('wikipedia')).toBe(wikipediaConnector); + expect(getConnector('stackexchange')).toBe(stackexchangeConnector); + expect(getConnector('youtube')).toBe(youtubeConnector); +}); - registry.delete(fake.source); - expect(registry.size).toBe(0); +test('getConnector returns null for unknown sources', () => { + expect(getConnector('vimeo')).toBeNull(); + expect(getConnector('')).toBeNull(); }); diff --git a/packages/connectors/test/stackexchange.test.ts b/packages/connectors/test/stackexchange.test.ts new file mode 100644 index 0000000..2d9aeef --- /dev/null +++ b/packages/connectors/test/stackexchange.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from 'bun:test'; +import { stackexchangeConnector } from '../src/stackexchange'; +import type { FetchLike } from '../src/types'; + +function stubFetch(body: string, status: number): FetchLike { + return () => Promise.resolve(new Response(body, { status })); +} + +function stubJson(body: unknown): FetchLike { + return stubFetch(JSON.stringify(body), 200); +} + +function rejectingFetch(message: string): FetchLike { + return () => Promise.reject(new Error(message)); +} + +function questionItem(i: number): Record<string, unknown> { + return { + tags: ['typescript'], + owner: { display_name: `user${i}` }, + is_answered: i % 2 === 0, + view_count: 1000 + i, + answer_count: i, + score: 10 + i, + creation_date: 1700000000 + i, + question_id: 5000 + i, + link: `https://stackoverflow.com/questions/${5000 + i}/some-question-${i}`, + title: `Question ${i}`, + }; +} + +test('stackexchange maps results, decodes html entities in titles and caps at 10', async () => { + const requested: string[] = []; + const items = Array.from({ length: 12 }, (_, i) => questionItem(i)); + items[0] = { + ...questionItem(0), + title: 'How to use "async" & await in a <script> tag's body?', + }; + const fixture = stubJson({ items, has_more: true, quota_max: 300, quota_remaining: 299 }); + const fetchFn: FetchLike = (url, init) => { + requested.push(url); + return fixture(url, init); + }; + + const results = await stackexchangeConnector.search('async await', fetchFn); + + expect(requested).toEqual([ + 'https://api.stackexchange.com/2.3/search/advanced?site=stackoverflow&order=desc&sort=relevance&q=async%20await', + ]); + expect(results.length).toBe(10); + expect(results[0]).toEqual({ + source: 'stackexchange', + external_id: '5000', + url: 'https://stackoverflow.com/questions/5000/some-question-0', + title: 'How to use "async" & await in a <script> tag\'s body?', + kind: 'qa', + description: null, + meta: { score: 10, answer_count: 0, is_answered: true }, + }); +}); + +test('stackexchange returns [] on non-2xx', async () => { + expect(await stackexchangeConnector.search('x', stubFetch('{"error_id":502}', 400))).toEqual([]); +}); + +test('stackexchange returns [] on malformed json', async () => { + expect(await stackexchangeConnector.search('x', stubFetch('not json at all', 200))).toEqual([]); +}); + +test('stackexchange returns [] on unexpected json shape', async () => { + expect(await stackexchangeConnector.search('x', stubJson({ items: { nope: true } }))).toEqual([]); +}); + +test('stackexchange returns [] when fetch throws', async () => { + expect(await stackexchangeConnector.search('x', rejectingFetch('offline'))).toEqual([]); +}); diff --git a/packages/connectors/test/timedtext.test.ts b/packages/connectors/test/timedtext.test.ts new file mode 100644 index 0000000..d85804d --- /dev/null +++ b/packages/connectors/test/timedtext.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from 'bun:test'; +import { decodeEntities, parseTimedText } from '../src/timedtext'; + +test('parseTimedText parses cues with named and numeric entities', () => { + const xml = `<?xml version="1.0" encoding="utf-8" ?><transcript> +<text start="0.16" dur="4.24">café & pão</text> +<text start="4.4" dur="2">"it's" <b>bold</b></text> +</transcript>`; + expect(parseTimedText(xml)).toEqual([ + { start: 0.16, dur: 4.24, text: 'café & pão' }, + { start: 4.4, dur: 2, text: '"it\'s" <b>bold</b>' }, + ]); +}); + +test('parseTimedText keeps multiline cue text', () => { + const xml = `<transcript><text start="1.5" dur="3.5">primeira linha +segunda linha</text></transcript>`; + expect(parseTimedText(xml)).toEqual([ + { start: 1.5, dur: 3.5, text: 'primeira linha\nsegunda linha' }, + ]); +}); + +test('parseTimedText handles attribute order variations and missing dur', () => { + const xml = + '<transcript><text dur="2.5" start="10">a</text><text start="12.5">b</text></transcript>'; + expect(parseTimedText(xml)).toEqual([ + { start: 10, dur: 2.5, text: 'a' }, + { start: 12.5, dur: 0, text: 'b' }, + ]); +}); + +test('parseTimedText skips cues without a valid start and empty cues', () => { + const xml = + '<transcript><text dur="2">sem start</text><text start="1" dur="2"> </text><text start="3" dur="1">ok</text></transcript>'; + expect(parseTimedText(xml)).toEqual([{ start: 3, dur: 1, text: 'ok' }]); +}); + +test('parseTimedText returns [] for empty or cue-less xml', () => { + expect(parseTimedText('')).toEqual([]); + expect( + parseTimedText('<?xml version="1.0" encoding="utf-8" ?><transcript></transcript>'), + ).toEqual([]); +}); + +test('decodeEntities decodes the supported set and leaves the rest untouched', () => { + expect(decodeEntities('&<>"''')).toBe("&<>\"''"); + expect(decodeEntities('AB')).toBe('AB'); + expect(decodeEntities('é &unknown; �')).toBe( + 'é &unknown; �', + ); +}); diff --git a/packages/connectors/test/wikipedia.test.ts b/packages/connectors/test/wikipedia.test.ts new file mode 100644 index 0000000..859f89b --- /dev/null +++ b/packages/connectors/test/wikipedia.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from 'bun:test'; +import { wikipediaConnector } from '../src/wikipedia'; +import type { FetchLike } from '../src/types'; + +function stubFetch(body: string, status: number): FetchLike { + return () => Promise.resolve(new Response(body, { status })); +} + +function stubJson(body: unknown): FetchLike { + return stubFetch(JSON.stringify(body), 200); +} + +function rejectingFetch(message: string): FetchLike { + return () => Promise.reject(new Error(message)); +} + +function searchItem(i: number): Record<string, unknown> { + return { + ns: 0, + title: `Resultado ${i}`, + pageid: 1000 + i, + size: 4321, + wordcount: 100 + i, + snippet: `trecho <span class="searchmatch">resultado</span> ${i}`, + timestamp: '2026-01-01T00:00:00Z', + }; +} + +test('wikipedia maps search results and caps at 10', async () => { + const requested: string[] = []; + const fixture = stubJson({ + batchcomplete: '', + query: { + searchinfo: { totalhits: 12 }, + search: Array.from({ length: 12 }, (_, i) => searchItem(i)), + }, + }); + const fetchFn: FetchLike = (url, init) => { + requested.push(url); + return fixture(url, init); + }; + + const results = await wikipediaConnector.search('fotossíntese clorofila', fetchFn); + + expect(requested).toEqual([ + 'https://pt.wikipedia.org/w/api.php?action=query&list=search&format=json&origin=*&srsearch=fotoss%C3%ADntese%20clorofila', + ]); + expect(results.length).toBe(10); + expect(results[0]).toEqual({ + source: 'wikipedia', + external_id: '1000', + url: 'https://pt.wikipedia.org/wiki/Resultado_0', + title: 'Resultado 0', + kind: 'article', + description: 'trecho resultado 0', + meta: { snippet: 'trecho <span class="searchmatch">resultado</span> 0', wordcount: 100 }, + }); +}); + +test('wikipedia decodes entities in description and keeps raw snippet in meta', async () => { + const fetchFn = stubJson({ + query: { + search: [ + { + title: 'Café', + pageid: 7, + wordcount: 50, + snippet: '<span class="searchmatch">café</span> & chá', + }, + ], + }, + }); + + const results = await wikipediaConnector.search('café', fetchFn); + expect(results.length).toBe(1); + // é is not in the supported set, stays as-is; & and numeric decode. + expect(results[0]?.description).toBe('café & chá'); + expect(results[0]?.meta['snippet']).toBe( + '<span class="searchmatch">café</span> & chá', + ); +}); + +test('wikipedia returns [] on non-2xx', async () => { + expect(await wikipediaConnector.search('x', stubFetch('server error', 500))).toEqual([]); +}); + +test('wikipedia returns [] on malformed json', async () => { + expect(await wikipediaConnector.search('x', stubFetch('<!doctype html>', 200))).toEqual([]); +}); + +test('wikipedia returns [] on unexpected json shape', async () => { + expect(await wikipediaConnector.search('x', stubJson({ query: { search: 'nope' } }))).toEqual([]); +}); + +test('wikipedia returns [] when fetch throws', async () => { + expect(await wikipediaConnector.search('x', rejectingFetch('network down'))).toEqual([]); +}); diff --git a/packages/connectors/test/youtube.test.ts b/packages/connectors/test/youtube.test.ts new file mode 100644 index 0000000..e191ae3 --- /dev/null +++ b/packages/connectors/test/youtube.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from 'bun:test'; +import { youtubeConnector } from '../src/youtube'; +import type { FetchLike } from '../src/types'; + +function stubFetch(body: string, status: number): FetchLike { + return () => Promise.resolve(new Response(body, { status })); +} + +function stubJson(body: unknown): FetchLike { + return stubFetch(JSON.stringify(body), 200); +} + +function rejectingFetch(message: string): FetchLike { + return () => Promise.reject(new Error(message)); +} + +test('youtube calls the app-relative proxy and maps the frozen wire format', async () => { + const requested: string[] = []; + const fixture = stubJson({ + items: [ + { + id: 'dQw4w9WgXcQ', + title: 'Aula 1 — introdução', + channel: 'Canal de Estudos', + thumbnail: 'https://i.ytimg.com/vi/dQw4w9WgXcQ/mqdefault.jpg', + duration: 'PT12M34S', + }, + { + id: 'abc123def45', + title: 'Aula 2', + channel: 'Canal de Estudos', + thumbnail: 'https://i.ytimg.com/vi/abc123def45/mqdefault.jpg', + duration: null, + }, + ], + }); + const fetchFn: FetchLike = (url, init) => { + requested.push(url); + return fixture(url, init); + }; + + const results = await youtubeConnector.search('fsrs revisão espaçada', fetchFn); + + expect(requested).toEqual(['/proxy/youtube/search?q=fsrs%20revis%C3%A3o%20espa%C3%A7ada']); + expect(results.length).toBe(2); + expect(results[0]).toEqual({ + source: 'youtube', + external_id: 'dQw4w9WgXcQ', + url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + title: 'Aula 1 — introdução', + kind: 'video', + description: null, + meta: { + channel: 'Canal de Estudos', + thumbnail: 'https://i.ytimg.com/vi/dQw4w9WgXcQ/mqdefault.jpg', + duration: 'PT12M34S', + }, + }); + expect(results[1]?.meta['duration']).toBeNull(); +}); + +test('youtube caps at 10 results', async () => { + const fetchFn = stubJson({ + items: Array.from({ length: 12 }, (_, i) => ({ + id: `video-id-${i}`, + title: `Vídeo ${i}`, + channel: 'Canal', + thumbnail: `https://i.ytimg.com/vi/video-id-${i}/mqdefault.jpg`, + duration: null, + })), + }); + const results = await youtubeConnector.search('x', fetchFn); + expect(results.length).toBe(10); +}); + +test('youtube returns [] on non-2xx (e.g. proxy 503 when unconfigured)', async () => { + const fetchFn = stubFetch('{"error":"youtube api not configured"}', 503); + expect(await youtubeConnector.search('x', fetchFn)).toEqual([]); +}); + +test('youtube returns [] on malformed json', async () => { + expect(await youtubeConnector.search('x', stubFetch('<html>oops</html>', 200))).toEqual([]); +}); + +test('youtube returns [] on unexpected json shape', async () => { + expect(await youtubeConnector.search('x', stubJson({ items: 'nope' }))).toEqual([]); +}); + +test('youtube returns [] when fetch throws', async () => { + expect(await youtubeConnector.search('x', rejectingFetch('fetch failed'))).toEqual([]); +}); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 7e89ed4..fb8d69d 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -15,6 +15,8 @@ export * from './repo/routines'; export * from './repo/targets'; export * from './repo/reminders'; export * from './repo/cycle'; +export * from './repo/content'; export * from './repo/stats-queries'; +export * from './search'; export * from './sync/apply'; export * from './sync/engine'; diff --git a/packages/db/src/repo/content.ts b/packages/db/src/repo/content.ts new file mode 100644 index 0000000..acad8a2 --- /dev/null +++ b/packages/db/src/repo/content.ts @@ -0,0 +1,76 @@ +import { newId, now, type ContentItemRow } from '@studyos/shared'; +import type { DbDriver, Row } from '../driver'; +import { localWrite } from './oplog'; +import { bumpedTs } from './ts'; + +export interface AttachContentInput { + topic_id: string; + source: string; + external_id?: string | null; + url?: string | null; + title: string; + kind: string; + meta_json?: string | null; +} + +function rowToContentItem(r: Row): ContentItemRow { + return { + id: r['id'] as string, + topic_id: (r['topic_id'] ?? null) as string | null, + source: r['source'] as string, + external_id: (r['external_id'] ?? null) as string | null, + url: (r['url'] ?? null) as string | null, + title: r['title'] as string, + kind: r['kind'] as string, + meta_json: (r['meta_json'] ?? null) as string | null, + added_at: r['added_at'] as number, + updated_at: r['updated_at'] as number, + deleted_at: (r['deleted_at'] ?? null) as number | null, + }; +} + +export async function attachContent( + db: DbDriver, + deviceId: string, + input: AttachContentInput, +): Promise<ContentItemRow> { + const ts = now(); + const item = { + id: newId(), + topic_id: input.topic_id, + source: input.source, + external_id: input.external_id ?? null, + url: input.url ?? null, + title: input.title, + kind: input.kind, + meta_json: input.meta_json ?? null, + added_at: ts, + updated_at: ts, + deleted_at: null, + } satisfies ContentItemRow; + await localWrite(db, 'content_items', item, deviceId); + return item; +} + +export async function getContent(db: DbDriver, id: string): Promise<ContentItemRow | null> { + const rows = await db.exec('SELECT * FROM content_items WHERE id = ?', [id]); + const r = rows[0]; + return r ? rowToContentItem(r) : null; +} + +export async function listContentByTopic(db: DbDriver, topicId: string): Promise<ContentItemRow[]> { + const rows = await db.exec( + 'SELECT * FROM content_items WHERE topic_id = ? AND deleted_at IS NULL ' + + 'ORDER BY added_at DESC, id DESC', + [topicId], + ); + return rows.map(rowToContentItem); +} + +/** Soft delete: on the wire this is an upsert with deleted_at set (docs/SYNC.md). */ +export async function deleteContent(db: DbDriver, deviceId: string, id: string): Promise<void> { + const existing = await getContent(db, id); + if (!existing || existing.deleted_at !== null) return; + const ts = bumpedTs(existing.updated_at); + await localWrite(db, 'content_items', { ...existing, deleted_at: ts, updated_at: ts }, deviceId); +} diff --git a/packages/db/src/search.ts b/packages/db/src/search.ts new file mode 100644 index 0000000..2c031df --- /dev/null +++ b/packages/db/src/search.ts @@ -0,0 +1,74 @@ +import type { DbDriver } from './driver'; + +export interface SearchHit { + kind: 'topic' | 'card' | 'content'; + ref_id: string; + title: string; + snippet: string; +} + +// Local-only FTS5 index. NOT in the shared migrations and NOT a synced table: +// D1 lacks FTS5, so each device builds its own index (ensureSearchIndex after +// migrate, reindexAll on app open / after imports). Incremental maintenance on +// every repo write is an M6 candidate — at M4 scale a full rebuild is cheap. +export async function ensureSearchIndex(db: DbDriver): Promise<void> { + await db.exec( + 'CREATE VIRTUAL TABLE IF NOT EXISTS search_index ' + + 'USING fts5(title, body, kind UNINDEXED, ref_id UNINDEXED)', + ); +} + +/** Full rebuild from topics, cards and content_items, in one atomic batch. */ +export async function reindexAll(db: DbDriver): Promise<void> { + await db.batch([ + { sql: 'DELETE FROM search_index' }, + { + sql: + 'INSERT INTO search_index (title, body, kind, ref_id) ' + + "SELECT title, coalesce(notes_md, ''), 'topic', id FROM topics WHERE deleted_at IS NULL", + }, + { + sql: + 'INSERT INTO search_index (title, body, kind, ref_id) ' + + "SELECT front_md, coalesce(back_md, ''), 'card', id FROM cards WHERE deleted_at IS NULL", + }, + { + sql: + 'INSERT INTO search_index (title, body, kind, ref_id) ' + + "SELECT title, '', 'content', id FROM content_items WHERE deleted_at IS NULL", + }, + ]); +} + +// Strip fts5 query syntax so user input can never be parsed as operators. +// Each surviving token is emitted as a quoted prefix term ("tok"*) — quoting +// neutralizes leftover punctuation and keywords like OR/NOT/NEAR. +function toMatchExpr(q: string): string | null { + const cleaned = q + .replace(/["*():^-]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (cleaned === '') return null; + return cleaned + .split(' ') + .map((tok) => `"${tok}"*`) + .join(' '); +} + +export async function searchLocal(db: DbDriver, q: string, limit = 20): Promise<SearchHit[]> { + const match = toMatchExpr(q); + if (match === null) return []; + const rows = await db.exec( + 'SELECT kind, ref_id, title, ' + + "snippet(search_index, 1, '[', ']', '…', 8) AS snippet " + + 'FROM search_index WHERE search_index MATCH ? ' + + 'ORDER BY bm25(search_index) LIMIT ?', + [match, limit], + ); + return rows.map((r) => ({ + kind: r['kind'] as SearchHit['kind'], + ref_id: r['ref_id'] as string, + title: r['title'] as string, + snippet: r['snippet'] as string, + })); +} diff --git a/packages/db/test/repo-content.test.ts b/packages/db/test/repo-content.test.ts new file mode 100644 index 0000000..c3e7e97 --- /dev/null +++ b/packages/db/test/repo-content.test.ts @@ -0,0 +1,99 @@ +import { expect, test } from 'bun:test'; +import { attachContent, deleteContent, getContent, listContentByTopic } from '../src/repo/content'; +import { createTopic } from '../src/repo/topics'; +import { createTrack } from '../src/repo/tracks'; +import { freshDb } from './load-migrations'; + +const DEVICE = 'device-test'; + +test('attachContent writes the item and exactly one oplog row', async () => { + const db = await freshDb(); + const track = await createTrack(db, DEVICE, { title: 't' }); + const topic = await createTopic(db, DEVICE, { track_id: track.id, title: 'x' }); + + const item = await attachContent(db, DEVICE, { + topic_id: topic.id, + source: 'youtube', + external_id: 'dQw4w9WgXcQ', + url: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + title: 'Aula 1', + kind: 'video', + meta_json: '{"duration":"PT10M"}', + }); + expect(item.added_at).toBe(item.updated_at); + expect(item.deleted_at).toBeNull(); + + const rows = await db.exec('SELECT * FROM content_items'); + expect(rows.length).toBe(1); + expect(rows[0]?.['title']).toBe('Aula 1'); + expect(rows[0]?.['source']).toBe('youtube'); + + const ops = await db.exec("SELECT * FROM oplog WHERE tbl = 'content_items'"); + expect(ops.length).toBe(1); + expect(ops[0]?.['row_id']).toBe(item.id); + expect(ops[0]?.['op']).toBe('upsert'); + expect(ops[0]?.['synced']).toBe(0); + expect(JSON.parse(ops[0]?.['payload'] as string)).toEqual({ ...item }); +}); + +test('attachContent defaults optional fields to null', async () => { + const db = await freshDb(); + const track = await createTrack(db, DEVICE, { title: 't' }); + const topic = await createTopic(db, DEVICE, { track_id: track.id, title: 'x' }); + + const item = await attachContent(db, DEVICE, { + topic_id: topic.id, + source: 'wikipedia', + title: 'Artigo', + kind: 'article', + }); + expect(item.external_id).toBeNull(); + expect(item.url).toBeNull(); + expect(item.meta_json).toBeNull(); + expect(await getContent(db, item.id)).toEqual(item); +}); + +test('listContentByTopic filters by topic, hides soft-deleted and orders newest first', async () => { + const db = await freshDb(); + const track = await createTrack(db, DEVICE, { title: 't' }); + const t1 = await createTopic(db, DEVICE, { track_id: track.id, title: 't1' }); + const t2 = await createTopic(db, DEVICE, { track_id: track.id, title: 't2' }); + + const base = { source: 'youtube', kind: 'video' }; + const a = await attachContent(db, DEVICE, { ...base, topic_id: t1.id, title: 'a' }); + const b = await attachContent(db, DEVICE, { ...base, topic_id: t1.id, title: 'b' }); + const c = await attachContent(db, DEVICE, { ...base, topic_id: t1.id, title: 'c' }); + await attachContent(db, DEVICE, { ...base, topic_id: t2.id, title: 'elsewhere' }); + + // deterministic timestamps: a oldest, c newest + for (const [id, ts] of [ + [a.id, 1000], + [b.id, 2000], + [c.id, 3000], + ] as const) { + await db.exec('UPDATE content_items SET added_at = ? WHERE id = ?', [ts, id]); + } + await deleteContent(db, DEVICE, b.id); + + expect((await listContentByTopic(db, t1.id)).map((i) => i.id)).toEqual([c.id, a.id]); +}); + +test('deleteContent soft-deletes, appends a second oplog row and is idempotent', async () => { + const db = await freshDb(); + const track = await createTrack(db, DEVICE, { title: 't' }); + const topic = await createTopic(db, DEVICE, { track_id: track.id, title: 'x' }); + const item = await attachContent(db, DEVICE, { + topic_id: topic.id, + source: 'stackexchange', + title: 'q', + kind: 'qa', + }); + + await deleteContent(db, DEVICE, item.id); + await deleteContent(db, DEVICE, item.id); // already deleted: no-op + await deleteContent(db, DEVICE, 'missing'); // unknown id: no-op + + const rows = await db.exec('SELECT deleted_at FROM content_items WHERE id = ?', [item.id]); + expect(rows[0]?.['deleted_at']).not.toBeNull(); + expect((await db.exec("SELECT * FROM oplog WHERE tbl = 'content_items'")).length).toBe(2); +}); diff --git a/packages/db/test/search.test.ts b/packages/db/test/search.test.ts new file mode 100644 index 0000000..0e929b9 --- /dev/null +++ b/packages/db/test/search.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from 'bun:test'; +import type { DbDriver } from '../src/driver'; +import { attachContent } from '../src/repo/content'; +import { createCard, deleteCard } from '../src/repo/cards'; +import { createTopic } from '../src/repo/topics'; +import { createTrack } from '../src/repo/tracks'; +import { ensureSearchIndex, reindexAll, searchLocal } from '../src/search'; +import { freshDb } from './load-migrations'; + +const DEVICE = 'device-test'; + +async function seededDb(): Promise<{ + db: DbDriver; + topicId: string; + cardId: string; + contentId: string; +}> { + const db = await freshDb(); + await ensureSearchIndex(db); + const track = await createTrack(db, DEVICE, { title: 'concurso' }); + const topic = await createTopic(db, DEVICE, { + track_id: track.id, + title: 'direito constitucional', + notes_md: 'controle de constitucionalidade', + }); + const card = await createCard(db, DEVICE, { + topic_id: topic.id, + front_md: 'o que é habeas corpus?', + back_md: 'remédio constitucional contra prisão ilegal', + }); + const content = await attachContent(db, DEVICE, { + topic_id: topic.id, + source: 'youtube', + title: 'aula de estatística descritiva', + kind: 'video', + }); + await reindexAll(db); + return { db, topicId: topic.id, cardId: card.id, contentId: content.id }; +} + +test('ensureSearchIndex is idempotent', async () => { + const db = await freshDb(); + await ensureSearchIndex(db); + await ensureSearchIndex(db); + expect(await searchLocal(db, 'anything')).toEqual([]); +}); + +test('reindexAll indexes topics, cards and content_items', async () => { + const { db, topicId, cardId, contentId } = await seededDb(); + + const topics = await searchLocal(db, 'constitucional'); + expect(topics.some((h) => h.kind === 'topic' && h.ref_id === topicId)).toBe(true); + + const cards = await searchLocal(db, 'habeas'); + expect(cards.map((h) => ({ kind: h.kind, ref_id: h.ref_id }))).toEqual([ + { kind: 'card', ref_id: cardId }, + ]); + + const content = await searchLocal(db, 'estatística'); + expect(content.map((h) => ({ kind: h.kind, ref_id: h.ref_id }))).toEqual([ + { kind: 'content', ref_id: contentId }, + ]); +}); + +test('matches by prefix: constitu finds constitucional', async () => { + const { db, topicId } = await seededDb(); + const hits = await searchLocal(db, 'constitu'); + expect(hits.some((h) => h.ref_id === topicId)).toBe(true); +}); + +test('snippet marks the match in the body column', async () => { + const { db, cardId } = await seededDb(); + const hit = (await searchLocal(db, 'remédio')).find((h) => h.ref_id === cardId); + expect(hit?.title).toBe('o que é habeas corpus?'); + expect(hit?.snippet).toContain('[remédio]'); +}); + +test('hostile input is sanitized, never a syntax error', async () => { + const { db } = await seededDb(); + expect(await searchLocal(db, 'a" OR 1=1 --')).toBeInstanceOf(Array); + expect(await searchLocal(db, '(constitu* AND NOT) ^ "')).toBeInstanceOf(Array); + expect(await searchLocal(db, '"" ** (((')).toEqual([]); + expect(await searchLocal(db, ' ')).toEqual([]); +}); + +test('soft-deleted rows disappear after reindexAll', async () => { + const { db, cardId } = await seededDb(); + expect((await searchLocal(db, 'habeas')).length).toBe(1); + + await deleteCard(db, DEVICE, cardId); + await reindexAll(db); + expect(await searchLocal(db, 'habeas')).toEqual([]); +}); + +test('bm25 order: title hit ranks above body hit for the same term', async () => { + const db = await freshDb(); + await ensureSearchIndex(db); + const track = await createTrack(db, DEVICE, { title: 't' }); + const inBody = await createTopic(db, DEVICE, { + track_id: track.id, + title: 'tema geral de estudos', + notes_md: 'este resumo menciona hermenêutica entre vários outros assuntos', + }); + const inTitle = await createTopic(db, DEVICE, { + track_id: track.id, + title: 'hermenêutica', + }); + await reindexAll(db); + + const hits = await searchLocal(db, 'hermenêutica'); + expect(hits.map((h) => h.ref_id)).toEqual([inTitle.id, inBody.id]); +}); + +test('respects the limit parameter', async () => { + const db = await freshDb(); + await ensureSearchIndex(db); + const track = await createTrack(db, DEVICE, { title: 't' }); + for (let i = 0; i < 5; i++) { + await createTopic(db, DEVICE, { track_id: track.id, title: `revisão ${i}` }); + } + await reindexAll(db); + expect((await searchLocal(db, 'revisão', 3)).length).toBe(3); +});