From 742232e25f18bb5776a0e81345fee152752a59e3 Mon Sep 17 00:00:00 2001 From: "Xunnamius (Romulus)" Date: Thu, 1 Jun 2023 22:49:46 -0700 Subject: [PATCH] refactor: 566/566 all test suites passing --- external-scripts/initialize-data/index.ts | 1129 +-------------------- external-scripts/prune-data.ts | 81 +- lib/mongo-item/unit.test.ts | 36 +- test/externals/unit-initialize.test.ts | 244 +---- test/externals/unit-prune.test.ts | 60 +- test/integration.ts | 24 +- 6 files changed, 163 insertions(+), 1411 deletions(-) diff --git a/external-scripts/initialize-data/index.ts b/external-scripts/initialize-data/index.ts index edc1b41..b231893 100644 --- a/external-scripts/initialize-data/index.ts +++ b/external-scripts/initialize-data/index.ts @@ -1,39 +1,14 @@ /* eslint-disable unicorn/no-process-exit */ /* eslint-disable no-await-in-loop */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { - AppError, - GuruMeditationError, - HttpError, - InvalidAppEnvironmentError -} from 'named-app-errors'; +import { AppError } from 'named-app-errors'; -import jsonFile from 'jsonfile'; -import { type AnyBulkWriteOperation, ObjectId } from 'mongodb'; -import { toss } from 'toss-expression'; -import { setTimeout as wait } from 'node:timers/promises'; import inquirer, { type PromptModule } from 'inquirer'; -import { decode as decodeEntities } from 'html-entities'; import { debugNamespace as namespace } from 'universe/constants'; import { getEnv } from 'universe/backend/env'; -import { itemToObjectId } from 'multiverse/mongo-item'; -import { RequestQueue } from 'multiverse/throttled-fetch'; import { debugFactory } from 'multiverse/debug-extended'; import { getDb } from 'multiverse/mongo-schema'; -import { hydrateDb } from 'multiverse/mongo-test'; - -import { dummyAppData } from 'testverse/db'; - -import { deriveKeyFromPassword } from 'externals/initialize-data/crypto'; - -import type { - InternalUser, - InternalInfo, - InternalPage, - InternalSession -} from 'universe/backend/db'; const debugNamespace = `${namespace}:initialize-data`; @@ -43,11 +18,6 @@ const logOrDebug = () => { return log.enabled ? log : debug; }; -/** - * Represents one second in milliseconds. - */ -const oneSecondInMs = 1000; - // eslint-disable-next-line no-console log.log = console.info.bind(console); @@ -83,16 +53,6 @@ const getPrompter = (testPrompterParams?: string): { prompt: PromptModule } => { : inquirer; }; -const commitRootDataToDb = async (data: Data | null) => { - if (!data) { - throw new GuruMeditationError('cannot commit null data'); - } - - logOrDebug()('committing dummy root data to database via hydration'); - - await hydrateDb({ name: 'root' }); -}; - /** * Setups up a database from scratch by creating collections (only if they do * not already exist) and populating them with a large amount of data. Suitable @@ -102,1062 +62,46 @@ const commitRootDataToDb = async (data: Data | null) => { * never overwritten or deleted) */ const invoked = async () => { - // TODO: logOrDebug()(`inserted ${mailResult.insertedCount} mail documents`); - // TODO: logOrDebug()(`inserted ${questionsResult.insertedCount} question documents`); - // TODO: logOrDebug()(`upserted ${usersResult.nUpserted}, updated ${usersResult.nModified} user documents`); - await getPrompter(process.env.TEST_PROMPTER_INITIALIZER) - .prompt<{ action: string; token: string }>([ - { - name: 'action', - message: 'select an initializer action', - type: 'list', - choices: [ - ...(typeof data?.cache.complete == 'boolean' - ? [ - { - name: 'reinterrogate API using cache', - value: 'hit' - }, - { - name: 'reinterrogate API using cache and a custom token', - value: 'hit-custom' - }, - { - name: 'ignore cache and reinterrogate API', - value: 'hit-ignore' - }, - { - name: 'ignore cache and reinterrogate API using a custom token', - value: 'hit-ignore-custom' - }, - { - name: data?.cache.complete - ? 'commit completed cache data to database' - : 'commit incomplete cache data to database', - value: 'commit' - } - ] - : [ - { - name: 'interrogate API', - value: 'hit' - }, - { - name: 'interrogate API using a custom token', - value: 'hit-custom' - } - ]), - { name: 'exit', value: 'exit' } - ] - }, - { - name: 'token', - message: 'enter new token', - type: 'input', - when: (answers) => answers.action.includes('custom') - } - ]) - .then(async (answers) => { - if (answers.action == 'exit') { - logOrDebug()('execution complete'); - process.exit(0); - } - - if (answers.action.startsWith('commit')) { - await commitApiDataToDb(data); - - logOrDebug()('execution complete'); - process.exit(0); - } - - interrogateApi = answers.action.startsWith('hit'); - usedStackExAuthKey = answers.token ?? usedStackExAuthKey; - - if (answers.action.startsWith('hit-ignore')) { - logOrDebug()('cache ignored'); - data = null; - } - }); - - if (interrogateApi) { - logOrDebug()('interrogating StackExchange API...'); - - data = - data?.cache?.complete === false - ? data - : { - questions: [], - users: [], - cache: { - complete: false - } - }; - - const { keyString, saltString } = await deriveKeyFromPassword('password'); - - const addOrUpdateUser = async ( - username: string, - { - points, - newQuestionId: newQuestionId, - newAnswerId: newAnswerId - }: { points: number; newQuestionId?: QuestionId; newAnswerId?: AnswerId } - ) => { - let user = data!.users.find((u) => u.username == username); - - if (!user) { - user = { - _id: new ObjectId(), - username, - email: `${username}@fake-email.com`, - salt: saltString, - key: keyString, - points, - questionIds: [], - answerIds: [] - }; - - data!.users.push(user); - } - - if (newAnswerId) { - if (!newQuestionId) { - throw new GuruMeditationError( - 'localUpsertUser cannot use answerId without corresponding questionId' - ); - } - - user.answerIds.push([newQuestionId, newAnswerId]); - } else if (newQuestionId) { - user.questionIds.push(newQuestionId); - } - - return user; - }; - - const endpointBackoffs: Record = {}; - - // * max 30req/1000ms https://api.stackexchange.com/docs/throttle - const queue = new RequestQueue({ - intervalPeriodMs, - maxRequestsPerInterval, - // requestInspector: dummyRequestInspector - fetchErrorInspector: ({ - error: error_, - queue: q, - requestInfo, - requestInit, - state - }) => { - const subDebug = logOrDebug().extend('err'); - - const apiEndpoint = - (state.apiEndpoint as string) || - toss(new GuruMeditationError('missing apiEndpoint metadata')); - - const tries = (state.tries as number) ?? 1; - const retriedTooManyTimes = tries >= maxRequestRetries; - - subDebug( - `handling fetch error for [${apiEndpoint}] (try #${tries}): ${requestInfo}` - ); - - if (retriedTooManyTimes) { - subDebug.warn('request retried too many times'); - throw new HttpError(`${error_}`); - } - - subDebug(`requeuing errored request: ${requestInfo}`); - return q.addRequestToQueue(requestInfo, requestInit, { - ...state, - rescheduled: true, - tries: tries + 1 - }); - }, - requestInspector: ({ queue: q, requestInfo, requestInit, state }) => { - const subDebug = logOrDebug().extend('req'); - - const apiEndpoint = - (state.apiEndpoint as string) || - toss(new GuruMeditationError('missing apiEndpoint metadata')); - const tries = (state.tries as number) ?? 1; - - subDebug(`saw [${apiEndpoint}] (try #${tries}): ${requestInfo}`); - - if ( - typeof endpointBackoffs[apiEndpoint] == 'number' && - Date.now() <= endpointBackoffs[apiEndpoint] - ) { - subDebug.warn( - `request will be delayed until ${new Date( - endpointBackoffs[apiEndpoint] - ).toLocaleString()}` - ); - - return wait( - Math.max(0, endpointBackoffs[apiEndpoint] - Date.now()), - undefined, - { - signal: requestInit.signal as AbortSignal - } - ).then(() => { - subDebug(`requeuing delayed request: ${requestInfo}`); - return q.addRequestToQueue(requestInfo, requestInit, { - ...state, - rescheduled: true - }); - }); - } - }, - responseInspector: async ({ - response, - queue: q, - requestInfo, - requestInit, - state - }) => { - const subDebug = logOrDebug().extend('res'); - const res = response as Response; - - if (state.rescheduled as boolean) { - subDebug(`passing through rescheduled request to ${requestInfo}`); - return res; - } else { - const apiEndpoint = - (state.apiEndpoint as string) || - toss(new GuruMeditationError('missing apiEndpoint metadata')); - const tries = (state.tries as number) ?? 1; - const retriedTooManyTimes = tries >= maxRequestRetries; - - const json: Partial> = await res - .json() - .catch(() => ({})); - - subDebug(`saw [${apiEndpoint}] (try #${tries}): ${requestInfo}`); - - if (retriedTooManyTimes) { - subDebug.warn('request retried too many times'); - } - - if (json.backoff) { - endpointBackoffs[apiEndpoint] = - Date.now() + json.backoff * oneSecondInMs; - - subDebug.warn( - `saw backoff demand in response, delaying further requests to endpoint "${apiEndpoint}" until ${new Date( - endpointBackoffs[apiEndpoint] - ).toLocaleString()}` - ); - } - - if (retriedTooManyTimes && json.error_message) { - throw new HttpError(res, json.error_message); - } else if (res.ok) { - return json; - } else { - if (retriedTooManyTimes || (res.status < 500 && res.status != 429)) { - throw new HttpError(res); - } else { - if (res.status == 502 || res.status == 429) { - subDebug.warn( - `rate limited detected (${res.status}), delaying next interval request processing for ${delayAfterRequestRateLimitMs}ms before requeuing` - ); - q.delayRequestProcessingByMs(delayAfterRequestRateLimitMs); - } else { - subDebug.warn( - `a server error occurred (${res.status}), delaying next interval request processing for ${delayAfterRequestErrorMs}ms before requeuing` - ); - q.delayRequestProcessingByMs(delayAfterRequestErrorMs); - } - - return q.addRequestToQueue(requestInfo, requestInit, { - ...state, - tries: tries + 1 - }); - } - } - } - } - }); - - queue.beginProcessingRequestQueue(); - - try { - while (interrogateApi) { - interrogateApi = false; - const api = getApi(usedStackExAuthKey); - - try { - if (process.env.TEST_SKIP_REQUESTS) { - debug.warn( - 'saw debug env var TEST_SKIP_REQUESTS. No requests will be made' - ); - debug('queue stats: %O', queue.getStats()); - } else { - await Promise.all([ - (async () => { - for ( - let questionPage = 1; - data.questions.length < desiredApiGeneratedQuestionsCount; - ++questionPage - ) { - const questions = await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.questions({ - page: questionPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'questions' } - ); - - logOrDebug()( - `remaining questions wanted: ${ - desiredApiGeneratedQuestionsCount - data.questions.length - }` - ); - - await Promise.all( - questions.items.map(async (question, questionIndex) => { - const dataQuestionsLength = data!.questions.length; - const questionAlreadyCached = - data!.cache.complete === false && - data!.questions.find((q) => question.title == q.title); - - if (questionAlreadyCached) { - logOrDebug()( - `retrieved question #${ - dataQuestionsLength + questionIndex + 1 - } directly from local cache: ${question.title}` - ); - } else if ( - question.body.length > maxQuestionBodyLength || - question.title.length > maxQuestionTitleLength - ) { - logOrDebug()( - `skipped question #${ - dataQuestionsLength + questionIndex + 1 - } for violating length constraints: ${question.title}` - ); - } else { - const newQuestionId = new ObjectId(); - const [questionUpvotes, questionDownvotes] = - getUpvotesDownvotesFromScore(question.score); - - const answerItems: InternalAnswer[] = []; - const questionCommentItems: InternalComment[] = []; - - logOrDebug()( - `received question #${ - dataQuestionsLength + questionIndex + 1 - } from API: ${question.title}` - ); - - await Promise.all([ - (async () => { - for ( - let answerPage = 1, shouldContinue = true; - shouldContinue; - ++answerPage - ) { - const answers = await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.questions.answers({ - question_id: question.question_id, - page: answerPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'questions.answers' } - ); - - const answerItemsLength = answerItems.length; - - logOrDebug()( - `received ${answers.items.length} of (estimated) ${ - question.answer_count - } answers for question #${ - dataQuestionsLength + questionIndex + 1 - } from API` - ); - - await Promise.all( - answers.items.map(async (answer, answerIndex) => { - if (answer.body.length <= maxAnswerLength) { - const newAnswerId = new ObjectId(); - const [answerUpvotes, answerDownvotes] = - getUpvotesDownvotesFromScore(answer.score); - const answerCommentItems: InternalComment[] = - []; - - for ( - let answerCommentPage = 1, - subShouldContinue = true; - subShouldContinue; - ++answerCommentPage - ) { - const comments = - await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.answers.comments({ - answer_id: answer.answer_id, - page: answerCommentPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'answers.comments' } - ); - - const answerCommentItemsLength = - answerCommentItems.length; - - logOrDebug()( - `received ${ - comments.items.length - } comments for answer #${ - answerItemsLength + answerIndex + 1 - } of question #${ - dataQuestionsLength + questionIndex + 1 - } from API` - ); - - await Promise.all( - comments.items.map(async (comment) => { - if ( - comment.body.length <= maxCommentLength - ) { - const [ - commentUpvotes, - commentDownvotes - ] = getUpvotesDownvotesFromScore( - comment.score - ); - - answerCommentItems.push({ - _id: new ObjectId(), - creator: comment.owner.display_name, - createdAt: - comment.creation_date * - oneSecondInMs, - text: comment.body, - upvotes: commentUpvotes, - upvoterUsernames: [], - downvotes: commentDownvotes, - downvoterUsernames: [] - }); - - await addOrUpdateUser( - comment.owner.display_name, - { points: comment.owner.reputation } - ); - } - }) - ); - - logOrDebug()( - `added ${ - answerCommentItems.length - - answerCommentItemsLength - } of ${ - comments.items.length - } received comments to answer #${ - answerItemsLength + answerIndex + 1 - } of question #${ - dataQuestionsLength + questionIndex + 1 - }` - ); - - logOrDebug()( - `---\nthere are now ${ - answerCommentItems.length - } total comments added to answer #${ - answerItemsLength + answerIndex + 1 - } of question #${ - dataQuestionsLength + questionIndex + 1 - }\n---` - ); - - subShouldContinue = comments.has_more; - } - - answerItems.push({ - _id: newAnswerId, - creator: answer.owner.display_name, - createdAt: - answer.creation_date * oneSecondInMs, - text: answer.body, - upvotes: answerUpvotes, - upvoterUsernames: [], - downvotes: answerDownvotes, - downvoterUsernames: [], - accepted: answer.is_accepted, - commentItems: answerCommentItems - }); - - logOrDebug()( - `added answer ${ - answerItemsLength + answerIndex + 1 - } to question #${ - dataQuestionsLength + questionIndex + 1 - }` - ); - - logOrDebug()( - `---\nthere are now ${ - answerItems.length - } total answers added to question #${ - dataQuestionsLength + questionIndex + 1 - }\n---` - ); - - await addOrUpdateUser( - answer.owner.display_name, - { - points: answer.owner.reputation, - newQuestionId, - newAnswerId - } - ); - } else { - logOrDebug()( - `skipped answer #${ - answerItemsLength + answerIndex + 1 - } of question #${ - dataQuestionsLength + questionIndex + 1 - } for violating length constraints` - ); - } - }) - ); - - shouldContinue = answers.has_more; - } - })(), - (async () => { - for ( - let commentPage = 1, shouldContinue = true; - shouldContinue; - ++commentPage - ) { - const comments = await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.questions.comments({ - question_id: question.question_id, - page: commentPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'questions.comments' } - ); - - const questionCommentItemsLength = - questionCommentItems.length; - - logOrDebug()( - `received ${ - comments.items.length - } comments for question #${ - dataQuestionsLength + questionIndex + 1 - } from API` - ); - - await Promise.all( - comments.items.map(async (comment) => { - if (comment.body.length <= maxCommentLength) { - const [commentUpvotes, commentDownvotes] = - getUpvotesDownvotesFromScore(comment.score); - - questionCommentItems.push({ - _id: new ObjectId(), - creator: comment.owner.display_name, - createdAt: - comment.creation_date * oneSecondInMs, - text: comment.body, - upvotes: commentUpvotes, - upvoterUsernames: [], - downvotes: commentDownvotes, - downvoterUsernames: [] - }); - - await addOrUpdateUser( - comment.owner.display_name, - { points: comment.owner.reputation } - ); - } - }) - ); - - logOrDebug()( - `added ${ - questionCommentItems.length - - questionCommentItemsLength - } of ${ - comments.items.length - } received comments to question #${ - dataQuestionsLength + questionIndex + 1 - }` - ); - - logOrDebug()( - `---\nthere are now ${ - questionCommentItems.length - } total comments added to question #${ - dataQuestionsLength + questionIndex + 1 - }\n---` - ); - - shouldContinue = comments.has_more; - } - })() - ]); - - data!.questions.push({ - _id: newQuestionId, - title: question.title, - 'title-lowercase': question.title.toLowerCase(), - creator: question.owner.display_name, - createdAt: question.creation_date * oneSecondInMs, - status: 'open', - text: question.body, - upvotes: questionUpvotes, - upvoterUsernames: [], - downvotes: questionDownvotes, - downvoterUsernames: [], - hasAcceptedAnswer: answerItems.some( - (answer) => answer.accepted - ), - views: question.view_count, - answers: answerItems.length, - answerItems, - comments: questionCommentItems.length, - commentItems: questionCommentItems, - sorter: { - uvc: - questionUpvotes + - question.view_count + - questionCommentItems.length, - uvac: - questionUpvotes + - question.view_count + - answerItems.length + - questionCommentItems.length - } - }); - - logOrDebug()( - `added question #${ - dataQuestionsLength + questionIndex + 1 - } to cache` - ); - - logOrDebug().extend('cached')( - `\n>>> there are now ${ - data!.questions.length - } total of ${desiredApiGeneratedQuestionsCount} wanted questions the cache <<<\n\n` - ); - - await addOrUpdateUser(question.owner.display_name, { - points: question.owner.reputation, - newQuestionId - }); - - await cacheDataToDisk(data); - } - }) - ); - - if (!questions.has_more) { - logOrDebug().warn( - 'somehow exhausted all questions in the StackExchange API?!' - ); - break; - } - } - })(), - (async () => { - const newQuestionId = new ObjectId(); - let questionCreatedAt = Date.now(); - const randomAnswers: InternalAnswer[] = []; - const randomComments: InternalComment[] = []; - const totalDesiredComments = - collectAllQuestionCommentsCount + - collectAllFirstAnswerCommentsCount; - - const questionAlreadyCached = - data.cache.complete === false && !!data.catchallQuestion; - - if (questionAlreadyCached) { - logOrDebug()( - 'retrieved catch-all question directly from local cache' - ); - } else { - await Promise.all([ - (async () => { - for ( - let answerPage = 1; - randomAnswers.length < collectAllQuestionAnswersCount; - ++answerPage - ) { - const answers = await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.answers({ - page: answerPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'answers' } - ); - - const randomAnswersLength = randomAnswers.length; - - logOrDebug()( - `received ${answers.items.length} random answers for catch-all question from API` - ); - - await Promise.all( - answers.items.map(async (answer) => { - if (answer.body.length <= maxAnswerLength) { - const createdAt = - answer.creation_date * oneSecondInMs; - - questionCreatedAt = - createdAt < questionCreatedAt - ? createdAt - : questionCreatedAt; - - const newAnswerId = new ObjectId(); - const [answerUpvotes, answerDownvotes] = - getUpvotesDownvotesFromScore(answer.score); - - randomAnswers.push({ - _id: newAnswerId, - creator: answer.owner.display_name, - createdAt, - text: answer.body, - upvotes: answerUpvotes, - upvoterUsernames: [], - downvotes: answerDownvotes, - downvoterUsernames: [], - commentItems: [], - accepted: false - }); - - await addOrUpdateUser(answer.owner.display_name, { - points: answer.owner.reputation, - newQuestionId, - newAnswerId - }); - } - }) - ); - - logOrDebug()( - `added ${randomAnswers.length - randomAnswersLength} of ${ - answers.items.length - } received answers to catch-all question` - ); - - logOrDebug()( - `there are now ${randomAnswers.length} total of ${collectAllQuestionAnswersCount} wanted random answers added to catch-all question` - ); - - if (!answers.has_more) { - logOrDebug().warn( - 'somehow exhausted all answers in the StackExchange API?!' - ); - break; - } - } - })(), - (async () => { - for ( - let commentPage = 1; - randomComments.length < totalDesiredComments; - ++commentPage - ) { - const comments = await queue.addRequestToQueue< - StackExchangeApiResponse - >( - api.comments({ - page: commentPage, - pageSize: maxPageSize - }), - undefined, - { apiEndpoint: 'comments' } - ); - - const randomCommentsLength = randomComments.length; - - logOrDebug()( - `received ${comments.items.length} random comments for catch-all question and its first answer from API` - ); - - await Promise.all( - comments.items.map(async (comment) => { - if (comment.body.length <= maxCommentLength) { - const createdAt = - comment.creation_date * oneSecondInMs; - - questionCreatedAt = - createdAt < questionCreatedAt - ? createdAt - : questionCreatedAt; - - const [commentUpvotes, commentDownvotes] = - getUpvotesDownvotesFromScore(comment.score); - - randomComments.push({ - _id: new ObjectId(), - creator: comment.owner.display_name, - createdAt, - text: comment.body, - upvotes: commentUpvotes, - upvoterUsernames: [], - downvotes: commentDownvotes, - downvoterUsernames: [] - }); - - await addOrUpdateUser(comment.owner.display_name, { - points: comment.owner.reputation - }); - } - }) - ); - - logOrDebug()( - `added ${ - randomComments.length - randomCommentsLength - } of ${ - comments.items.length - } received comments to local storage for catch-all question` - ); - - logOrDebug()( - `---\nthere are now ${randomComments.length} total of ${totalDesiredComments} wanted random comments stored\n---` - ); - - if (!comments.has_more) { - logOrDebug().warn( - 'somehow exhausted all comments in the StackExchange API?!' - ); - break; - } - } - })() - ]); - - // ? Ensure answers and comments are in insertion order (oldest first) - randomAnswers.sort((a, b) => a.createdAt - b.createdAt); - randomComments.sort((a, b) => a.createdAt - b.createdAt); - - data.catchallQuestion = { - _id: newQuestionId, - title: - 'What are the best answers and comments you can come up with?', - 'title-lowercase': - 'what are the best answers and comments you can come up with?', - creator: 'Hordak', - createdAt: questionCreatedAt - 10 ** 6, - status: 'protected', - text: '**Hello, world!** What are some of the best random answers, question comments, and answer comments you can come up with? Post below.', - upvotes: 15, - upvoterUsernames: [], - downvotes: 2, - downvoterUsernames: [], - hasAcceptedAnswer: false, - views: 1024, - answers: collectAllQuestionAnswersCount, - answerItems: randomAnswers.slice( - 0, - collectAllQuestionAnswersCount - ), - comments: collectAllQuestionCommentsCount, - commentItems: randomComments.slice( - 0, - collectAllQuestionCommentsCount - ), - sorter: { - uvc: 15 + 1024 + collectAllQuestionCommentsCount, - uvac: - 15 + - 1024 + - collectAllQuestionAnswersCount + - collectAllQuestionCommentsCount - } - }; - - logOrDebug()( - `added ${collectAllQuestionCommentsCount} random comments to catch-all question` - ); - - if (randomAnswers[0]) { - randomAnswers[0].commentItems = randomComments.slice( - collectAllQuestionCommentsCount, - totalDesiredComments - ); - - logOrDebug()( - `added ${ - totalDesiredComments - collectAllQuestionCommentsCount - } random comments to catch-all question's first answer` - ); - } else { - logOrDebug().warn('catch-all question has no answers?!'); - } - - logOrDebug().extend('cached')( - '\n>>> added catch-all question to cache <<<\n\n' - ); - - await addOrUpdateUser('Hordak', { - points: 1_234_567, - newQuestionId - }); - } - })() - ]); - - if (data.catchallQuestion) { - data.questions.push(data.catchallQuestion); - delete data.catchallQuestion; - } else { - throw new GuruMeditationError('missing catchall question'); - } - - // ? Ensure questions are in insertion order (oldest first) - data.questions.sort((a, b) => a.createdAt - b.createdAt); - - data.cache.complete = true; - } - } catch (error) { - logOrDebug().error(error); - - queue.immediatelyStopProcessingRequestQueue(); - - logOrDebug()('interrupted queue stats: %O', queue.getStats()); - logOrDebug()('incomplete cache stats: %O', getDataStats(data)); - - await getPrompter(process.env.TEST_PROMPTER_ERRORHANDLER) - .prompt<{ action: string; token: string }>([ - { - name: 'action', - message: 'what now?', - type: 'list', - choices: [ - { - name: 'attempt to continue using a custom token', - value: 'hit-custom' - }, - { - name: 'commit incomplete cache data to database', - value: 'commit' - }, - { - name: 'save incomplete cache data to disk and exit', - value: 'exit-save' - }, - { name: 'exit without saving any data', value: 'exit' } - ] - }, - { - name: 'token', - message: 'enter new token', - type: 'input', - when: (answers) => answers.action.includes('custom') - } - ]) - .then(async (answers) => { - if (answers.action.startsWith('exit')) { - if (answers.action == 'exit-save') { - await cacheDataToDisk(data); - } - - logOrDebug()('execution interrupted'); - process.exit(1); - } - - if (answers.action.startsWith('commit')) { - await commitApiDataToDb(data); - - logOrDebug()('execution interrupted'); - process.exit(1); - } - - interrogateApi = answers.action.startsWith('hit'); - usedStackExAuthKey = answers.token ?? usedStackExAuthKey; - queue.beginProcessingRequestQueue(); - }); - } - } - } finally { - if (queue.isProcessingRequestQueue) { - queue.gracefullyStopProcessingRequestQueue(); - } - - logOrDebug()('waiting for request queue to terminate...'); - await queue.waitForQueueProcessingToStop(); - logOrDebug()('request queue terminated'); + try { + const answers = await getPrompter(process.env.TEST_PROMPTER_INITIALIZER).prompt<{ + action: string; + token: string; + }>([ + { + name: 'action', + message: 'select an initializer action', + type: 'list', + choices: [ + { + name: 'commit initial state to database', + value: 'commit' + }, + { name: 'exit', value: 'exit' } + ] } + ]); - logOrDebug()('interrogation complete'); - logOrDebug()('final queue stats: %O', queue.getStats()); - } + switch (answers.action) { + case 'exit': { + break; + } - if (process.env.TEST_SKIP_REQUESTS) { - debug.warn( - 'saw debug env var TEST_SKIP_REQUESTS. Post-request tasks will be skipped' - ); - } else { - logOrDebug()('final cache stats: %O', getDataStats(data)); - await cacheDataToDisk(data); + case 'commit': { + await Promise.all([getDb({ name: 'root' }), getDb({ name: 'app' })]); - await getPrompter(process.env.TEST_PROMPTER_FINALIZER) - .prompt<{ action: string; token: string }>([ + await getPrompter(process.env.TEST_PROMPTER_FINALIZER).prompt<{ + action: string; + token: string; + }>([ { name: 'action', message: 'what now?', type: 'list', - choices: [ - { - name: 'commit results to database', - value: 'commit' - }, - { - name: 'commit results to database (include dummy root data)', - value: 'commit-root' - }, - { name: 'exit', value: 'exit' } - ] - }, - { - name: 'token', - message: 'enter new token', - type: 'input', - when: (answers) => answers.action.includes('custom') + choices: [{ name: 'exit', value: 'exit' }] } - ]) - .then(async (answers) => { - if (!data) { - throw new GuruMeditationError('data cannot be null'); - } - - if (answers.action == 'exit-save') { - await cacheDataToDisk(data); - } - - if (answers.action.startsWith('commit')) { - if (answers.action == 'commit-root') { - await commitRootDataToDb(data); - } - - await commitApiDataToDb(data); - } - }); + ]); + break; + } } logOrDebug()('execution complete'); @@ -1167,15 +111,6 @@ const invoked = async () => { } }; -export type Data = { - questions: InternalQuestion[]; - catchallQuestion?: InternalQuestion; - users: InternalUser[]; - cache: { - complete: boolean; - }; -}; - export default invoked().catch((error: Error) => { log.error(error.message); process.exit(2); diff --git a/external-scripts/prune-data.ts b/external-scripts/prune-data.ts index e9c4f05..d4e114a 100644 --- a/external-scripts/prune-data.ts +++ b/external-scripts/prune-data.ts @@ -9,14 +9,14 @@ import { import { debugNamespace as namespace } from 'universe/constants'; import { getEnv } from 'universe/backend/env'; -import { deleteUser } from 'universe/backend'; +import { deletePage, deleteUser } from 'universe/backend'; import { debugFactory } from 'multiverse/debug-extended'; import { getDb } from 'multiverse/mongo-schema'; import type { Document, ObjectId, WithId } from 'mongodb'; import type { Promisable } from 'type-fest'; -import type { InternalUser } from 'universe/backend/db'; +import type { InternalInfo, InternalPage, InternalUser } from 'universe/backend/db'; const debugNamespace = `${namespace}:prune-data`; @@ -29,9 +29,6 @@ type DataLimit = { deleteFn?: (thresholdEntry: WithId) => Promisable; }; -// TODO: PRUNE_DATA_MAX_SESSIONS_BYTES -// TODO: PRUNE_DATA_MAX_PAGES_BYTES - // eslint-disable-next-line no-console log.log = console.info.bind(console); @@ -72,27 +69,64 @@ const getDbCollectionLimits = (env: ReturnType) => { } }, app: { - mail: { + pages: { limit: { maxBytes: - env.PRUNE_DATA_MAX_MAIL_BYTES && env.PRUNE_DATA_MAX_MAIL_BYTES > 0 - ? env.PRUNE_DATA_MAX_MAIL_BYTES + env.PRUNE_DATA_MAX_PAGES_BYTES && env.PRUNE_DATA_MAX_PAGES_BYTES > 0 + ? env.PRUNE_DATA_MAX_PAGES_BYTES : toss( new InvalidAppEnvironmentError( - 'PRUNE_DATA_MAX_MAIL_BYTES must be greater than zero' + 'PRUNE_DATA_MAX_PAGES_BYTES must be greater than zero' ) ) + }, + async deleteFn(thresholdEntry) { + const db = await getDb({ name: 'app' }); + const usersDb = db.collection('users'); + const pagesDb = db.collection('pages'); + const infoDb = db.collection('pages'); + + const pages = await pagesDb + .find( + { _id: { $lte: thresholdEntry._id } }, + { projection: { _id: true, blog_id: true, name: true } } + ) + .toArray(); + + await Promise.all( + pages.map(async ({ _id: page_id, blog_id, name: pageName }) => { + const { blogName } = + (await usersDb.findOne( + { _id: blog_id }, + { projection: { _id: false, blogName: true } } + )) || {}; + + if (!blogName) { + debug.warn( + `database contained orphaned page that had to be deleted manually: ${page_id}` + ); + + await Promise.all([ + pagesDb.deleteOne({ _id: page_id }), + infoDb.updateOne({}, { $inc: { pages: -1 } }) + ]); + } else { + await deletePage({ blogName, pageName }); + } + }) + ); + + return pages.length; } }, - questions: { + sessions: { limit: { maxBytes: - env.PRUNE_DATA_MAX_QUESTIONS_BYTES && - env.PRUNE_DATA_MAX_QUESTIONS_BYTES > 0 - ? env.PRUNE_DATA_MAX_QUESTIONS_BYTES + env.PRUNE_DATA_MAX_SESSIONS_BYTES && env.PRUNE_DATA_MAX_SESSIONS_BYTES > 0 + ? env.PRUNE_DATA_MAX_SESSIONS_BYTES : toss( new InvalidAppEnvironmentError( - 'PRUNE_DATA_MAX_QUESTIONS_BYTES must be greater than zero' + 'PRUNE_DATA_MAX_SESSIONS_BYTES must be greater than zero' ) ) } @@ -113,12 +147,19 @@ const getDbCollectionLimits = (env: ReturnType) => { 'users' ); - const usernames = ( - await users.find({ _id: { $lte: thresholdEntry._id } }).toArray() - ).map((user) => user.username); - - await Promise.all(usernames.map((username) => deleteUser({ username }))); - return usernames.length; + const emails = ( + await users + .find( + { _id: { $lte: thresholdEntry._id } }, + { projection: { _id: false, email: true } } + ) + .toArray() + ).map((user) => user.email); + + await Promise.all( + emails.map((email) => deleteUser({ usernameOrEmail: email })) + ); + return emails.length; } } } diff --git a/lib/mongo-item/unit.test.ts b/lib/mongo-item/unit.test.ts index 6202804..271b8c7 100644 --- a/lib/mongo-item/unit.test.ts +++ b/lib/mongo-item/unit.test.ts @@ -173,14 +173,20 @@ describe('::itemToObjectId', () => { it('throws if an item is irreducible or invalid', async () => { expect.hasAssertions(); - expect(() => itemToObjectId(null)).toThrow('irreducible'); - expect(() => itemToObjectId(undefined)).toThrow('irreducible'); - expect(() => itemToObjectId([null])).toThrow('irreducible'); - expect(() => itemToObjectId([undefined])).toThrow('irreducible'); + expect(() => itemToObjectId(null)).toThrow('unable to reduce item to id: null'); + expect(() => itemToObjectId(undefined)).toThrow( + 'unable to reduce item to id: undefined' + ); + expect(() => itemToObjectId([null])).toThrow( + 'unable to reduce sub-item to id: null' + ); + expect(() => itemToObjectId([undefined])).toThrow( + 'unable to reduce sub-item to id: undefined' + ); // @ts-expect-error: bad param - expect(() => itemToObjectId({})).toThrow('irreducible'); + expect(() => itemToObjectId({})).toThrow('unable to reduce item to id'); // @ts-expect-error: bad param - expect(() => itemToObjectId([{}])).toThrow('irreducible'); + expect(() => itemToObjectId([{}])).toThrow('unable to reduce sub-item to id'); expect(() => itemToObjectId('bad')).toThrow('invalid id "bad"'); expect(() => itemToObjectId(['bad'])).toThrow('invalid id "bad"'); expect(() => itemToObjectId([new ObjectId(), 'bad'])).toThrow('invalid id "bad"'); @@ -219,14 +225,20 @@ describe('::itemToStringId', () => { it('throws if item is irreducible', async () => { expect.hasAssertions(); - expect(() => itemToStringId(null)).toThrow('irreducible'); - expect(() => itemToStringId(undefined)).toThrow('irreducible'); - expect(() => itemToStringId([null])).toThrow('irreducible'); - expect(() => itemToStringId([undefined])).toThrow('irreducible'); + expect(() => itemToStringId(null)).toThrow('unable to reduce item to id: null'); + expect(() => itemToStringId(undefined)).toThrow( + 'unable to reduce item to id: undefined' + ); + expect(() => itemToStringId([null])).toThrow( + 'unable to reduce sub-item to id: null' + ); + expect(() => itemToStringId([undefined])).toThrow( + 'unable to reduce sub-item to id: undefined' + ); // @ts-expect-error: bad param - expect(() => itemToStringId({})).toThrow('irreducible'); + expect(() => itemToStringId({})).toThrow('unable to reduce item to id'); // @ts-expect-error: bad param - expect(() => itemToStringId([{}])).toThrow('irreducible'); + expect(() => itemToStringId([{}])).toThrow('unable to reduce sub-item to id'); expect(() => itemToStringId('bad')).toThrow('invalid id "bad"'); expect(() => itemToStringId(['bad'])).toThrow('invalid id "bad"'); expect(() => itemToStringId([new ObjectId(), 'bad'])).toThrow('invalid id "bad"'); diff --git a/test/externals/unit-initialize.test.ts b/test/externals/unit-initialize.test.ts index 809c935..7cf6e67 100644 --- a/test/externals/unit-initialize.test.ts +++ b/test/externals/unit-initialize.test.ts @@ -1,30 +1,12 @@ -import { rest, type ResponseTransformer, type RestContext } from 'msw'; -import { setupServer } from 'msw/node'; - import { debugNamespace as namespace } from 'universe/constants'; - import { setupMemoryServerOverride } from 'multiverse/mongo-test'; -import { - getDummyQuestions, - getDummyQuestionAnswers, - getDummyAnswers, - getDummyAnswerComments, - getDummyComments, - getDummyQuestionComments -} from 'externals/initialize-data/api-test'; - import { mockEnvFactory, protectedImportFactory, withMockedOutput } from 'testverse/setup'; -import type { - SecondsFromNow, - StackExchangeApiResponse -} from 'types/stackexchange-api'; - void namespace; // ? Ensure the isolated external picks up the memory server override @@ -32,33 +14,17 @@ jest.mock('multiverse/mongo-schema', (): typeof import('multiverse/mongo-schema' return jest.requireActual('multiverse/mongo-schema'); }); -const totalGeneratedQuestions = 5; -const intervalPeriodMs = 0; - const withMockedEnv = mockEnvFactory({ // ! For max test perf, ensure this next line is commented out unless needed - //DEBUG: `throttled-fetch:*,${namespace}:initialize-data,${namespace}:initialize-data:*`, //DEBUG: `${namespace}:initialize-data,${namespace}:initialize-data:*`, // ? Use these to control the options auto-selected for inquirer. Note that // ? these values must either be empty/undefined or a valid URL query string. - TEST_PROMPTER_INITIALIZER: 'action=hit-ignore', - TEST_PROMPTER_ERRORHANDLER: 'action=exit', - TEST_PROMPTER_FINALIZER: 'action=commit', + TEST_PROMPTER_INITIALIZER: 'action=commit', + TEST_PROMPTER_FINALIZER: 'action=exit', NODE_ENV: 'test', - MONGODB_URI: 'fake', - MAX_ANSWER_BODY_LENGTH_BYTES: '100', - MAX_COMMENT_LENGTH: '100', - MAX_QUESTION_BODY_LENGTH_BYTES: '100', - STACKAPPS_INTERVAL_PERIOD_MS: intervalPeriodMs.toString(), - STACKAPPS_MAX_REQUESTS_PER_INTERVAL: '25', - STACKAPPS_TOTAL_API_GENERATED_QUESTIONS: totalGeneratedQuestions.toString(), - STACKAPPS_COLLECTALL_QUESTION_ANSWERS: '4', - STACKAPPS_COLLECTALL_QUESTION_COMMENTS: '3', - STACKAPPS_COLLECTALL_FIRST_ANSWER_COMMENTS: '2', - STACKAPPS_MAX_PAGE_SIZE: '2', // * Should be <= half of the above constants - STACKAPPS_AUTH_KEY: 'special-stack-exchange-key' + MONGODB_URI: 'fake' }); const importInitializeData = protectedImportFactory< @@ -68,157 +34,8 @@ const importInitializeData = protectedImportFactory< useDefault: true }); -let counter = 0; -const mockedResponseJson: Partial> = {}; - -const calcBackoffModulo = Math.ceil(5.49 * totalGeneratedQuestions); -const calcError500Modulo = - totalGeneratedQuestions + Math.max(2, Math.ceil(0.1 * totalGeneratedQuestions)); -const calcError502Modulo = calcError500Modulo + 1; -const calcError503Modulo = calcError500Modulo * 2; -const calcError429Modulo = calcError500Modulo * 2 + 1; - -const maybeErrorResponse = ( - context: RestContext, - { okTransformers }: { okTransformers: ResponseTransformer[] } -) => { - delete mockedResponseJson.backoff; - - if (counter) { - if (counter % calcBackoffModulo == 0) { - mockedResponseJson.backoff = Math.max( - 0.1, - intervalPeriodMs / 200 - ) as SecondsFromNow; - } - - const results = - counter % calcError500Modulo == 0 - ? [ - context.status(500), - context.json({ - error_id: 123, - error_message: 'fake 500 error', - error_name: 'fake_500' - }) - ] - : counter % calcError503Modulo == 0 - ? [ - context.status(503), - context.json({ - error_id: 123, - error_message: 'fake 503 error', - error_name: 'fake_503' - }) - ] - : counter % calcError502Modulo == 0 - ? [ - context.status(502), - context.json({ - error_id: 123, - error_message: 'fake 502 error', - error_name: 'fake_502' - }) - ] - : counter % calcError429Modulo == 0 - ? [context.status(429)] - : okTransformers; - - counter++; - return results; - } else { - counter++; - return okTransformers; - } -}; - -const server = setupServer( - rest.get('*/questions/:question_id/answers', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyQuestionAnswers(req), - ...mockedResponseJson - }) - ] - }) - ); - }), - rest.get('*/questions/:question_id/comments', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyQuestionComments(req), - ...mockedResponseJson - }) - ] - }) - ); - }), - rest.get('*/answers/:answer_id/comments', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyAnswerComments(req), - ...mockedResponseJson - }) - ] - }) - ); - }), - rest.get('*/questions', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyQuestions(req), - ...mockedResponseJson - }) - ] - }) - ); - }), - rest.get('*/answers', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyAnswers(req), - ...mockedResponseJson - }) - ] - }) - ); - }), - rest.get('*/comments', async (req, res, context) => { - return res( - ...maybeErrorResponse(context, { - okTransformers: [ - context.status(200), - context.json({ - ...getDummyComments(req), - ...mockedResponseJson - }) - ] - }) - ); - }) -); - setupMemoryServerOverride(); -beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); -afterEach(() => server.resetHandlers()); -afterAll(() => server.close()); - it('is verbose when no DEBUG environment variable set and compiled NODE_ENV is not test', async () => { expect.hasAssertions(); @@ -226,61 +43,16 @@ it('is verbose when no DEBUG environment variable set and compiled NODE_ENV is n await withMockedEnv(() => importInitializeData({ expectedExitCode: 0 }), { DEBUG: undefined, NODE_ENV: 'something-else', - OVERRIDE_EXPECT_ENV: 'force-no-check', - TEST_SKIP_REQUESTS: 'true' + OVERRIDE_EXPECT_ENV: 'force-no-check' }); - expect(infoSpy).toBeCalledWith(expect.stringContaining('execution complete')); + expect(infoSpy.mock.calls.at(-1)?.[0]).toStrictEqual( + expect.stringContaining('execution complete') + ); }); await withMockedOutput(async ({ infoSpy }) => { - await withMockedEnv(() => importInitializeData({ expectedExitCode: 0 }), { - TEST_SKIP_REQUESTS: 'true' - }); + await withMockedEnv(() => importInitializeData({ expectedExitCode: 0 })); expect(infoSpy).not.toBeCalled(); }); }); - -it('rejects on bad environment', async () => { - expect.hasAssertions(); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_INTERVAL_PERIOD_MS: 'bad', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_MAX_REQUESTS_PER_INTERVAL: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_TOTAL_API_GENERATED_QUESTIONS: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_COLLECTALL_QUESTION_ANSWERS: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_COLLECTALL_QUESTION_COMMENTS: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_COLLECTALL_FIRST_ANSWER_COMMENTS: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_MAX_PAGE_SIZE: '', - TEST_SKIP_REQUESTS: 'true' - }); - - await withMockedEnv(() => importInitializeData({ expectedExitCode: 2 }), { - STACKAPPS_AUTH_KEY: '', - TEST_SKIP_REQUESTS: 'true' - }); -}); diff --git a/test/externals/unit-prune.test.ts b/test/externals/unit-prune.test.ts index 442ccca..c1f50ed 100644 --- a/test/externals/unit-prune.test.ts +++ b/test/externals/unit-prune.test.ts @@ -22,16 +22,14 @@ jest.mock('multiverse/mongo-schema', (): typeof import('multiverse/mongo-schema' // const testCollectionsMap = { // 'root.request-log': dummyRootData['request-log'].length, // 'root.limited-log': dummyRootData['limited-log'].length, -// 'app.mail': dummyAppData['mail'].length, -// 'app.questions': dummyAppData['questions'].length, -// 'app.users': dummyAppData['users'].length +// ... // }; const testCollections = [ 'root.request-log', 'root.limited-log', - 'app.mail', - 'app.questions', + 'app.pages', + 'app.sessions', 'app.users' ] as const; @@ -41,9 +39,9 @@ const withMockedEnv = mockEnvFactory({ PRUNE_DATA_MAX_LOGS_BYTES: '50mb', PRUNE_DATA_MAX_BANNED_BYTES: '10mb', // * Step 2: Add new env var default values here - PRUNE_DATA_MAX_MAIL_BYTES: '50mb', - PRUNE_DATA_MAX_QUESTIONS_BYTES: '250mb', - PRUNE_DATA_MAX_USERS_BYTES: '75mb' + PRUNE_DATA_MAX_PAGES_BYTES: '300mb', + PRUNE_DATA_MAX_SESSIONS_BYTES: '20mb', + PRUNE_DATA_MAX_USERS_BYTES: '50mb' }); const importPruneData = protectedImportFactory< @@ -131,7 +129,9 @@ it('is verbose when no DEBUG environment variable set and compiled NODE_ENV is n OVERRIDE_EXPECT_ENV: 'force-no-check' }); - expect(infoSpy).toBeCalledWith(expect.stringContaining('execution complete')); + expect(infoSpy.mock.calls.at(-1)?.[0]).toStrictEqual( + expect.stringContaining('execution complete') + ); }); await withMockedOutput(async ({ infoSpy }) => { @@ -151,8 +151,8 @@ it('rejects on bad environment', async () => { await withMockedEnv(() => importPruneData({ expectedExitCode: 2 }), { PRUNE_DATA_MAX_LOGS_BYTES: '', PRUNE_DATA_MAX_BANNED_BYTES: '', - PRUNE_DATA_MAX_MAIL_BYTES: '', - PRUNE_DATA_MAX_QUESTIONS_BYTES: '', + PRUNE_DATA_MAX_PAGES_BYTES: '', + PRUNE_DATA_MAX_SESSIONS_BYTES: '', PRUNE_DATA_MAX_USERS_BYTES: '' }); @@ -165,11 +165,11 @@ it('rejects on bad environment', async () => { }); await withMockedEnv(() => importPruneData({ expectedExitCode: 2 }), { - PRUNE_DATA_MAX_MAIL_BYTES: '' + PRUNE_DATA_MAX_PAGES_BYTES: '' }); await withMockedEnv(() => importPruneData({ expectedExitCode: 2 }), { - PRUNE_DATA_MAX_QUESTIONS_BYTES: '' + PRUNE_DATA_MAX_SESSIONS_BYTES: '' }); await withMockedEnv(() => importPruneData({ expectedExitCode: 2 }), { @@ -188,16 +188,16 @@ it('respects the limits imposed by PRUNE_DATA_MAX_X environment variables', asyn const expectedSizes = { 'root.request-log': initialSizes['root.request-log'] / 2, 'root.limited-log': initialSizes['root.limited-log'] / 2, - 'app.mail': initialSizes['app.mail'] / 2, - 'app.questions': initialSizes['app.questions'] / 2, + 'app.pages': initialSizes['app.pages'] / 2, + 'app.sessions': initialSizes['app.sessions'] / 2, 'app.users': initialSizes['app.users'] / 2 }; await withMockedEnv(() => importPruneData({ expectedExitCode: 0 }), { PRUNE_DATA_MAX_LOGS_BYTES: String(expectedSizes['root.request-log']), PRUNE_DATA_MAX_BANNED_BYTES: String(expectedSizes['root.limited-log']), - PRUNE_DATA_MAX_MAIL_BYTES: String(expectedSizes['app.mail']), - PRUNE_DATA_MAX_QUESTIONS_BYTES: String(expectedSizes['app.questions']), + PRUNE_DATA_MAX_PAGES_BYTES: String(expectedSizes['app.pages']), + PRUNE_DATA_MAX_SESSIONS_BYTES: String(expectedSizes['app.sessions']), PRUNE_DATA_MAX_USERS_BYTES: String(expectedSizes['app.users']) }); @@ -211,19 +211,15 @@ it('respects the limits imposed by PRUNE_DATA_MAX_X environment variables', asyn expectedSizes['root.limited-log'] ); - expect(newSizes['app.mail']).toBeLessThanOrEqual(expectedSizes['app.mail']); - - expect(newSizes['app.questions']).toBeLessThanOrEqual( - expectedSizes['app.questions'] - ); - + expect(newSizes['app.pages']).toBeLessThanOrEqual(expectedSizes['app.pages']); + expect(newSizes['app.sessions']).toBeLessThanOrEqual(expectedSizes['app.sessions']); expect(newSizes['app.users']).toBeLessThanOrEqual(expectedSizes['app.users']); await withMockedEnv(() => importPruneData({ expectedExitCode: 0 }), { PRUNE_DATA_MAX_LOGS_BYTES: '1', PRUNE_DATA_MAX_BANNED_BYTES: '1', - PRUNE_DATA_MAX_MAIL_BYTES: '1', - PRUNE_DATA_MAX_QUESTIONS_BYTES: '1', + PRUNE_DATA_MAX_PAGES_BYTES: '1', + PRUNE_DATA_MAX_SESSIONS_BYTES: '1', PRUNE_DATA_MAX_USERS_BYTES: '1' }); @@ -231,8 +227,8 @@ it('respects the limits imposed by PRUNE_DATA_MAX_X environment variables', asyn expect(latestSizes['root.request-log']).toBe(0); expect(latestSizes['root.limited-log']).toBe(0); - expect(latestSizes['app.mail']).toBe(0); - expect(latestSizes['app.questions']).toBe(0); + expect(latestSizes['app.pages']).toBe(0); + expect(latestSizes['app.sessions']).toBe(0); expect(latestSizes['app.users']).toBe(0); }); @@ -246,8 +242,8 @@ it('only deletes entries if necessary', async () => { PRUNE_DATA_MAX_LOGS_BYTES: '100gb', PRUNE_DATA_MAX_BANNED_BYTES: '100gb', // * Step 5: Add new env vars high-prune-threshold values here - PRUNE_DATA_MAX_MAIL_BYTES: '100gb', - PRUNE_DATA_MAX_QUESTIONS_BYTES: '100gb', + PRUNE_DATA_MAX_PAGES_BYTES: '100gb', + PRUNE_DATA_MAX_SESSIONS_BYTES: '100gb', PRUNE_DATA_MAX_USERS_BYTES: '100gb' }); @@ -256,9 +252,7 @@ it('only deletes entries if necessary', async () => { expect(newSizes['root.request-log']).toBe(initialSizes['root.request-log']); expect(newSizes['root.limited-log']).toBe(initialSizes['root.limited-log']); - expect(newSizes['app.mail']).toBe(initialSizes['app.mail']); - - expect(newSizes['app.questions']).toBe(initialSizes['app.questions']); - + expect(newSizes['app.pages']).toBe(initialSizes['app.pages']); + expect(newSizes['app.sessions']).toBe(initialSizes['app.sessions']); expect(newSizes['app.users']).toBe(initialSizes['app.users']); }); diff --git a/test/integration.ts b/test/integration.ts index 344faf1..73fcdf4 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -5,29 +5,27 @@ import { ObjectId } from 'mongodb'; import debugFactory from 'debug'; import { GuruMeditationError } from 'universe/error'; +import { defaultNavLinks } from 'universe/backend'; import { getEnv } from 'universe/backend/env'; import { - NewPage, - PatchBlog, - PatchPage, - PublicBlog, - PublicInfo, - PublicPage, - PublicPageMetadata, - toPublicBlog, - toPublicPageMetadata, + type NewPage, + type PatchBlog, + type PatchPage, + type PublicBlog, + type PublicInfo, + type PublicPage, + type PublicPageMetadata, + type NewUser, + type PatchUser, + type PublicUser, toPublicUser } from 'universe/backend/db'; import { dummyAppData } from 'testverse/db'; import type { Promisable } from 'type-fest'; - -import type { NewUser, PatchUser, PublicUser } from 'universe/backend/db'; - import type { NextApiHandlerMixin } from 'testverse/util'; -import { defaultNavLinks } from 'universe/backend'; // TODO: XXX: turn a lot of this into some kind of package; needs to be generic // TODO: XXX: enough to handle various use cases though :) Maybe