From 6bb0aa62a6f88a64dbfa59f2115d941234e67935 Mon Sep 17 00:00:00 2001 From: malik Date: Wed, 3 Jul 2024 15:03:48 +0200 Subject: [PATCH 1/6] fix loginCookie not being updated in jsonRequestOptions --- src/MediaWiki.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 013a32ea..99045575 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -289,6 +289,7 @@ class MediaWiki { } downloader.loginCookie = resp.headers['set-cookie'].join(';') + downloader.jsonRequestOptions.headers.cookie = downloader.loginCookie }) .catch((err) => { throw err From 022f31a510770f2c6c36a12bdd50ee34b89623a8 Mon Sep 17 00:00:00 2001 From: malik Date: Wed, 3 Jul 2024 16:42:14 +0200 Subject: [PATCH 2/6] add loginCookie to supported renderer check --- src/MediaWiki.ts | 16 ++++++++-------- src/renderers/renderer.builder.ts | 10 +++++----- src/util/saveArticles.ts | 29 +++++++++++++++++++++++------ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 99045575..7f86b4b5 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -172,30 +172,30 @@ class MediaWiki { this.initializeMediaWikiDefaults() } - public async hasWikimediaDesktopApi(): Promise { + public async hasWikimediaDesktopApi(loginCookie: String): Promise { if (this.#hasWikimediaDesktopApi === null) { this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.wikimediaDesktopApiUrl.href) - this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId)) + this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) return this.#hasWikimediaDesktopApi } return this.#hasWikimediaDesktopApi } - public async hasWikimediaMobileApi(): Promise { + public async hasWikimediaMobileApi(loginCookie: String): Promise { if (this.#hasWikimediaMobileApi === null) { this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.wikimediaMobileApiUrl.href) - this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId)) + this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) return this.#hasWikimediaMobileApi } return this.#hasWikimediaMobileApi } - public async hasVisualEditorApi(): Promise { + public async hasVisualEditorApi(loginCookie: String): Promise { if (this.#hasVisualEditorApi === null) { this.visualEditorUrlDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) this.#hasVisualEditorApi = await checkApiAvailability( this.visualEditorUrlDirector.buildArticleURL(this.apiCheckArticleId), - '' /* empty login cookie */, + loginCookie, this.visualEditorUrlDirector.validMimeTypes, ) return this.#hasVisualEditorApi @@ -203,10 +203,10 @@ class MediaWiki { return this.#hasVisualEditorApi } - public async hasRestApi(): Promise { + public async hasRestApi(loginCookie: String): Promise { if (this.#hasRestApi === null) { this.restApiUrlDirector = new RestApiURLDirector(this.restApiUrl.href) - this.#hasRestApi = await checkApiAvailability(this.restApiUrlDirector.buildArticleURL(this.apiCheckArticleId)) + this.#hasRestApi = await checkApiAvailability(this.restApiUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) return this.#hasRestApi } return this.#hasRestApi diff --git a/src/renderers/renderer.builder.ts b/src/renderers/renderer.builder.ts index df06648a..7960663c 100644 --- a/src/renderers/renderer.builder.ts +++ b/src/renderers/renderer.builder.ts @@ -8,14 +8,14 @@ import { RendererBuilderOptions } from './abstract.renderer.js' import * as logger from './../Logger.js' export class RendererBuilder { - public async createRenderer(options: RendererBuilderOptions): Promise { + public async createRenderer(options: RendererBuilderOptions, loginCookie: String): Promise { const { renderType, renderName } = options const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi] = await Promise.all([ - MediaWiki.hasVisualEditorApi(), - MediaWiki.hasWikimediaDesktopApi(), - MediaWiki.hasWikimediaMobileApi(), - MediaWiki.hasRestApi(), + MediaWiki.hasVisualEditorApi(loginCookie), + MediaWiki.hasWikimediaDesktopApi(loginCookie), + MediaWiki.hasWikimediaMobileApi(loginCookie), + MediaWiki.hasRestApi(loginCookie), ]) switch (renderType) { diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index f32577ca..97b1a1c9 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -54,7 +54,12 @@ export async function downloadFiles(fileStore: RKVS, retryStore: RKV } finally { if (isFailed) { if (doRetry && resp.status !== 404) { - await retryStore.set(resp.path, { url: resp.url, namespace: resp.namespace, mult: resp.mult, width: resp.width }) + await retryStore.set(resp.path, { + url: resp.url, + namespace: resp.namespace, + mult: resp.mult, + width: resp.width, + }) } else { logger.warn(`Error downloading file [${urlHelper.deserializeUrl(resp.url)}], skipping`) dump.status.files.fail += 1 @@ -245,14 +250,14 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade const renderer = await rendererBuilder.createRenderer({ renderType: 'specific', renderName: forceRender, - }) + }, downloader.loginCookie) mainPageRenderer = renderer articlesRenderer = renderer } else { - mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' }) + mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' }, downloader.loginCookie) articlesRenderer = await rendererBuilder.createRenderer({ renderType: hasWikimediaMobileApi ? 'mobile' : 'auto', - }) + }, downloader.loginCookie) } downloader.setUrlsDirectors(mainPageRenderer, articlesRenderer) @@ -296,7 +301,15 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage) - for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, moduleDependencies, staticFiles, subtitles } of rets) { + for (const { + articleId, + displayTitle: articleTitle, + html: finalHTML, + mediaDependencies, + moduleDependencies, + staticFiles, + subtitles + } of rets) { if (!finalHTML) { logger.warn(`No HTML returned for article [${articleId}], skipping`) continue @@ -390,7 +403,11 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade logger.log(`Done with downloading a total of [${articlesTotal}] articles`) if (jsConfigVars) { - const jsConfigVarArticle = new ZimArticle({ url: jsPath('jsConfigVars', config.output.dirs.mediawiki), data: jsConfigVars, ns: '-' }) + const jsConfigVarArticle = new ZimArticle({ + url: jsPath('jsConfigVars', config.output.dirs.mediawiki), + data: jsConfigVars, + ns: '-', + }) zimCreator.addArticle(jsConfigVarArticle) } From 5a3236f7a885ad440e8b8bcef5fbe62a9ffe8f7c Mon Sep 17 00:00:00 2001 From: malik Date: Wed, 3 Jul 2024 17:09:18 +0200 Subject: [PATCH 3/6] call hasApi functions with login cookie --- src/mwoffliner.lib.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index 43b9ac45..eba1aa6c 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -212,10 +212,10 @@ async function execute(argv: any) { MediaWiki.apiCheckArticleId = mwMetaData.mainPage await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie) + const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie) + await MediaWiki.hasRestApi(downloader.loginCookie) + await MediaWiki.hasVisualEditorApi(downloader.loginCookie) RedisStore.setOptions(argv.redis || config.defaults.redisPath) await RedisStore.connect() From 9c4fc8e00317fcbc3e11c58670a4c8712448f331 Mon Sep 17 00:00:00 2001 From: malik Date: Tue, 9 Jul 2024 13:14:49 +0200 Subject: [PATCH 4/6] String -> string --- src/MediaWiki.ts | 8 ++++---- src/renderers/renderer.builder.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 7f86b4b5..5b7c4064 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -172,7 +172,7 @@ class MediaWiki { this.initializeMediaWikiDefaults() } - public async hasWikimediaDesktopApi(loginCookie: String): Promise { + public async hasWikimediaDesktopApi(loginCookie: string): Promise { if (this.#hasWikimediaDesktopApi === null) { this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.wikimediaDesktopApiUrl.href) this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) @@ -181,7 +181,7 @@ class MediaWiki { return this.#hasWikimediaDesktopApi } - public async hasWikimediaMobileApi(loginCookie: String): Promise { + public async hasWikimediaMobileApi(loginCookie: string): Promise { if (this.#hasWikimediaMobileApi === null) { this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.wikimediaMobileApiUrl.href) this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) @@ -190,7 +190,7 @@ class MediaWiki { return this.#hasWikimediaMobileApi } - public async hasVisualEditorApi(loginCookie: String): Promise { + public async hasVisualEditorApi(loginCookie: string): Promise { if (this.#hasVisualEditorApi === null) { this.visualEditorUrlDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) this.#hasVisualEditorApi = await checkApiAvailability( @@ -203,7 +203,7 @@ class MediaWiki { return this.#hasVisualEditorApi } - public async hasRestApi(loginCookie: String): Promise { + public async hasRestApi(loginCookie: string): Promise { if (this.#hasRestApi === null) { this.restApiUrlDirector = new RestApiURLDirector(this.restApiUrl.href) this.#hasRestApi = await checkApiAvailability(this.restApiUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie) diff --git a/src/renderers/renderer.builder.ts b/src/renderers/renderer.builder.ts index 7960663c..7139527c 100644 --- a/src/renderers/renderer.builder.ts +++ b/src/renderers/renderer.builder.ts @@ -8,7 +8,7 @@ import { RendererBuilderOptions } from './abstract.renderer.js' import * as logger from './../Logger.js' export class RendererBuilder { - public async createRenderer(options: RendererBuilderOptions, loginCookie: String): Promise { + public async createRenderer(options: RendererBuilderOptions, loginCookie: string): Promise { const { renderType, renderName } = options const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi] = await Promise.all([ From f49e935398e1b7c824ec70d1c12c08e88d093868 Mon Sep 17 00:00:00 2001 From: malik Date: Tue, 9 Jul 2024 13:15:32 +0200 Subject: [PATCH 5/6] pass loginCookie where required --- src/Downloader.ts | 2 +- test/unit/downloader.test.ts | 8 +-- test/unit/mwApi.test.ts | 6 +-- test/unit/mwApiCapabilities.test.ts | 56 ++++++++++---------- test/unit/renderers/renderer.builder.test.ts | 23 ++++---- test/unit/saveArticles.test.ts | 8 +-- test/util.ts | 8 +-- 7 files changed, 56 insertions(+), 55 deletions(-) diff --git a/src/Downloader.ts b/src/Downloader.ts index d96ffab4..b1cf852b 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -694,7 +694,7 @@ class Downloader { jsConfigVars = jsConfigVars.replace('nosuchaction', 'view') // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view' // Download mobile page dependencies only once - if ((await MediaWiki.hasWikimediaMobileApi()) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) { + if ((await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) { try { // TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources const mobileModulesData = await this.getJSON(`${MediaWiki.mobileModulePath}Test`) diff --git a/test/unit/downloader.test.ts b/test/unit/downloader.test.ts index 4dfc34ce..096c6034 100644 --- a/test/unit/downloader.test.ts +++ b/test/unit/downloader.test.ts @@ -34,10 +34,10 @@ describe('Downloader class', () => { await MediaWiki.getMwMetaData(downloader) await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - await MediaWiki.hasWikimediaMobileApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie) + await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie) + await MediaWiki.hasRestApi(downloader.loginCookie) + await MediaWiki.hasVisualEditorApi(downloader.loginCookie) }) test('Test Action API version 2 response in comparison with version 1', async () => { diff --git a/test/unit/mwApi.test.ts b/test/unit/mwApi.test.ts index 87bb9158..df20a68f 100644 --- a/test/unit/mwApi.test.ts +++ b/test/unit/mwApi.test.ts @@ -18,9 +18,9 @@ afterAll(stopRedis) const initMW = async (downloader: Downloader) => { await MediaWiki.getMwMetaData(downloader) await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie) + await MediaWiki.hasRestApi(downloader.loginCookie) + await MediaWiki.hasVisualEditorApi(downloader.loginCookie) await MediaWiki.getNamespaces([], downloader) } diff --git a/test/unit/mwApiCapabilities.test.ts b/test/unit/mwApiCapabilities.test.ts index 9ad50c3c..5bc1f113 100644 --- a/test/unit/mwApiCapabilities.test.ts +++ b/test/unit/mwApiCapabilities.test.ts @@ -14,28 +14,28 @@ describe('Checking Mediawiki capabilities', () => { test('test capabilities of en.wikipedia.org', async () => { MediaWiki.base = 'https://en.wikipedia.org' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(true) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(true) - expect(await MediaWiki.hasRestApi()).toBe(true) - expect(await MediaWiki.hasVisualEditorApi()).toBe(true) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(true) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(true) + expect(await MediaWiki.hasRestApi('')).toBe(true) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(true) }) test('test capabilities of wiki.openstreetmap.org', async () => { MediaWiki.base = 'https://wiki.openstreetmap.org' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasRestApi()).toBe(true) - expect(await MediaWiki.hasVisualEditorApi()).toBe(true) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasRestApi('')).toBe(true) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(true) }) test('test capabilities of fo.wikisource.org', async () => { MediaWiki.base = 'https://fo.wikisource.org' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(true) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasRestApi()).toBe(true) - expect(await MediaWiki.hasVisualEditorApi()).toBe(true) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(true) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasRestApi('')).toBe(true) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(true) }) test('test capabilities of minecraft.wiki with correct VisualEditor receipt', async () => { @@ -43,10 +43,10 @@ describe('Checking Mediawiki capabilities', () => { MediaWiki.wikiPath = '/' MediaWiki.actionApiPath = 'api.php' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasRestApi()).toBe(false) - expect(await MediaWiki.hasVisualEditorApi()).toBe(true) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasRestApi('')).toBe(false) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(true) }) test('test capabilities of pokemon.fandom.com with correct VisualEditor receipt', async () => { @@ -54,19 +54,19 @@ describe('Checking Mediawiki capabilities', () => { MediaWiki.wikiPath = '/' MediaWiki.actionApiPath = 'api.php' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasRestApi()).toBe(false) - expect(await MediaWiki.hasVisualEditorApi()).toBe(true) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasRestApi('')).toBe(false) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(true) }) test('test capabilities of pokemon.fandom.com with default receipt', async () => { MediaWiki.base = 'https://pokemon.fandom.com/' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasRestApi()).toBe(false) - expect(await MediaWiki.hasVisualEditorApi()).toBe(false) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasRestApi('')).toBe(false) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(false) }) test('test capabilities of pokemon.fandom.com with RestApi receipt', async () => { @@ -74,15 +74,15 @@ describe('Checking Mediawiki capabilities', () => { MediaWiki.wikiPath = '/' MediaWiki.restApiPath = 'rest.php' - expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false) - expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false) - expect(await MediaWiki.hasVisualEditorApi()).toBe(false) + expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false) + expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false) + expect(await MediaWiki.hasVisualEditorApi('')).toBe(false) /* TODO: Title MediaWiki:Sidebar does not exist for Mediawiki Rest Api in pokemon.fandom.com for some reason. This will lead to incorrect capability check See: https://pokemon.fandom.com/rest.php/v1/page/MediaWiki%3ASidebar/html */ MediaWiki.apiCheckArticleId = 'Volcarona' - expect(await MediaWiki.hasRestApi()).toBe(true) + expect(await MediaWiki.hasRestApi('')).toBe(true) }) }) diff --git a/test/unit/renderers/renderer.builder.test.ts b/test/unit/renderers/renderer.builder.test.ts index 4f0c6900..f7262829 100644 --- a/test/unit/renderers/renderer.builder.test.ts +++ b/test/unit/renderers/renderer.builder.test.ts @@ -5,6 +5,7 @@ import { RendererBuilderOptions } from '../../../src/renderers/abstract.renderer import { WikimediaDesktopRenderer } from '../../../src/renderers/wikimedia-desktop.renderer.js' import { VisualEditorRenderer } from '../../../src/renderers/visual-editor.renderer.js' import { RestApiRenderer } from '../../../src/renderers/rest-api.renderer.js' +import downloader from '../../../src/Downloader.js' jest.setTimeout(10000) @@ -20,7 +21,7 @@ describe('RendererBuilder', () => { const renderer = await rendererBuilder.createRenderer({ MediaWiki, renderType: 'desktop', - } as RendererBuilderOptions) + } as RendererBuilderOptions, '') expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) @@ -30,7 +31,7 @@ describe('RendererBuilder', () => { const renderer = await rendererBuilder.createRenderer({ MediaWiki, renderType: 'auto', - } as RendererBuilderOptions) + } as RendererBuilderOptions, '') expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) @@ -41,7 +42,7 @@ describe('RendererBuilder', () => { await rendererBuilder.createRenderer({ MediaWiki, renderType: 'unknownMode' as any, - } as RendererBuilderOptions) + } as RendererBuilderOptions, '') }).rejects.toThrow('Unknown render: unknownMode') }) @@ -57,7 +58,7 @@ describe('RendererBuilder', () => { renderName: 'VisualEditor', } - const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions) + const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions, '') expect(renderer).toBeInstanceOf(VisualEditorRenderer) }) @@ -74,7 +75,7 @@ describe('RendererBuilder', () => { renderName: 'WikimediaDesktop', } - const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions) + const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions, '') expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) @@ -91,7 +92,7 @@ describe('RendererBuilder', () => { renderName: 'RestApi', } - const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions) + const renderer = await rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions, '') expect(renderer).toBeInstanceOf(RestApiRenderer) }) @@ -99,10 +100,10 @@ describe('RendererBuilder', () => { it('should throw an error for unknown RendererAPI in specific mode', async () => { const { downloader, MediaWiki } = await setupScrapeClasses() // en wikipedia await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - await MediaWiki.hasWikimediaMobileApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie) + await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie) + await MediaWiki.hasRestApi(downloader.loginCookie) + await MediaWiki.hasVisualEditorApi(downloader.loginCookie) const rendererBuilderOptions = { MediaWiki, @@ -110,7 +111,7 @@ describe('RendererBuilder', () => { renderName: 'UnknownAPI', // Using an invalid RendererAPI for the test } - expect(async () => rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions)).rejects.toThrow( + expect(async () => rendererBuilder.createRenderer(rendererBuilderOptions as RendererBuilderOptions, downloader.loginCookie)).rejects.toThrow( `Unknown renderName for specific mode: ${rendererBuilderOptions.renderName}`, ) }) diff --git a/test/unit/saveArticles.test.ts b/test/unit/saveArticles.test.ts index 085a92c9..c3ffdb81 100644 --- a/test/unit/saveArticles.test.ts +++ b/test/unit/saveArticles.test.ts @@ -41,10 +41,10 @@ describe('saveArticles', () => { test(`Article html processing using ${renderer} renderer`, async () => { const { MediaWiki, downloader, dump } = await setupScrapeClasses() // en wikipedia await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - await MediaWiki.hasWikimediaMobileApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi('') + await MediaWiki.hasWikimediaMobileApi('') + await MediaWiki.hasRestApi('') + await MediaWiki.hasVisualEditorApi('') const _articlesDetail = await downloader.getArticleDetailsIds(['London']) const articlesDetail = mwRetToArticleDetail(_articlesDetail) diff --git a/test/util.ts b/test/util.ts index 06204a58..29582779 100644 --- a/test/util.ts +++ b/test/util.ts @@ -41,10 +41,10 @@ export async function setupScrapeClasses({ mwUrl = 'https://en.wikipedia.org', f await MediaWiki.getMwMetaData(downloader) await MediaWiki.hasCoordinates(downloader) - await MediaWiki.hasWikimediaDesktopApi() - await MediaWiki.hasWikimediaMobileApi() - await MediaWiki.hasRestApi() - await MediaWiki.hasVisualEditorApi() + await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie) + await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie) + await MediaWiki.hasRestApi(downloader.loginCookie) + await MediaWiki.hasVisualEditorApi(downloader.loginCookie) const dump = new Dump(format, {} as any, MediaWiki.metaData) From 5f72e0ac0bf0d3f403f70a21af929b2cd8586948 Mon Sep 17 00:00:00 2001 From: malik Date: Sat, 13 Jul 2024 21:43:55 +0200 Subject: [PATCH 6/6] reformat --- src/Downloader.ts | 42 +++++++++++++++++--- src/util/saveArticles.ts | 30 +++++++------- test/unit/renderers/renderer.builder.test.ts | 33 +++++++++------ 3 files changed, 72 insertions(+), 33 deletions(-) diff --git a/src/Downloader.ts b/src/Downloader.ts index b1cf852b..26b81f63 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -34,7 +34,13 @@ imageminOptions.set('default', new Map()) imageminOptions.set('webp', new Map()) imageminOptions.get('default').set('image/png', { - plugins: [(imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }), imageminAdvPng({ optimizationLevel: 4, iterations: 5 })], + plugins: [ + (imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }), + imageminAdvPng({ + optimizationLevel: 4, + iterations: 5, + }), + ], }) imageminOptions.get('default').set('image/jpeg', { plugins: [imageminJpegoptim({ max: 60, stripAll: true })], @@ -81,6 +87,7 @@ export const defaultStreamRequestOptions: AxiosRequestConfig = { } type URLDirector = WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector | RestApiURLDirector + /** * Downloader is a class providing content retrieval functionalities for both Mediawiki and S3 remote instances. */ @@ -258,7 +265,13 @@ class Downloader { return finalProcessedResp } - public async getArticleDetailsNS(ns: number, gapcontinue = ''): Promise<{ gapContinue: string; articleDetails: QueryMwRet }> { + public async getArticleDetailsNS( + ns: number, + gapcontinue = '', + ): Promise<{ + gapContinue: string + articleDetails: QueryMwRet + }> { let queryContinuation: QueryContinueOpts let finalProcessedResp: QueryMwRet let gCont: string = null @@ -366,7 +379,13 @@ class Downloader { }) } - public async downloadContent(_url: string, retry = true): Promise<{ content: Buffer | string; responseHeaders: any }> { + public async downloadContent( + _url: string, + retry = true, + ): Promise<{ + content: Buffer | string + responseHeaders: any + }> { if (!_url) { throw new Error(`Parameter [${_url}] is not a valid url`) } @@ -616,7 +635,16 @@ class Downloader { handler(err) } - private async getSubCategories(articleId: string, continueStr = ''): Promise> { + private async getSubCategories( + articleId: string, + continueStr = '', + ): Promise< + Array<{ + pageid: number + ns: number + title: string + }> + > { const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr)) @@ -694,7 +722,11 @@ class Downloader { jsConfigVars = jsConfigVars.replace('nosuchaction', 'view') // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view' // Download mobile page dependencies only once - if ((await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) { + if ( + (await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) && + this.wikimediaMobileJsDependenciesList.length === 0 && + this.wikimediaMobileStyleDependenciesList.length === 0 + ) { try { // TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources const mobileModulesData = await this.getJSON(`${MediaWiki.mobileModulePath}Test`) diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index 97b1a1c9..351d7a63 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -247,17 +247,23 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade let articlesRenderer if (forceRender) { // All articles and main page will use the same renderer if 'forceRender' is specified - const renderer = await rendererBuilder.createRenderer({ - renderType: 'specific', - renderName: forceRender, - }, downloader.loginCookie) + const renderer = await rendererBuilder.createRenderer( + { + renderType: 'specific', + renderName: forceRender, + }, + downloader.loginCookie, + ) mainPageRenderer = renderer articlesRenderer = renderer } else { mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' }, downloader.loginCookie) - articlesRenderer = await rendererBuilder.createRenderer({ - renderType: hasWikimediaMobileApi ? 'mobile' : 'auto', - }, downloader.loginCookie) + articlesRenderer = await rendererBuilder.createRenderer( + { + renderType: hasWikimediaMobileApi ? 'mobile' : 'auto', + }, + downloader.loginCookie, + ) } downloader.setUrlsDirectors(mainPageRenderer, articlesRenderer) @@ -301,15 +307,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage) - for (const { - articleId, - displayTitle: articleTitle, - html: finalHTML, - mediaDependencies, - moduleDependencies, - staticFiles, - subtitles - } of rets) { + for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, moduleDependencies, staticFiles, subtitles } of rets) { if (!finalHTML) { logger.warn(`No HTML returned for article [${articleId}], skipping`) continue diff --git a/test/unit/renderers/renderer.builder.test.ts b/test/unit/renderers/renderer.builder.test.ts index f7262829..246ff5c6 100644 --- a/test/unit/renderers/renderer.builder.test.ts +++ b/test/unit/renderers/renderer.builder.test.ts @@ -18,20 +18,26 @@ describe('RendererBuilder', () => { it('should create a WikimediaDesktopRenderer for desktop mode', async () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia - const renderer = await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'desktop', - } as RendererBuilderOptions, '') + const renderer = await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'desktop', + } as RendererBuilderOptions, + '', + ) expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) it('should create a WikimediaDesktopRenderer for auto mode for en wikipedia', async () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia - const renderer = await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'auto', - } as RendererBuilderOptions, '') + const renderer = await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'auto', + } as RendererBuilderOptions, + '', + ) expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) @@ -39,10 +45,13 @@ describe('RendererBuilder', () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia expect(async () => { - await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'unknownMode' as any, - } as RendererBuilderOptions, '') + await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'unknownMode' as any, + } as RendererBuilderOptions, + '', + ) }).rejects.toThrow('Unknown render: unknownMode') })