diff --git a/src/Downloader.ts b/src/Downloader.ts index b1cf852b..26b81f63 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -34,7 +34,13 @@ imageminOptions.set('default', new Map()) imageminOptions.set('webp', new Map()) imageminOptions.get('default').set('image/png', { - plugins: [(imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }), imageminAdvPng({ optimizationLevel: 4, iterations: 5 })], + plugins: [ + (imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }), + imageminAdvPng({ + optimizationLevel: 4, + iterations: 5, + }), + ], }) imageminOptions.get('default').set('image/jpeg', { plugins: [imageminJpegoptim({ max: 60, stripAll: true })], @@ -81,6 +87,7 @@ export const defaultStreamRequestOptions: AxiosRequestConfig = { } type URLDirector = WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector | RestApiURLDirector + /** * Downloader is a class providing content retrieval functionalities for both Mediawiki and S3 remote instances. */ @@ -258,7 +265,13 @@ class Downloader { return finalProcessedResp } - public async getArticleDetailsNS(ns: number, gapcontinue = ''): Promise<{ gapContinue: string; articleDetails: QueryMwRet }> { + public async getArticleDetailsNS( + ns: number, + gapcontinue = '', + ): Promise<{ + gapContinue: string + articleDetails: QueryMwRet + }> { let queryContinuation: QueryContinueOpts let finalProcessedResp: QueryMwRet let gCont: string = null @@ -366,7 +379,13 @@ class Downloader { }) } - public async downloadContent(_url: string, retry = true): Promise<{ content: Buffer | string; responseHeaders: any }> { + public async downloadContent( + _url: string, + retry = true, + ): Promise<{ + content: Buffer | string + responseHeaders: any + }> { if (!_url) { throw new Error(`Parameter [${_url}] is not a valid url`) } @@ -616,7 +635,16 @@ class Downloader { handler(err) } - private async getSubCategories(articleId: string, continueStr = ''): Promise> { + private async getSubCategories( + articleId: string, + continueStr = '', + ): Promise< + Array<{ + pageid: number + ns: number + title: string + }> + > { const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href) const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr)) @@ -694,7 +722,11 @@ class Downloader { jsConfigVars = jsConfigVars.replace('nosuchaction', 'view') // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view' // Download mobile page dependencies only once - if ((await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) { + if ( + (await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) && + this.wikimediaMobileJsDependenciesList.length === 0 && + this.wikimediaMobileStyleDependenciesList.length === 0 + ) { try { // TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources const mobileModulesData = await this.getJSON(`${MediaWiki.mobileModulePath}Test`) diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index 97b1a1c9..351d7a63 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -247,17 +247,23 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade let articlesRenderer if (forceRender) { // All articles and main page will use the same renderer if 'forceRender' is specified - const renderer = await rendererBuilder.createRenderer({ - renderType: 'specific', - renderName: forceRender, - }, downloader.loginCookie) + const renderer = await rendererBuilder.createRenderer( + { + renderType: 'specific', + renderName: forceRender, + }, + downloader.loginCookie, + ) mainPageRenderer = renderer articlesRenderer = renderer } else { mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' }, downloader.loginCookie) - articlesRenderer = await rendererBuilder.createRenderer({ - renderType: hasWikimediaMobileApi ? 'mobile' : 'auto', - }, downloader.loginCookie) + articlesRenderer = await rendererBuilder.createRenderer( + { + renderType: hasWikimediaMobileApi ? 'mobile' : 'auto', + }, + downloader.loginCookie, + ) } downloader.setUrlsDirectors(mainPageRenderer, articlesRenderer) @@ -301,15 +307,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage) - for (const { - articleId, - displayTitle: articleTitle, - html: finalHTML, - mediaDependencies, - moduleDependencies, - staticFiles, - subtitles - } of rets) { + for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, moduleDependencies, staticFiles, subtitles } of rets) { if (!finalHTML) { logger.warn(`No HTML returned for article [${articleId}], skipping`) continue diff --git a/test/unit/renderers/renderer.builder.test.ts b/test/unit/renderers/renderer.builder.test.ts index f7262829..246ff5c6 100644 --- a/test/unit/renderers/renderer.builder.test.ts +++ b/test/unit/renderers/renderer.builder.test.ts @@ -18,20 +18,26 @@ describe('RendererBuilder', () => { it('should create a WikimediaDesktopRenderer for desktop mode', async () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia - const renderer = await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'desktop', - } as RendererBuilderOptions, '') + const renderer = await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'desktop', + } as RendererBuilderOptions, + '', + ) expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) it('should create a WikimediaDesktopRenderer for auto mode for en wikipedia', async () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia - const renderer = await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'auto', - } as RendererBuilderOptions, '') + const renderer = await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'auto', + } as RendererBuilderOptions, + '', + ) expect(renderer).toBeInstanceOf(WikimediaDesktopRenderer) }) @@ -39,10 +45,13 @@ describe('RendererBuilder', () => { const { MediaWiki } = await setupScrapeClasses() // en wikipedia expect(async () => { - await rendererBuilder.createRenderer({ - MediaWiki, - renderType: 'unknownMode' as any, - } as RendererBuilderOptions, '') + await rendererBuilder.createRenderer( + { + MediaWiki, + renderType: 'unknownMode' as any, + } as RendererBuilderOptions, + '', + ) }).rejects.toThrow('Unknown render: unknownMode') })