Move static files list preparation into renders (partial impl)

Author: VadimKovalenkoSNF

Diff for: res/wm_mobile/wm_mobile_override_script.js renamed to res/wm_mobile_override_script.js

@@ -37,7 +37,6 @@ import {
   mkdirPromise,
   sanitizeString,
   saveStaticFiles,
-  saveStaticWikimediaMobileFiles,
   importPolyfillModules,
   extractArticleList,
   getTmpDirectory,
@@ -400,11 +399,6 @@ async function execute(argv: any) {
     })
     zimCreator.addArticle(scraperArticle)
 
-    logger.info('Copying Static Wikimedia Mobile Override Files')
-    await saveStaticWikimediaMobileFiles(config, zimCreator)
-    logger.info('Copying Static Resource Files')
-    await saveStaticFiles(config, zimCreator)
-
     logger.info('Finding stylesheets to download')
     const stylesheetsToGet = await dump.getRelevantStylesheetUrls(downloader)
     logger.log(`Found [${stylesheetsToGet.length}] stylesheets to download`)
@@ -424,12 +418,15 @@ async function execute(argv: any) {
 
     logger.log('Getting articles')
     stime = Date.now()
-    const { jsModuleDependencies, cssModuleDependencies } = await saveArticles(zimCreator, downloader, dump, hasWikimediaMobileApi, forceRender)
+    const { jsModuleDependencies, cssModuleDependencies, staticFilesList } = await saveArticles(zimCreator, downloader, dump, hasWikimediaMobileApi, forceRender)
     logger.log(`Fetching Articles finished in ${(Date.now() - stime) / 1000} seconds`)
 
     logger.log(`Found [${jsModuleDependencies.size}] js module dependencies`)
     logger.log(`Found [${cssModuleDependencies.size}] style module dependencies`)
 
+    logger.info('Copying Static Resource Files')
+    await saveStaticFiles(staticFilesList, zimCreator)
+
     const allDependenciesWithType = [
       { type: 'js', moduleList: Array.from(jsModuleDependencies) },
       { type: 'css', moduleList: Array.from(cssModuleDependencies) },

Diff for: res/wm_mobile/wm_mobile_override_style.css renamed to res/wm_mobile_override_style.css

@@ -56,6 +56,7 @@ export interface RenderSingleOutput {
   html: string
   mediaDependencies: any
   moduleDependencies: any
+  staticFiles: string[]
   subtitles: any
 }
 

Diff for: src/mwoffliner.lib.ts

@@ -1,14 +1,17 @@
 import * as domino from 'domino'
 import { Renderer } from './abstract.renderer.js'
+import { getStaticFiles } from '../util/misc.js'
 import { config } from '../config.js'
 import MediaWiki from '../MediaWiki.js'
 
 import { htmlWikimediaDesktopTemplateCode } from '../Templates.js'
 import { genCanonicalLink, genHeaderScript, genHeaderCSSLink } from '../util/misc.js'
 
 export abstract class DesktopRenderer extends Renderer {
+  public staticFilesListDesktop: string[] = []
   constructor() {
     super()
+    this.staticFilesListDesktop = getStaticFiles(config.output.jsResources, config.output.mainPageCssResources.concat(config.output.cssResources))
   }
 
   public filterWikimediaDesktopModules(_moduleDependencies) {

Diff for: src/renderers/abstract.renderer.ts

@@ -1,14 +1,17 @@
 import * as domino from 'domino'
 import { Renderer } from './abstract.renderer.js'
+import { getStaticFiles } from '../util/misc.js'
 import { config } from '../config.js'
 import MediaWiki from '../MediaWiki.js'
 
 import { htmlWikimediaMobileTemplateCode } from '../Templates.js'
 import { genCanonicalLink, genHeaderScript, genHeaderCSSLink } from '../util/misc.js'
 
 export abstract class MobileRenderer extends Renderer {
+  public staticFilesListMobile: string[] = []
   constructor() {
     super()
+    this.staticFilesListMobile = getStaticFiles(config.output.mwMobileJsResources, config.output.wikimediaMobileCssResources)
   }
 
   public filterWikimediaMobileModules(_moduleDependencies) {

Diff for: src/renderers/abstractDesktop.render.ts

@@ -71,6 +71,7 @@ export class VisualEditorRenderer extends DesktopRenderer {
           html: finalHTML,
           mediaDependencies,
           moduleDependencies: moduleDependenciesFiltered,
+          staticFiles: this.staticFilesListDesktop,
           subtitles,
         })
         return result

Diff for: src/renderers/abstractMobile.render.ts

@@ -69,6 +69,7 @@ export class WikimediaDesktopRenderer extends DesktopRenderer {
         html: finalHTML,
         mediaDependencies,
         moduleDependencies: moduleDependenciesFiltered,
+        staticFiles: this.staticFilesListDesktop,
         subtitles,
       })
     }

Diff for: src/renderers/visual-editor.renderer.ts

@@ -33,7 +33,7 @@ export class WikimediaMobileRenderer extends MobileRenderer {
         const mobileHTML = domino.createDocument(data)
         const finalHTMLMobile = await this.pipeMobileTransformations(
           mobileHTML,
-          this.convertLazyLoadToImages.bind(this),
+          this.convertLazyLoadToImages,
           this.removeEditContainer,
           this.removeHiddenClass,
           async (doc) => {
@@ -60,6 +60,7 @@ export class WikimediaMobileRenderer extends MobileRenderer {
           html: finalHTMLMobile.documentElement.outerHTML,
           mediaDependencies: mediaDependenciesVal,
           moduleDependencies: moduleDependenciesFiltered,
+          staticFiles: this.staticFilesListMobile,
           subtitles: subtitlesVal,
         })
         return result
@@ -109,34 +110,6 @@ export class WikimediaMobileRenderer extends MobileRenderer {
       span.parentNode.replaceChild(img, span)
     })
 
-    doc = this.resizeMobileImages(doc)
-
-    return doc
-  }
-
-  private resizeMobileImages(doc: DominoElement) {
-    const mobileImageWidth = 375
-    const imageWidthPattern = /(\.jpg\/|\.png\/|\.svg\/|\.gif\/)(\d+)px/i
-
-    // Directly filter images hosted on Commons wiki
-    const imgs: NodeList = doc.querySelectorAll('img[src*="/commons/"]')
-
-    imgs.forEach((img: DominoElement) => {
-      const imgWidth = img.getAttribute('width')
-      const imgHeight = img.getAttribute('height')
-      const imgSrc = img.getAttribute('src')
-      const imageWidthMatchSrc = imgSrc.match(imageWidthPattern)
-
-      if (imgWidth && imgWidth > mobileImageWidth && imageWidthMatchSrc) {
-        const heightScaleFactor = Math.round((imgWidth / imgHeight) * 100) / 100
-        const newImgSrc = imgSrc.replace(imageWidthMatchSrc[2], mobileImageWidth.toString())
-
-        img.setAttribute('src', newImgSrc)
-        img.setAttribute('width', mobileImageWidth.toString())
-        img.setAttribute('height', Math.round(mobileImageWidth / heightScaleFactor).toString())
-      }
-    })
-
     return doc
   }
 

Diff for: src/renderers/wikimedia-desktop.renderer.ts

@@ -162,35 +162,22 @@ export function interpolateTranslationString(str: string, parameters: { [key: st
   return newString
 }
 
-function saveResourceFile(resource: string, type: 'css' | 'js', basePath: string, config: Config, zimCreator: ZimCreator) {
-  return async () => {
-    try {
-      const content = await readFilePromise(pathParser.resolve(__dirname, `../../res/${basePath}${resource}.${type}`))
-      const article = new ZimArticle({
-        url: type === 'css' ? cssPath(resource) : jsPath(resource),
-        data: content,
-        ns: '-',
-      })
+export async function saveStaticFiles(staticFiles: Set<string>, zimCreator: ZimCreator) {
+  try {
+    staticFiles.forEach(async (file) => {
+      const staticFilesContent = await readFilePromise(pathParser.resolve(__dirname, `../../res/${file}`))
+      const article = new ZimArticle({ url: file.endsWith('.css') ? cssPath(file) : jsPath(file), data: staticFilesContent, ns: '-' })
       zimCreator.addArticle(article)
-    } catch (error) {
-      const fileType = type === 'css' ? (basePath.includes('wm_mobile') ? 'style Wikimedia mobile override' : 'style') : 'script'
-      logger.warn(`Could not create ${fileType} ${resource} file : ${error}`)
-    }
+    })
+  } catch (err) {
+    logger.error(err)
   }
 }
 
-export function saveStaticFiles(config: Config, zimCreator: ZimCreator) {
-  const cssPromises = config.output.cssResources.concat(config.output.mainPageCssResources).map((css) => saveResourceFile(css, 'css', '', config, zimCreator)())
-  const jsPromises = config.output.jsResources.map((js) => saveResourceFile(js, 'js', '', config, zimCreator)())
-  return Promise.all([...cssPromises, ...jsPromises])
-}
-
-export function saveStaticWikimediaMobileFiles(config: Config, zimCreator: ZimCreator) {
-  const wikimediaMobileCssPromises = config.output.wikimediaMobileCssResources.map((wikimediaMobileCss) =>
-    saveResourceFile(wikimediaMobileCss, 'css', 'wm_mobile/', config, zimCreator)(),
-  )
-  const wikimediaMobileJsPromises = config.output.mwMobileJsResources.map((wikimediaMobileJs) => saveResourceFile(wikimediaMobileJs, 'js', 'wm_mobile/', config, zimCreator)())
-  return Promise.all([...wikimediaMobileCssPromises, ...wikimediaMobileJsPromises])
+export function getStaticFiles(jsStaticFiles: string[], cssStaticFiles: string[]): string[] {
+  jsStaticFiles = jsStaticFiles.map((jsFile) => jsFile.concat('.js'))
+  cssStaticFiles = cssStaticFiles.map((cssFile) => cssFile.concat('.css'))
+  return jsStaticFiles.concat(cssStaticFiles)
 }
 
 export function cssPath(css: string, subDirectory = '') {

Diff for: src/renderers/wikimedia-mobile.renderer.ts

@@ -234,6 +234,7 @@ export function getArticleUrl(downloader: Downloader, dump: Dump, articleId: str
 export async function saveArticles(zimCreator: ZimCreator, downloader: Downloader, dump: Dump, hasWikimediaMobileApi: boolean, forceRender = null) {
   const jsModuleDependencies = new Set<string>()
   const cssModuleDependencies = new Set<string>()
+  const staticFilesList = new Set<string>()
   let jsConfigVars = ''
   let prevPercentProgress: string
   const { articleDetailXId } = RedisStore
@@ -298,7 +299,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
 
           rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage)
 
-          for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, moduleDependencies, subtitles } of rets) {
+          for (const { articleId, displayTitle: articleTitle, html: finalHTML, mediaDependencies, moduleDependencies, staticFiles, subtitles } of rets) {
             if (!finalHTML) {
               logger.warn(`No HTML returned for article [${articleId}], skipping`)
               continue
@@ -312,6 +313,10 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
               cssModuleDependencies.add(dep)
             }
 
+            for (const file of staticFiles) {
+              staticFilesList.add(file)
+            }
+
             jsConfigVars = moduleDependencies.jsConfigVars || ''
 
             /*
@@ -393,6 +398,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
   }
 
   return {
+    staticFilesList,
     jsModuleDependencies,
     cssModuleDependencies,
   }