Implement Mediawiki REST API render

Author: VadimKovalenkoSNF

src/Downloader.ts

@@ -177,13 +177,15 @@ class Downloader {
         { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
         { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
         { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
+        { condition: await MediaWiki.hasMediawikiRestApi(), value: MediaWiki.mediawikiRestApiUrl.href },
       ])
 
       //* Objects order in array matters!
       this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([
         { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
         { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
         { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
+        { condition: await MediaWiki.hasMediawikiRestApi(), value: MediaWiki.mediawikiRestApiUrl.href },
       ])
     } else {
       switch (forceRender) {
@@ -208,6 +210,13 @@ class Downloader {
             break
           }
           break
+        case 'MediawikiRestApi':
+          if (MediaWiki.hasMediawikiRestApi()) {
+            this.baseUrl = MediaWiki.mediawikiRestApiUrl.href
+            this.baseUrlForMainPage = MediaWiki.mediawikiRestApiUrl.href
+            break
+          }
+          break
         default:
           throw new Error('Unable to find specific API end-point to retrieve article HTML')
       }

src/MediaWiki.ts

@@ -12,6 +12,7 @@ import ApiURLDirector from './util/builders/url/api.director.js'
 import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
 import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
 import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
+import MediawikiRestApiDirector from './util/builders/url/mediawiki-rest-api.director.js'
 import { checkApiAvailability } from './util/mw-api.js'
 import { BLACKLISTED_NS } from './util/const.js'
 
@@ -47,6 +48,7 @@ class MediaWiki {
   #actionApiPath: string
   #restApiPath: string
   #modulePathOpt: string
+  #mediawikiRestApiPath: string
   #username: string
   #password: string
   #domain: string
@@ -55,6 +57,7 @@ class MediaWiki {
   private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
   private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
   private visualEditorURLDirector: VisualEditorURLDirector
+  private mediawikiRestApiDirector: MediawikiRestApiDirector
 
   public visualEditorApiUrl: URL
   public actionApiUrl: URL
@@ -63,10 +66,12 @@ class MediaWiki {
   public webUrl: URL
   public WikimediaDesktopApiUrl: URL
   public WikimediaMobileApiUrl: URL
+  public mediawikiRestApiUrl: URL
 
   #hasWikimediaDesktopApi: boolean | null
   #hasWikimediaMobileApi: boolean | null
   #hasVisualEditorApi: boolean | null
+  #hasMediawikiRestApi: boolean | null
   #hasCoordinates: boolean | null
 
   set username(value: string) {
@@ -134,6 +139,8 @@ class MediaWiki {
 
     this.#actionApiPath = 'w/api.php'
     this.#restApiPath = 'api/rest_v1'
+    // TODO:  there is no CLI param for this api yet
+    this.#mediawikiRestApiPath = 'w/rest.php/v1/page/'
     this.#wikiPath = 'wiki/'
     this.#modulePathOpt = 'w/load.php'
 
@@ -154,6 +161,7 @@ class MediaWiki {
     this.#hasWikimediaDesktopApi = null
     this.#hasWikimediaMobileApi = null
     this.#hasVisualEditorApi = null
+    this.#hasMediawikiRestApi = null
     this.#hasCoordinates = null
   }
 
@@ -185,6 +193,14 @@ class MediaWiki {
     return this.#hasVisualEditorApi
   }
 
+  public async hasMediawikiRestApi(): Promise<boolean> {
+    if (this.#hasMediawikiRestApi === null) {
+      this.#hasMediawikiRestApi = await checkApiAvailability(this.mediawikiRestApiDirector.buildArticleURL(this.apiCheckArticleId))
+      return this.#hasMediawikiRestApi
+    }
+    return this.#hasMediawikiRestApi
+  }
+
   public async hasCoordinates(downloader: Downloader): Promise<boolean> {
     if (this.#hasCoordinates === null) {
       const validNamespaceIds = this.namespacesToMirror.map((ns) => this.namespaces[ns].num)
@@ -215,9 +231,12 @@ class MediaWiki {
   private initApiURLDirector() {
     this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath)
     this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath)
+    // TODO: refactor this workaround once PR#1929 is merged
+    this.mediawikiRestApiUrl = this.baseUrlDirector.buildURL(this.#mediawikiRestApiPath)
     this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
     this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
     this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
+    this.mediawikiRestApiDirector = new MediawikiRestApiDirector(this.baseUrl.href)
   }
 
   public async login(downloader: Downloader) {
@@ -442,6 +461,7 @@ class MediaWiki {
     const mwMetaData: MWMetaData = {
       webUrl: this.webUrl.href,
       actionApiUrl: this.actionApiUrl.href,
+      mediawikiRestApiPath: this.mediawikiRestApiUrl.href,
       modulePathOpt: this.#modulePathOpt,
       modulePath: this.modulePath,
       mobileModulePath: this.mobileModulePath,

src/mwoffliner.lib.ts

@@ -214,6 +214,7 @@ async function execute(argv: any) {
   await MediaWiki.hasWikimediaDesktopApi()
   const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
   await MediaWiki.hasVisualEditorApi()
+  await MediaWiki.hasMediawikiRestApi()
   await downloader.setBaseUrls(forceRender)
 
   RedisStore.setOptions(argv.redis || config.defaults.redisPath)

src/renderers/abstract.renderer.ts

@@ -22,7 +22,7 @@ import {
 } from '../util/misc.js'
 
 type renderType = 'auto' | 'desktop' | 'mobile' | 'specific'
-type renderName = 'VisualEditor' | 'WikimediaDesktop' | 'WikimediaMobile'
+type renderName = 'VisualEditor' | 'WikimediaDesktop' | 'WikimediaMobile' | 'MediawikiRestApi'
 
 interface RendererBuilderOptionsBase {
   renderType: renderType

src/renderers/mediawiki-rest-api.renderer.ts

@@ -0,0 +1,7 @@
+import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js'
+
+export class MediawikiRestApiRenderer extends WikimediaDesktopRenderer {
+  constructor() {
+    super()
+  }
+}

src/renderers/renderer.builder.ts

@@ -3,17 +3,19 @@ import { Renderer } from './abstract.renderer.js'
 import { VisualEditorRenderer } from './visual-editor.renderer.js'
 import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js'
 import { WikimediaMobileRenderer } from './wikimedia-mobile.renderer.js'
+import { MediawikiRestApiRenderer } from './mediawiki-rest-api.renderer.js'
 import { RendererBuilderOptions } from './abstract.renderer.js'
 import * as logger from './../Logger.js'
 
 export class RendererBuilder {
   public async createRenderer(options: RendererBuilderOptions): Promise<Renderer> {
     const { renderType, renderName } = options
 
-    const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi] = await Promise.all([
+    const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasMediawikiRestApi] = await Promise.all([
       MediaWiki.hasVisualEditorApi(),
       MediaWiki.hasWikimediaDesktopApi(),
       MediaWiki.hasWikimediaMobileApi(),
+      MediaWiki.hasMediawikiRestApi(),
     ])
 
     switch (renderType) {
@@ -23,6 +25,8 @@ export class RendererBuilder {
           return new WikimediaDesktopRenderer()
         } else if (hasVisualEditorApi) {
           return new VisualEditorRenderer()
+        } else if (hasMediawikiRestApi) {
+          return new MediawikiRestApiRenderer()
         } else {
           logger.error('No available desktop renderer.')
           process.exit(1)
@@ -39,6 +43,8 @@ export class RendererBuilder {
           return new WikimediaDesktopRenderer()
         } else if (hasVisualEditorApi) {
           return new VisualEditorRenderer()
+        } else if (hasMediawikiRestApi) {
+          return new MediawikiRestApiRenderer()
         } else if (hasWikimediaMobileApi) {
           return new WikimediaMobileRenderer()
         } else {
@@ -47,13 +53,13 @@ export class RendererBuilder {
         }
       case 'specific':
         // renderName argument is required for 'specific' mode
-        return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi)
+        return this.handleSpecificRender(renderName, hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasMediawikiRestApi)
       default:
         throw new Error(`Unknown render: ${renderType}`)
     }
   }
 
-  private handleSpecificRender(renderName: string, hasVisualEditorApi: boolean, hasWikimediaDesktopApi: boolean, hasWikimediaMobileApi: boolean) {
+  private handleSpecificRender(renderName: string, hasVisualEditorApi: boolean, hasWikimediaDesktopApi: boolean, hasWikimediaMobileApi: boolean, hasMediawikiRestApi: boolean) {
     // renderName argument is required for 'specific' mode
     switch (renderName) {
       case 'WikimediaDesktop':
@@ -74,6 +80,12 @@ export class RendererBuilder {
         }
         logger.error('No available mobile renderer.')
         process.exit(1)
+      case 'MediawikiRestApi':
+        if (hasMediawikiRestApi) {
+          return new MediawikiRestApiRenderer()
+        }
+        logger.error('Cannot create an instance of MediawikiRestApi renderer.')
+        process.exit(1)
       default:
         throw new Error(`Unknown renderName for specific mode: ${renderName}`)
     }

src/types.d.ts

@@ -160,6 +160,7 @@ interface MWMetaData {
   baseUrl: string
   wikiPath: string
   actionApiPath: string
+  mediawikiRestApiPath: string
   restApiPath: string
   domain: string
   webUrl: string

src/util/builders/url/mediawiki-rest-api.director.ts

@@ -0,0 +1,18 @@
+import urlBuilder from './url.builder.js'
+
+/**
+ * Interface to build URLs based on MediaWiki REST API URL
+ */
+export default class MediawikiRestApiURL {
+  baseDomain: string
+
+  constructor(baseDomain: string) {
+    this.baseDomain = baseDomain
+  }
+
+  buildArticleURL(articleId?: string) {
+    const base = urlBuilder.setDomain(this.baseDomain).build()
+    // TODO: refactor this workaround once PR#1929 is merged
+    return articleId ? `${base}/w/rest.php/v1/page/${articleId}/html` : `${base}/w/rest.php/v1/page/`
+  }
+}

src/util/const.ts

@@ -20,4 +20,4 @@ export const RULE_TO_REDIRECT = /window\.top !== window\.self/
 export const WEBP_HANDLER_URL = 'https://gist.githubusercontent.com/rgaudin/60bb9cc6f187add506584258028b8ee1/raw/9d575b8e25d67eed2a9c9a91d3e053a0062d2fc7/web-handler.js'
 export const MAX_FILE_DOWNLOAD_RETRIES = 5
 export const BLACKLISTED_NS = ['Story'] // 'Story' Wikipedia namespace is content, but not indgestable by Parsoid https://github.com/openzim/mwoffliner/issues/1853
-export const RENDERERS_LIST = ['WikimediaDesktop', 'VisualEditor', 'WikimediaMobile']
+export const RENDERERS_LIST = ['WikimediaDesktop', 'VisualEditor', 'WikimediaMobile', 'MediawikiRestApi']

src/util/saveArticles.ts

@@ -131,9 +131,9 @@ async function getAllArticlesToKeep(downloader: Downloader, articleDetailXId: RK
       const _moduleDependencies = await downloader.getModuleDependencies(articleDetail.title)
       let rets: any
       try {
-        const articleUrl = getArticleUrl(downloader, dump, articleId)
         const isMainPage = dump.isMainPage(articleId)
         const renderer = isMainPage ? mainPageRenderer : articlesRenderer
+        const articleUrl = getArticleUrl(downloader, dump, articleId, renderer)
 
         rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage)
         for (const { articleId, html } of rets) {
@@ -224,8 +224,12 @@ async function saveArticle(
   }
 }
 
-export function getArticleUrl(downloader: Downloader, dump: Dump, articleId: string): string {
-  return `${dump.isMainPage(articleId) ? downloader.baseUrlForMainPage : downloader.baseUrl}${encodeURIComponent(articleId)}`
+// TODO: remove this workaround once PR#1929 is merged
+export function getArticleUrl(downloader: Downloader, dump: Dump, articleId: string, renderer?): string {
+  const baseUrl = dump.isMainPage(articleId) ? downloader.baseUrlForMainPage : downloader.baseUrl
+  const articleUrl = `${baseUrl}${encodeURIComponent(articleId)}`
+
+  return renderer?.constructor?.name === 'MediawikiRestApiRenderer' ? `${articleUrl}/html` : articleUrl
 }
 
 /*
@@ -293,9 +297,9 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
 
         let rets: any
         try {
-          const articleUrl = getArticleUrl(downloader, dump, articleId)
           const isMainPage = dump.isMainPage(articleId)
           const renderer = isMainPage ? mainPageRenderer : articlesRenderer
+          const articleUrl = getArticleUrl(downloader, dump, articleId, renderer)
 
           rets = await downloader.getArticle(downloader.webp, _moduleDependencies, articleId, articleDetailXId, renderer, articleUrl, dump, articleDetail, isMainPage)
 

test/e2e/openstreetmap.e2e.test.ts

@@ -0,0 +1,57 @@
+import { testRenders } from '../testRenders.js'
+import domino from 'domino'
+import { zimdump } from '../util.js'
+import 'dotenv/config.js'
+import { jest } from '@jest/globals'
+import rimraf from 'rimraf'
+
+jest.setTimeout(60000)
+
+// Check the integrity of img elements between zim file and article html taken from it
+const verifyImgElements = (imgFilesArr, imgElements) => {
+  for (const img of imgElements) {
+    for (const imgFile of imgFilesArr) {
+      if (img.getAttribute('src').includes(imgFile)) {
+        return true
+      }
+    }
+  }
+  return false
+}
+
+const parameters = {
+  mwUrl: 'https://wiki.openstreetmap.org',
+  articleList: 'London',
+  adminEmail: '[email protected]',
+}
+
+await testRenders(
+  parameters,
+  async (outFiles) => {
+    const articleFromDump = await zimdump(`show --url A/${parameters.articleList} ${outFiles[0].outFile}`)
+    describe('e2e test for wiki.openstreetmap.org', () => {
+      const articleDoc = domino.createDocument(articleFromDump)
+      // TODO: blocked by issues/1931
+      /*
+      test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => {
+        await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
+      })
+      */
+      test(`test article header for ${outFiles[0]?.renderer} renderer`, async () => {
+        expect(articleDoc.querySelector('h1.article-header, h1.pcs-edit-section-title')).toBeTruthy()
+      })
+      test(`test article image integrity for ${outFiles[0]?.renderer} renderer`, async () => {
+        const mediaFiles = await zimdump(`list --ns I ${outFiles[0].outFile}`)
+        const mediaFilesArr = mediaFiles.split('\n')
+        const imgFilesArr = mediaFilesArr.filter((elem) => elem.endsWith('pdf') || elem.endsWith('png') || elem.endsWith('jpg'))
+        const imgElements = Array.from(articleDoc.querySelectorAll('img'))
+        expect(verifyImgElements(imgFilesArr, imgElements)).toBe(true)
+      })
+
+      afterAll(() => {
+        rimraf.sync(`./${outFiles[0].testId}`)
+      })
+    })
+  },
+  ['MediawikiRestApi'],
+)

test/unit/downloader.test.ts

@@ -17,6 +17,7 @@ import domino from 'domino'
 import { WikimediaDesktopRenderer } from '../../src/renderers/wikimedia-desktop.renderer.js'
 import { VisualEditorRenderer } from '../../src/renderers/visual-editor.renderer.js'
 import { WikimediaMobileRenderer } from '../../src/renderers/wikimedia-mobile.renderer.js'
+import { MediawikiRestApiRenderer } from '../../src/renderers/mediawiki-rest-api.renderer.js'
 import { RENDERERS_LIST } from '../../src/util/const.js'
 
 jest.setTimeout(200000)
@@ -38,6 +39,7 @@ describe('Downloader class', () => {
     await MediaWiki.hasWikimediaDesktopApi()
     await MediaWiki.hasWikimediaMobileApi()
     await MediaWiki.hasVisualEditorApi()
+    await MediaWiki.hasMediawikiRestApi()
     await downloader.setBaseUrls()
   })
 
@@ -225,6 +227,9 @@ describe('Downloader class', () => {
         case 'WikimediaMobile':
           rendererInstance = new WikimediaMobileRenderer()
           break
+        case 'MediawikiRestApi':
+          rendererInstance = new MediawikiRestApiRenderer()
+          break
         default:
           throw new Error(`Unknown renderer: ${renderer}`)
       }