Implement MediaWiki REST API render

src/Downloader.ts

@@ -20,10 +20,16 @@ import { normalizeMwResponse, DB_ERROR, WEAK_ETAG_REGEX, stripHttpFromUrl, isBit
 import S3 from './S3.js'
 import * as logger from './Logger.js'
 import MediaWiki, { QueryOpts } from './MediaWiki.js'
+import { Dump } from './Dump.js'
 import ApiURLDirector from './util/builders/url/api.director.js'
 import basicURLDirector from './util/builders/url/basic.director.js'
 import urlHelper from './util/url.helper.js'
 
+import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
+import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
+import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
+import MediawikiRESTApiURL from './util/builders/url/mediawiki-rest-api.director.js'
+
 const imageminOptions = new Map()
 imageminOptions.set('default', new Map())
 imageminOptions.set('webp', new Map())
@@ -61,6 +67,8 @@ interface BackoffOptions {
   backoffHandler: (number: number, delay: number, error?: any) => void
 }
 
+type Director = WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector | MediawikiRESTApiURL
+
 export const defaultStreamRequestOptions: AxiosRequestConfig = {
   headers: {
     accept: 'application/octet-stream',
@@ -89,6 +97,8 @@ class Downloader {
   public streamRequestOptions: AxiosRequestConfig
   public wikimediaMobileJsDependenciesList: string[] = []
   public wikimediaMobileStyleDependenciesList: string[] = []
+  public articleUrlDirector: Director
+  public mainPageUrlDirector: Director
 
   private readonly uaString: string
   private activeRequests = 0
@@ -169,52 +179,79 @@ class Downloader {
     }
   }
 
-  public async setBaseUrls(forceRender = null) {
+  private getUrlDirector(capabilitiesList): Director {
+    for (const capabilityInfo of capabilitiesList) {
+      if (capabilityInfo.condition) {
+        return new capabilityInfo.Director(capabilityInfo.value)
+      }
+    }
+    throw new Error('No suitable URL director found.')
+  }
+
+  public async setBaseUrlsDirectors(forceRender = null) {
     if (!forceRender) {
       //* Objects order in array matters!
-      this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([
-        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
-        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
-        { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
-      ])
+      const articlesCapabilitiesList = [
+        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href, Director: WikimediaMobileURLDirector },
+        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href, Director: WikimediaDesktopURLDirector },
+        { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.VisualEditorApiUrl.href, Director: VisualEditorURLDirector },
+        { condition: await MediaWiki.hasMediawikiRESTApi(), value: MediaWiki.baseUrl.href, Director: MediawikiRESTApiURL },
+      ]
+
+      this.baseUrl = basicURLDirector.buildDownloaderBaseUrl(articlesCapabilitiesList)
+      this.articleUrlDirector = this.getUrlDirector(articlesCapabilitiesList)
 
       //* Objects order in array matters!
-      this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([
-        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
-        { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
-        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
-      ])
+      const mainPageCapabilitiesList = [
+        { condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href, Director: WikimediaDesktopURLDirector },
+        { condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.VisualEditorApiUrl.href, Director: VisualEditorURLDirector },
+        { condition: await MediaWiki.hasMediawikiRESTApi(), value: MediaWiki.baseUrl.href, Director: MediawikiRESTApiURL },
+        { condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href, Director: WikimediaMobileURLDirector },
+      ]
+      this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl(mainPageCapabilitiesList)
+      this.mainPageUrlDirector = this.getUrlDirector(mainPageCapabilitiesList)
     } else {
       switch (forceRender) {
         case 'WikimediaDesktop':
           if (MediaWiki.hasWikimediaDesktopApi()) {
             this.baseUrl = MediaWiki.WikimediaDesktopApiUrl.href
             this.baseUrlForMainPage = MediaWiki.WikimediaDesktopApiUrl.href
+            this.articleUrlDirector = this.mainPageUrlDirector = new WikimediaDesktopURLDirector(MediaWiki.WikimediaDesktopApiUrl.href)
             break
           }
           break
         case 'VisualEditor':
           if (MediaWiki.hasVisualEditorApi()) {
-            this.baseUrl = MediaWiki.visualEditorApiUrl.href
-            this.baseUrlForMainPage = MediaWiki.visualEditorApiUrl.href
+            this.baseUrl = MediaWiki.VisualEditorApiUrl.href
+            this.baseUrlForMainPage = MediaWiki.VisualEditorApiUrl.href
+            this.articleUrlDirector = this.mainPageUrlDirector = new VisualEditorURLDirector(MediaWiki.VisualEditorApiUrl.href)
             break
           }
           break
         case 'WikimediaMobile':
           if (MediaWiki.hasWikimediaMobileApi()) {
             this.baseUrl = MediaWiki.WikimediaMobileApiUrl.href
             this.baseUrlForMainPage = MediaWiki.WikimediaMobileApiUrl.href
+            this.articleUrlDirector = this.mainPageUrlDirector = new WikimediaMobileURLDirector(MediaWiki.WikimediaMobileApiUrl.href)
+            break
+          }
+          break
+        case 'MediawikiRESTApi':
+          if (MediaWiki.hasMediawikiRESTApi()) {
+            this.baseUrl = MediaWiki.baseUrl.href
+            this.baseUrlForMainPage = MediaWiki.baseUrl.href
+            this.articleUrlDirector = this.mainPageUrlDirector = new MediawikiRESTApiURL(MediaWiki.baseUrl.href)
             break
           }
           break
         default:
           throw new Error('Unable to find specific API end-point to retrieve article HTML')
       }
     }
-    logger.log('Base Url: ', this.baseUrl)
-    logger.log('Base Url for Main Page: ', this.baseUrlForMainPage)
+  }
 
-    if (!this.baseUrl || !this.baseUrlForMainPage) throw new Error('Unable to find appropriate API end-point to retrieve article HTML')
+  public getArticleUrl(dump: Dump, articleId: string): string {
+    return `${dump.isMainPage(articleId) ? this.mainPageUrlDirector.buildArticleURL(articleId) : this.articleUrlDirector.buildArticleURL(articleId)}`
   }
 
   public removeEtagWeakPrefix(etag: string): string {
@@ -333,7 +370,7 @@ class Downloader {
     articleDetailXId: RKVS<ArticleDetail>,
     articleRenderer,
     articleUrl,
-    dump,
+    dump: Dump,
     articleDetail?: ArticleDetail,
     isMainPage?: boolean,
   ): Promise<any> {
@@ -382,7 +419,7 @@ class Downloader {
     await this.claimRequest()
 
     try {
-      return await new Promise((resolve, reject) => {
+      return new Promise((resolve, reject) => {
         const cb = (err: any, val: any) => {
           if (err) {
             reject(err)
@@ -721,7 +758,7 @@ class Downloader {
 
   // Solution to handle aws js sdk v3 from https://github.com/aws/aws-sdk-js-v3/issues/1877
   private async streamToBuffer(stream: Readable): Promise<Buffer> {
-    return await new Promise((resolve, reject) => {
+    return new Promise((resolve, reject) => {
       const chunks: Uint8Array[] = []
       stream.on('data', (chunk) => chunks.push(chunk))
       stream.on('error', reject)

src/MediaWiki.ts

@@ -12,6 +12,7 @@ import ApiURLDirector from './util/builders/url/api.director.js'
 import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
 import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
 import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
+import MediawikiRESTApiDirector from './util/builders/url/mediawiki-rest-api.director.js'
 import { checkApiAvailability } from './util/mw-api.js'
 import { BLACKLISTED_NS } from './util/const.js'
 
@@ -49,23 +50,26 @@ class MediaWiki {
   #password: string
   #apiActionPath: string
   #domain: string
-  private apiUrlDirector: ApiURLDirector
-  private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
-  private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
-  private VisualEditorURLDirector: VisualEditorURLDirector
 
-  public visualEditorApiUrl: URL
   public apiUrl: URL
   public modulePath: string // only for reading
   public _modulePathOpt: string // only for whiting to generate modulePath
   public mobileModulePath: string
   public webUrl: URL
   public WikimediaDesktopApiUrl: URL
   public WikimediaMobileApiUrl: URL
-
+  // public MediawikiRESTApiURL: URL
+  public VisualEditorApiUrl: URL
+
+  #apiUrlDirector: ApiURLDirector
+  #wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
+  #wikimediaMobileUrlDirector: WikimediaMobileURLDirector
+  #visualEditorURLDirector: VisualEditorURLDirector
+  #mediawikiRESTApiDirector: MediawikiRESTApiDirector
   #hasWikimediaDesktopApi: boolean | null
   #hasWikimediaMobileApi: boolean | null
   #hasVisualEditorApi: boolean | null
+  #hasMediawikiRESTApi: boolean | null
   #hasCoordinates: boolean | null
 
   set username(value: string) {
@@ -111,6 +115,7 @@ class MediaWiki {
     this.namespacesToMirror = []
 
     this.#apiActionPath = 'w/api.php'
+    this.#apiPath = 'w/api.php'
     this.#wikiPath = 'wiki/'
     this.apiCheckArticleId = 'MediaWiki:Sidebar'
 
@@ -127,6 +132,7 @@ class MediaWiki {
     this.#hasWikimediaDesktopApi = null
     this.#hasWikimediaMobileApi = null
     this.#hasVisualEditorApi = null
+    this.#hasMediawikiRESTApi = null
     this.#hasCoordinates = null
   }
 
@@ -136,28 +142,40 @@ class MediaWiki {
 
   public async hasWikimediaDesktopApi(): Promise<boolean> {
     if (this.#hasWikimediaDesktopApi === null) {
-      this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
+      this.#hasWikimediaDesktopApi = await checkApiAvailability(this.#wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasWikimediaDesktopApi
     }
     return this.#hasWikimediaDesktopApi
   }
 
   public async hasWikimediaMobileApi(): Promise<boolean> {
     if (this.#hasWikimediaMobileApi === null) {
-      this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
+      this.#hasWikimediaMobileApi = await checkApiAvailability(this.#wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasWikimediaMobileApi
     }
     return this.#hasWikimediaMobileApi
   }
 
   public async hasVisualEditorApi(): Promise<boolean> {
     if (this.#hasVisualEditorApi === null) {
-      this.#hasVisualEditorApi = await checkApiAvailability(this.VisualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#visualEditorURLDirector = new VisualEditorURLDirector(this.VisualEditorApiUrl.href)
+      this.#hasVisualEditorApi = await checkApiAvailability(this.#visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
       return this.#hasVisualEditorApi
     }
     return this.#hasVisualEditorApi
   }
 
+  public async hasMediawikiRESTApi(): Promise<boolean> {
+    if (this.#hasMediawikiRESTApi === null) {
+      this.#mediawikiRESTApiDirector = new MediawikiRESTApiDirector(this.baseUrl.href)
+      this.#hasMediawikiRESTApi = await checkApiAvailability(this.#mediawikiRESTApiDirector.buildArticleURL(this.apiCheckArticleId))
+      return this.#hasMediawikiRESTApi
+    }
+    return this.#hasMediawikiRESTApi
+  }
+
   public async hasCoordinates(downloader: Downloader): Promise<boolean> {
     if (this.#hasCoordinates === null) {
       const validNamespaceIds = this.namespacesToMirror.map((ns) => this.namespaces[ns].num)
@@ -166,7 +184,7 @@ class MediaWiki {
         rdnamespace: validNamespaceIds,
       }
 
-      const resp = await downloader.getJSON<MwApiResponse>(this.apiUrlDirector.buildQueryURL(reqOpts))
+      const resp = await downloader.getJSON<MwApiResponse>(this.#apiUrlDirector.buildQueryURL(reqOpts))
       const isCoordinateWarning = JSON.stringify(resp?.warnings?.query ?? '').includes('coordinates')
       if (isCoordinateWarning) {
         logger.info('Coordinates not available on this wiki')
@@ -181,15 +199,12 @@ class MediaWiki {
     const baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
     this.webUrl = baseUrlDirector.buildURL(this.#wikiPath)
     this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
-    this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
-    this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
-    this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
-    this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
+    this.#apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
+    this.VisualEditorApiUrl = this.#apiUrlDirector.buildVisualEditorURL()
+    this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl()
+    this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl()
     this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
     this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
-    this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
-    this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
-    this.VisualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
   }
 
   public async login(downloader: Downloader) {
@@ -233,7 +248,7 @@ class MediaWiki {
   }
 
   public async getNamespaces(addNamespaces: number[], downloader: Downloader) {
-    const url = this.apiUrlDirector.buildNamespacesURL()
+    const url = this.#apiUrlDirector.buildNamespacesURL()
 
     const json: any = await downloader.getJSON(url)
     ;['namespaces', 'namespacealiases'].forEach((type) => {
@@ -414,6 +429,7 @@ class MediaWiki {
     const mwMetaData: MWMetaData = {
       webUrl: this.webUrl.href,
       apiUrl: this.apiUrl.href,
+      apiPath: this.#apiPath,
       modulePath: this.modulePath,
       mobileModulePath: this.mobileModulePath,
       webUrlPath: this.webUrl.pathname,

src/mwoffliner.lib.ts

@@ -214,7 +214,8 @@ async function execute(argv: any) {
   await MediaWiki.hasWikimediaDesktopApi()
   const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
   await MediaWiki.hasVisualEditorApi()
-  await downloader.setBaseUrls(forceRender)
+  await MediaWiki.hasMediawikiRESTApi()
+  await downloader.setBaseUrlsDirectors(forceRender)
 
   RedisStore.setOptions(argv.redis || config.defaults.redisPath)
   await RedisStore.connect()

src/renderers/abstract.renderer.ts

@@ -22,7 +22,7 @@ import {
 } from '../util/misc.js'
 
 type renderType = 'auto' | 'desktop' | 'mobile' | 'specific'
-type renderName = 'VisualEditor' | 'WikimediaDesktop' | 'WikimediaMobile'
+type renderName = 'VisualEditor' | 'WikimediaDesktop' | 'WikimediaMobile' | 'MediawikiRESTApi'
 
 interface RendererBuilderOptionsBase {
   renderType: renderType

src/renderers/mediawiki-rest-api.renderer.ts

@@ -0,0 +1,7 @@
+import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js'
+
+export class MediawikiRESTApiRenderer extends WikimediaDesktopRenderer {
+  constructor() {
+    super()
+  }
+}

src/renderers/renderer.builder.ts

@@ -3,17 +3,19 @@ import { Renderer } from './abstract.renderer.js'
 import { VisualEditorRenderer } from './visual-editor.renderer.js'
 import { WikimediaDesktopRenderer } from './wikimedia-desktop.renderer.js'
 import { WikimediaMobileRenderer } from './wikimedia-mobile.renderer.js'
+import { MediawikiRESTApiRenderer } from './mediawiki-rest-api.renderer.js'
 import { RendererBuilderOptions } from './abstract.renderer.js'
 import * as logger from './../Logger.js'
 
 export class RendererBuilder {
   public async createRenderer(options: RendererBuilderOptions): Promise<Renderer> {
     const { renderType, renderName } = options
 
-    const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi] = await Promise.all([
+    const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasMediawikiRESTApi] = await Promise.all([
       MediaWiki.hasVisualEditorApi(),
       MediaWiki.hasWikimediaDesktopApi(),
       MediaWiki.hasWikimediaMobileApi(),
+      MediaWiki.hasMediawikiRESTApi(),
     ])
 
     switch (renderType) {
@@ -23,6 +25,8 @@ export class RendererBuilder {
           return new WikimediaDesktopRenderer()
         } else if (hasVisualEditorApi) {
           return new VisualEditorRenderer()
+        } else if (hasMediawikiRESTApi) {
+          return new MediawikiRESTApiRenderer()
         } else {
           logger.error('No available desktop renderer.')
           process.exit(1)
@@ -41,6 +45,8 @@ export class RendererBuilder {
           return new VisualEditorRenderer()
         } else if (hasWikimediaMobileApi) {
           return new WikimediaMobileRenderer()
+        } else if (hasMediawikiRESTApi) {
+          return new MediawikiRESTApiRenderer()
         } else {
           logger.error('No render available at all.')
           process.exit(1)
@@ -60,6 +66,12 @@ export class RendererBuilder {
             }
             logger.error('Cannot create an instance of VisualEditor renderer.')
             process.exit(1)
+          case 'MediawikiRESTApi':
+            if (hasMediawikiRESTApi) {
+              return new MediawikiRESTApiRenderer()
+            }
+            logger.error('Cannot create an instance of MediawikiRESTApi renderer.')
+            process.exit(1)
           case 'WikimediaMobile':
             if (hasWikimediaMobileApi) {
               return new WikimediaMobileRenderer()

src/types.d.ts

@@ -160,6 +160,7 @@ interface MWMetaData {
   baseUrl: string
   wikiPath: string
   apiActionPath: string
+  apiPath: string
   domain: string
   webUrl: string
   apiUrl: string