Skip to content

Commit d3c6f11

Browse files
VadimKovalenkoSNFVadimKovalenkoSNF
committed
Adjust MediaWiki REST API render
1 parent c7a9228 commit d3c6f11

19 files changed

+48
-74
lines changed

Diff for: ‎src/Downloader.ts

+12-6
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import urlHelper from './util/url.helper.js'
2828
import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
2929
import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
3030
import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
31+
import MediawikiRESTApiURL from './util/builders/url/mediawiki-rest-api.director.js'
3132

3233
const imageminOptions = new Map()
3334
imageminOptions.set('default', new Map())
@@ -66,6 +67,8 @@ interface BackoffOptions {
6667
backoffHandler: (number: number, delay: number, error?: any) => void
6768
}
6869

70+
type Director = WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector | MediawikiRESTApiURL
71+
6972
export const defaultStreamRequestOptions: AxiosRequestConfig = {
7073
headers: {
7174
accept: 'application/octet-stream',
@@ -94,8 +97,8 @@ class Downloader {
9497
public streamRequestOptions: AxiosRequestConfig
9598
public wikimediaMobileJsDependenciesList: string[] = []
9699
public wikimediaMobileStyleDependenciesList: string[] = []
97-
public articleUrlDirector: WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector
98-
public mainPageUrlDirector: WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector
100+
public articleUrlDirector: Director
101+
public mainPageUrlDirector: Director
99102

100103
private readonly uaString: string
101104
private activeRequests = 0
@@ -176,7 +179,7 @@ class Downloader {
176179
}
177180
}
178181

179-
private getUrlDirector(capabilitiesList): WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector {
182+
private getUrlDirector(capabilitiesList): Director {
180183
for (const capabilityInfo of capabilitiesList) {
181184
if (capabilityInfo.condition) {
182185
return new capabilityInfo.Director(capabilityInfo.value)
@@ -192,6 +195,7 @@ class Downloader {
192195
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href, Director: WikimediaMobileURLDirector },
193196
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href, Director: WikimediaDesktopURLDirector },
194197
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.VisualEditorApiUrl.href, Director: VisualEditorURLDirector },
198+
{ condition: await MediaWiki.hasMediawikiRESTApi(), value: MediaWiki.baseUrl.href, Director: MediawikiRESTApiURL },
195199
]
196200

197201
this.baseUrl = basicURLDirector.buildDownloaderBaseUrl(articlesCapabilitiesList)
@@ -201,6 +205,7 @@ class Downloader {
201205
const mainPageCapabilitiesList = [
202206
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href, Director: WikimediaDesktopURLDirector },
203207
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.VisualEditorApiUrl.href, Director: VisualEditorURLDirector },
208+
{ condition: await MediaWiki.hasMediawikiRESTApi(), value: MediaWiki.baseUrl.href, Director: MediawikiRESTApiURL },
204209
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href, Director: WikimediaMobileURLDirector },
205210
]
206211
this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl(mainPageCapabilitiesList)
@@ -232,9 +237,10 @@ class Downloader {
232237
}
233238
break
234239
case 'MediawikiRESTApi':
235-
if (MediaWiki.hasMediaWikiRESTApi()) {
236-
this.baseUrl = MediaWiki.mediawikiRESTApiURL.href
237-
this.baseUrlForMainPage = MediaWiki.mediawikiRESTApiURL.href
240+
if (MediaWiki.hasMediawikiRESTApi()) {
241+
this.baseUrl = MediaWiki.baseUrl.href
242+
this.baseUrlForMainPage = MediaWiki.baseUrl.href
243+
this.articleUrlDirector = this.mainPageUrlDirector = new MediawikiRESTApiURL(MediaWiki.baseUrl.href)
238244
break
239245
}
240246
break

Diff for: ‎src/MediaWiki.ts

+16-19
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ import ApiURLDirector from './util/builders/url/api.director.js'
1212
import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
1313
import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
1414
import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
15-
import MediaWikiRESTApiDirector from './util/builders/url/mediawiki-rest-api.director.js'
15+
import MediawikiRESTApiDirector from './util/builders/url/mediawiki-rest-api.director.js'
1616
import { checkApiAvailability } from './util/mw-api.js'
17-
import { BLACKLISTED_NS, TITLE_PLACEHOLDER } from './util/const.js'
17+
import { BLACKLISTED_NS } from './util/const.js'
1818

1919
export interface QueryOpts {
2020
action: string
@@ -51,24 +51,25 @@ class MediaWiki {
5151
#apiActionPath: string
5252
#domain: string
5353

54-
public VisualEditorApiUrl: URL
5554
public apiUrl: URL
56-
public mediawikiRESTApiURL: URL
5755
public modulePath: string // only for reading
5856
public _modulePathOpt: string // only for whiting to generate modulePath
5957
public mobileModulePath: string
6058
public webUrl: URL
6159
public WikimediaDesktopApiUrl: URL
6260
public WikimediaMobileApiUrl: URL
61+
// public MediawikiRESTApiURL: URL
62+
public VisualEditorApiUrl: URL
6363

6464
#apiUrlDirector: ApiURLDirector
6565
#wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
6666
#wikimediaMobileUrlDirector: WikimediaMobileURLDirector
6767
#visualEditorURLDirector: VisualEditorURLDirector
68+
#mediawikiRESTApiDirector: MediawikiRESTApiDirector
6869
#hasWikimediaDesktopApi: boolean | null
6970
#hasWikimediaMobileApi: boolean | null
7071
#hasVisualEditorApi: boolean | null
71-
#hasMediaWikiRESTApi: boolean | null
72+
#hasMediawikiRESTApi: boolean | null
7273
#hasCoordinates: boolean | null
7374

7475
set username(value: string) {
@@ -95,10 +96,6 @@ class MediaWiki {
9596
this.#wikiPath = value
9697
}
9798

98-
set mediawikiRESTAPiPath(value: string) {
99-
this.#mediawikiRESTApiPath = value
100-
}
101-
10299
set base(value: string) {
103100
this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
104101
this.initMWApis()
@@ -135,7 +132,7 @@ class MediaWiki {
135132
this.#hasWikimediaDesktopApi = null
136133
this.#hasWikimediaMobileApi = null
137134
this.#hasVisualEditorApi = null
138-
this.#hasMediaWikiRESTApi = null
135+
this.#hasMediawikiRESTApi = null
139136
this.#hasCoordinates = null
140137
}
141138

@@ -170,13 +167,13 @@ class MediaWiki {
170167
return this.#hasVisualEditorApi
171168
}
172169

173-
public async hasMediaWikiRESTApi(): Promise<boolean> {
174-
if (this.#hasMediaWikiRESTApi === null) {
175-
this.mediaWikiRESTApiDirector = new MediaWikiRESTApiDirector(this.mediawikiRESTApiURL.href)
176-
this.#hasMediaWikiRESTApi = await checkApiAvailability(this.mediaWikiRESTApiDirector.buildArticleURL(this.apiCheckArticleId))
177-
return this.#hasMediaWikiRESTApi
170+
public async hasMediawikiRESTApi(): Promise<boolean> {
171+
if (this.#hasMediawikiRESTApi === null) {
172+
this.#mediawikiRESTApiDirector = new MediawikiRESTApiDirector(this.baseUrl.href)
173+
this.#hasMediawikiRESTApi = await checkApiAvailability(this.#mediawikiRESTApiDirector.buildArticleURL(this.apiCheckArticleId))
174+
return this.#hasMediawikiRESTApi
178175
}
179-
return this.#hasMediaWikiRESTApi
176+
return this.#hasMediawikiRESTApi
180177
}
181178

182179
public async hasCoordinates(downloader: Downloader): Promise<boolean> {
@@ -204,8 +201,8 @@ class MediaWiki {
204201
this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
205202
this.#apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
206203
this.VisualEditorApiUrl = this.#apiUrlDirector.buildVisualEditorURL()
207-
this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
208-
this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
204+
this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl()
205+
this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl()
209206
this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
210207
this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
211208
}
@@ -432,7 +429,7 @@ class MediaWiki {
432429
const mwMetaData: MWMetaData = {
433430
webUrl: this.webUrl.href,
434431
apiUrl: this.apiUrl.href,
435-
mediawikiRESTAPiPath: this.mediawikiRESTAPiPath,
432+
apiPath: this.#apiPath,
436433
modulePath: this.modulePath,
437434
mobileModulePath: this.mobileModulePath,
438435
webUrlPath: this.webUrl.pathname,

Diff for: ‎src/mwoffliner.lib.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ async function execute(argv: any) {
214214
await MediaWiki.hasWikimediaDesktopApi()
215215
const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
216216
await MediaWiki.hasVisualEditorApi()
217-
await MediaWiki.hasMediaWikiRESTApi()
217+
await MediaWiki.hasMediawikiRESTApi()
218218
await downloader.setBaseUrlsDirectors(forceRender)
219219

220220
RedisStore.setOptions(argv.redis || config.defaults.redisPath)

Diff for: ‎src/renderers/renderer.builder.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@ export class RendererBuilder {
1111
public async createRenderer(options: RendererBuilderOptions): Promise<Renderer> {
1212
const { renderType, renderName } = options
1313

14-
const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasMediaWikiRESTApi] = await Promise.all([
14+
const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasMediawikiRESTApi] = await Promise.all([
1515
MediaWiki.hasVisualEditorApi(),
1616
MediaWiki.hasWikimediaDesktopApi(),
1717
MediaWiki.hasWikimediaMobileApi(),
18-
MediaWiki.hasMediaWikiRESTApi(),
18+
MediaWiki.hasMediawikiRESTApi(),
1919
])
2020

2121
switch (renderType) {
@@ -25,7 +25,7 @@ export class RendererBuilder {
2525
return new WikimediaDesktopRenderer()
2626
} else if (hasVisualEditorApi) {
2727
return new VisualEditorRenderer()
28-
} else if (hasMediaWikiRESTApi) {
28+
} else if (hasMediawikiRESTApi) {
2929
return new MediawikiRESTApiRenderer()
3030
} else {
3131
logger.error('No available desktop renderer.')
@@ -45,7 +45,7 @@ export class RendererBuilder {
4545
return new VisualEditorRenderer()
4646
} else if (hasWikimediaMobileApi) {
4747
return new WikimediaMobileRenderer()
48-
} else if (hasMediaWikiRESTApi) {
48+
} else if (hasMediawikiRESTApi) {
4949
return new MediawikiRESTApiRenderer()
5050
} else {
5151
logger.error('No render available at all.')
@@ -67,7 +67,7 @@ export class RendererBuilder {
6767
logger.error('Cannot create an instance of VisualEditor renderer.')
6868
process.exit(1)
6969
case 'MediawikiRESTApi':
70-
if (hasMediaWikiRESTApi) {
70+
if (hasMediawikiRESTApi) {
7171
return new MediawikiRESTApiRenderer()
7272
}
7373
logger.error('Cannot create an instance of MediawikiRESTApi renderer.')

Diff for: ‎src/types.d.ts

-1
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,6 @@ interface MWMetaData {
160160
baseUrl: string
161161
wikiPath: string
162162
apiActionPath: string
163-
mediawikiRESTAPiPath: string
164163
apiPath: string
165164
domain: string
166165
webUrl: string

Diff for: ‎src/util/builders/url/api.director.ts

-8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import urlBuilder from './url.builder.js'
2-
import { TITLE_PLACEHOLDER } from '../../const.js'
32

43
/**
54
* Interface to build URLs based on MediaWiki API URL
@@ -50,14 +49,7 @@ export default class ApiURLDirector {
5049
}
5150

5251
buildVisualEditorURL() {
53-
<<<<<<< HEAD
5452
return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse', formatversion: '2' }).build(true)
55-
=======
56-
return urlBuilder
57-
.setDomain(this.baseDomain)
58-
.setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse', formatversion: '2', page: TITLE_PLACEHOLDER })
59-
.build(true)
60-
>>>>>>> Switchover to WikimediaDesktop render while using MediawikiRESTApi, add temp workaround with TITLE_PLACEHOLDER for URL builders
6153
}
6254

6355
buildArticleApiURL(articleId: string) {

Diff for: ‎src/util/builders/url/base.director.ts

+1-9
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import urlBuilder from './url.builder.js'
2-
import { TITLE_PLACEHOLDER } from '../../const.js'
32

43
/**
54
* Interface to build URLs based on base URL
@@ -25,7 +24,7 @@ export default class BaseURLDirector {
2524
buildWikimediaDesktopApiUrl(path?: string) {
2625
return urlBuilder
2726
.setDomain(this.baseDomain)
28-
.setPath(path ?? `api/rest_v1/page/html/${TITLE_PLACEHOLDER}`)
27+
.setPath(path ?? 'api/rest_v1/page/html/')
2928
.build(true, '/')
3029
}
3130

@@ -49,11 +48,4 @@ export default class BaseURLDirector {
4948
.setPath(path ?? 'api/rest_v1/page/mobile-html-offline-resources')
5049
.build(false, '/')
5150
}
52-
53-
buildMediaWikiREST(path?: string) {
54-
return urlBuilder
55-
.setDomain(this.baseDomain)
56-
.setPath(path ?? `w/rest.php/v1/page/${TITLE_PLACEHOLDER}/html`)
57-
.build(true, '/')
58-
}
5951
}

Diff for: ‎src/util/builders/url/mediawiki-rest-api.director.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import urlBuilder from './url.builder.js'
33
/**
44
* Interface to build URLs based on MediaWiki REST API URL
55
*/
6-
export default class MediaWikiRESTApiURL {
6+
export default class MediawikiRESTApiURL {
77
baseDomain: string
88

99
constructor(baseDomain: string) {
@@ -12,6 +12,6 @@ export default class MediaWikiRESTApiURL {
1212

1313
buildArticleURL(articleId: string) {
1414
const base = urlBuilder.setDomain(this.baseDomain).build()
15-
return `${base}${articleId}/html`
15+
return `${base}/w/rest.php/v1/page/${articleId}/html`
1616
}
1717
}

Diff for: ‎src/util/builders/url/url.builder.ts

-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { ensureTrailingChar } from '../../misc.js'
2-
import { TITLE_PLACEHOLDER } from '../../const.js'
32

43
/**
54
* Concat the path to the domain and setting query params
@@ -100,10 +99,6 @@ class URLBuilder {
10099

101100
return link
102101
}
103-
104-
buildArticleUrl(renderApiUrl: string, articleId: string) {
105-
return `${renderApiUrl.replace(TITLE_PLACEHOLDER, articleId)}`.slice(0, -1)
106-
}
107102
}
108103

109104
const urlBuilder = new URLBuilder()

Diff for: ‎src/util/const.ts

-1
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,3 @@ export const WEBP_HANDLER_URL = 'https://gist.githubusercontent.com/rgaudin/60bb
2121
export const MAX_FILE_DOWNLOAD_RETRIES = 5
2222
export const BLACKLISTED_NS = ['Story'] // 'Story' Wikipedia namespace is content, but not indgestable by Parsoid https://github.com/openzim/mwoffliner/issues/1853
2323
export const RENDERERS_LIST = ['WikimediaDesktop', 'VisualEditor', 'WikimediaMobile', 'MediawikiRESTApi']
24-
export const TITLE_PLACEHOLDER = 'TITLE_PLACEHOLDER'

Diff for: ‎src/util/saveArticles.ts

+1-5
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,7 @@ import { jsPath } from './index.js'
1111
import { config } from '../config.js'
1212
import { getSizeFromUrl, cleanupAxiosError } from './misc.js'
1313
import { CONCURRENCY_LIMIT, DELETED_ARTICLE_ERROR, MAX_FILE_DOWNLOAD_RETRIES } from './const.js'
14-
import DesktopURLDirector from './builders/url/desktop.director.js'
15-
import VisualEditorURLDirector from './builders/url/visual-editor.director.js'
16-
import MediaWikiRESTApiDirector from './builders/url/mediawiki-rest-api.director.js'
1714
import urlHelper from './url.helper.js'
18-
import urlBuilder from './builders/url/url.builder.js'
1915
import { Renderer } from '../renderers/abstract.renderer.js'
2016
import { RendererBuilder } from '../renderers/renderer.builder.js'
2117

@@ -238,7 +234,7 @@ export function getArticleUrl(renderer, downloader: Downloader, dump: Dump, arti
238234
articleUrl = this.visualEditorURLDirector.buildArticleURL(articleId)
239235
break
240236
case 'MediawikiRESTApiRenderer':
241-
articleUrl = this.mediaWikiRESTApiDirector.buildArticleURL(this.apiCheckArticleId)
237+
articleUrl = this.MediawikiRESTApiDirector.buildArticleURL(this.apiCheckArticleId)
242238
break
243239
}
244240
return articleUrl

Diff for: ‎test/e2e/bm.e2e.test.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import * as mwoffliner from '../../src/mwoffliner.lib.js'
22
import { execa } from 'execa'
33
import rimraf from 'rimraf'
4-
import { zimcheckAvailable, zimcheck } from '../util.js'
54
import 'dotenv/config.js'
65
import { jest } from '@jest/globals'
76
import { zimdumpAvailable, zimdump } from '../util.js'
@@ -40,11 +39,14 @@ describe('bm', () => {
4039
}
4140
}
4241

42+
// TODO: blocked by issues/1931
43+
/*
4344
if (await zimcheckAvailable()) {
4445
await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
4546
} else {
4647
console.log('Zimcheck not installed, skipping test')
4748
}
49+
*/
4850

4951
if (await zimdumpAvailable()) {
5052
const discussionArticlesStr = await zimdump(`list --ns A/Discussion ${outFiles[0].outFile}`)

Diff for: ‎test/unit/builders/url/api.director.test.ts

-4
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,7 @@ describe('ApiURLDirector', () => {
5757
it('should return base visual editor URL object with default query params', () => {
5858
const url = apiUrlDirector.buildVisualEditorURL()
5959

60-
<<<<<<< HEAD
6160
expect(url.href).toBe('https://en.wikipedia.org/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&formatversion=2')
62-
=======
63-
expect(url.href).toBe('https://en.wikipedia.org/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&formatversion=2&page=TITLE_PLACEHOLDER')
64-
>>>>>>> Switchover to WikimediaDesktop render while using MediawikiRESTApi, add temp workaround with TITLE_PLACEHOLDER for URL builders
6561
})
6662
})
6763
})

Diff for: ‎test/unit/builders/url/base.director.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ describe('BaseURLDirector', () => {
4949
it('should return a desktop URL with default path and trailing char', () => {
5050
const url = baseUrlDirector.buildWikimediaDesktopApiUrl()
5151

52-
expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/html/TITLE_PLACEHOLDER/')
52+
expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/html/')
5353
})
5454
})
5555

Diff for: ‎test/unit/downloader.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ describe('Downloader class', () => {
3434
await MediaWiki.hasWikimediaDesktopApi()
3535
await MediaWiki.hasWikimediaMobileApi()
3636
await MediaWiki.hasVisualEditorApi()
37-
await MediaWiki.hasMediaWikiRESTApi()
37+
await MediaWiki.hasMediawikiRESTApi()
3838
await downloader.setBaseUrlsDirectors()
3939
})
4040

Diff for: ‎test/unit/mwApi.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ const initMW = async (downloader: Downloader) => {
2020
await MediaWiki.hasCoordinates(downloader)
2121
await MediaWiki.hasWikimediaDesktopApi()
2222
await MediaWiki.hasVisualEditorApi()
23-
await MediaWiki.hasMediaWikiRESTApi()
23+
await MediaWiki.hasMediawikiRESTApi()
2424

2525
await MediaWiki.getNamespaces([], downloader)
2626
}

0 commit comments

Comments
 (0)