From abe3fe0b2eb6ef9c89d5cc7d9f0ce3eaca72f5b1 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 18 Feb 2025 15:05:52 +0000 Subject: [PATCH] Return to forward slash mandatory before paths to support root wikis --- src/MediaWiki.ts | 8 ++++---- src/mwoffliner.lib.ts | 4 ++++ src/sanitize-argument.ts | 6 +++--- src/util/builders/url/base.director.ts | 8 ++++---- src/util/builders/url/basic.director.ts | 2 +- src/util/const.ts | 2 +- test/e2e/apiPathParamsSanitizing.e2e.test.ts | 10 +++++----- test/e2e/forceRender.test.ts | 2 +- test/unit/builders/url/base.director.test.ts | 6 +++--- test/unit/mwApiCapabilities.test.ts | 6 +++--- 10 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index c2066c34..3f95f7ff 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -141,10 +141,10 @@ class MediaWiki { this.#password = '' this.getCategories = false - this.#actionApiPath = 'w/api.php' - this.#restApiPath = 'w/rest.php' - this.#wikiPath = 'wiki/' - this.#modulePathOpt = 'w/load.php' + this.#actionApiPath = '/w/api.php' + this.#restApiPath = '/w/rest.php' + this.#wikiPath = '/wiki/' + this.#modulePathOpt = '/w/load.php' this.namespaces = {} this.namespacesToMirror = [] diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index cec72974..4fe8efe8 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -203,7 +203,11 @@ async function execute(argv: any) { if (customMainPage) { mainPage = customMainPage + logger.log(`webURL: ${MediaWiki.webUrl}`) + logger.log(`mainPage: ${mainPage}`) + logger.log(`encodeURIComponent: ${encodeURIComponent(mainPage)}`) const mainPageUrl = MediaWiki.webUrl + encodeURIComponent(mainPage) + logger.log(`mainPageUrl: ${mainPageUrl}`) if (!(await checkApiAvailability(downloader, mainPageUrl))) { throw new Error(`customMainPage doesn't return 200 status code for url ${mainPageUrl}`) } diff --git a/src/sanitize-argument.ts b/src/sanitize-argument.ts index 00f6721b..972347a6 100644 --- a/src/sanitize-argument.ts +++ b/src/sanitize-argument.ts @@ -135,9 +135,9 @@ export function sanitizeApiPathParam(apiPathParam: string) { return } - // No api params should start from forward slash - if (apiPathParam.startsWith('/')) { - apiPathParam = apiPathParam.slice(1) + // All API path must start with a forward slash + if (!apiPathParam?.startsWith('/')) { + apiPathParam = '/' + apiPathParam } return apiPathParam diff --git a/src/util/builders/url/base.director.ts b/src/util/builders/url/base.director.ts index 1a82e112..db096e88 100644 --- a/src/util/builders/url/base.director.ts +++ b/src/util/builders/url/base.director.ts @@ -8,7 +8,7 @@ export default class BaseURLDirector { private baseDomain: string constructor(baseDomain: string) { - this.baseDomain = baseDomain + this.baseDomain = baseDomain.endsWith('/') ? baseDomain.substring(0, baseDomain.length - 1) : baseDomain } buildURL(path: string) { @@ -32,21 +32,21 @@ export default class BaseURLDirector { buildModuleURL(path?: string) { return urlBuilder .setDomain(this.baseDomain) - .setPath(path ?? 'w/load.php') + .setPath(path ?? '/w/load.php') .build(false, '?') } buildMobileModuleURL(path?: string) { return urlBuilder .setDomain(this.baseDomain) - .setPath(path ?? 'api/rest_v1/page/mobile-html-offline-resources') + .setPath(path ?? '/api/rest_v1/page/mobile-html-offline-resources') .build(false, '/') } buildRestApiUrl(path?: string) { return urlBuilder .setDomain(this.baseDomain) - .setPath(path ?? 'w/rest.php') + .setPath(path ?? '/w/rest.php') .build(true, '/') } } diff --git a/src/util/builders/url/basic.director.ts b/src/util/builders/url/basic.director.ts index 4f6b90d5..80412146 100644 --- a/src/util/builders/url/basic.director.ts +++ b/src/util/builders/url/basic.director.ts @@ -7,7 +7,7 @@ type DownloaderBaseUrlConditions = Array<{ condition: boolean; value: string }> */ class BasicURLDirector { buildMediawikiBaseURL(domain: string) { - return urlBuilder.setDomain(domain).build(true, '/') + return urlBuilder.setDomain(domain).build(true, '') } buildDownloaderBaseUrl(conditions: DownloaderBaseUrlConditions): string | undefined { diff --git a/src/util/const.ts b/src/util/const.ts index d232e2db..6ff83bd4 100644 --- a/src/util/const.ts +++ b/src/util/const.ts @@ -20,7 +20,7 @@ export const WEBP_HANDLER_URL = 'https://gist.githubusercontent.com/rgaudin/60bb export const MAX_FILE_DOWNLOAD_RETRIES = 5 export const BLACKLISTED_NS = ['Story'] // 'Story' Wikipedia namespace is content, but not indgestable by Parsoid https://github.com/openzim/mwoffliner/issues/1853 export const RENDERERS_LIST = ['WikimediaDesktop', 'VisualEditor', 'WikimediaMobile', 'RestApi'] -export const WIKIMEDIA_REST_API_PATH = 'api/rest_v1/' +export const WIKIMEDIA_REST_API_PATH = '/api/rest_v1/' /* Handle redirection pages for 3rd party wikis that have 200 response code diff --git a/test/e2e/apiPathParamsSanitizing.e2e.test.ts b/test/e2e/apiPathParamsSanitizing.e2e.test.ts index e0efb0dd..2a19e4ea 100644 --- a/test/e2e/apiPathParamsSanitizing.e2e.test.ts +++ b/test/e2e/apiPathParamsSanitizing.e2e.test.ts @@ -14,25 +14,25 @@ const parameters = { mwActionApiPath: sanitizeApiPathParam('/w/api.php'), mwRestApiPath: sanitizeApiPathParam('/w/rest.php'), mwModulePath: sanitizeApiPathParam('/w/load.php'), - mwWikiPath: sanitizeWikiPath('wiki'), + mwWikiPath: sanitizeWikiPath('/wiki'), } await testAllRenders(parameters, async (outFiles) => { describe(`e2e test for api url params for en.wikipedia.org for ${outFiles[0]?.renderer} renderer`, () => { test('Mediawiki actionApiPath option sanitized', () => { - expect(outFiles[0].mwMetaData.actionApiPath).toBe('w/api.php') + expect(outFiles[0].mwMetaData.actionApiPath).toBe('/w/api.php') }) test('Mediawiki restApiPath option sanitized', () => { - expect(outFiles[0].mwMetaData.restApiPath).toBe('w/rest.php') + expect(outFiles[0].mwMetaData.restApiPath).toBe('/w/rest.php') }) test('Mediawiki wikiPath option sanitized', () => { - expect(outFiles[0].mwMetaData.wikiPath).toBe('wiki/') + expect(outFiles[0].mwMetaData.wikiPath).toBe('/wiki/') }) test('Mediawiki modulePathOpt option sanitized', () => { - expect(outFiles[0].mwMetaData.modulePathOpt).toBe('w/load.php') + expect(outFiles[0].mwMetaData.modulePathOpt).toBe('/w/load.php') }) test('Mediawiki modulePath and actionApiUrl options', () => { diff --git a/test/e2e/forceRender.test.ts b/test/e2e/forceRender.test.ts index 69234f11..385f75fb 100644 --- a/test/e2e/forceRender.test.ts +++ b/test/e2e/forceRender.test.ts @@ -17,7 +17,7 @@ describe('forceRender', () => { redis: process.env.REDIS, format: ['nopic'], articleList: 'France', - mwActionApiPath: 'w/api.php', + mwActionApiPath: '/w/api.php', } afterAll(async () => { diff --git a/test/unit/builders/url/base.director.test.ts b/test/unit/builders/url/base.director.test.ts index a9a4a637..81f5f833 100644 --- a/test/unit/builders/url/base.director.test.ts +++ b/test/unit/builders/url/base.director.test.ts @@ -5,13 +5,13 @@ describe('BaseURLDirector', () => { describe('buildURL', () => { it('should return URL object with path', () => { - const url = baseUrlDirector.buildURL('v1/test/api') + const url = baseUrlDirector.buildURL('/v1/test/api') expect(url.href).toBe('https://en.m.wikipedia.com/v1/test/api') }) it('should return URL object with mwActionApiPath param', () => { - const url = baseUrlDirector.buildURL('api.php') + const url = baseUrlDirector.buildURL('/api.php') expect(url.href).toBe('https://en.m.wikipedia.com/api.php') }) @@ -35,7 +35,7 @@ describe('BaseURLDirector', () => { describe('buildModuleURL', () => { it('should return a module URL with provided path and question mark as a trailing char', () => { - const url = baseUrlDirector.buildModuleURL('w/reload.php') + const url = baseUrlDirector.buildModuleURL('/w/reload.php') expect(url).toBe('https://en.m.wikipedia.com/w/reload.php?') }) diff --git a/test/unit/mwApiCapabilities.test.ts b/test/unit/mwApiCapabilities.test.ts index 44362075..4b99ca78 100644 --- a/test/unit/mwApiCapabilities.test.ts +++ b/test/unit/mwApiCapabilities.test.ts @@ -46,7 +46,7 @@ describe('Checking Mediawiki capabilities', () => { test('test capabilities of minecraft.wiki with correct VisualEditor receipt', async () => { MediaWiki.base = 'https://minecraft.wiki' MediaWiki.wikiPath = '/' - MediaWiki.actionApiPath = 'api.php' + MediaWiki.actionApiPath = '/api.php' const downloader = new Downloader({ uaString: `${config.userAgent} (contact@kiwix.org)`, speed: 1, reqTimeout: 1000 * 60, webp: true, optimisationCacheUrl: '' }) expect(await MediaWiki.hasWikimediaDesktopApi(downloader)).toBe(false) @@ -58,7 +58,7 @@ describe('Checking Mediawiki capabilities', () => { test('test capabilities of pokemon.fandom.com with correct VisualEditor receipt', async () => { MediaWiki.base = 'https://pokemon.fandom.com/' MediaWiki.wikiPath = '/' - MediaWiki.actionApiPath = 'api.php' + MediaWiki.actionApiPath = '/api.php' const downloader = new Downloader({ uaString: `${config.userAgent} (contact@kiwix.org)`, speed: 1, reqTimeout: 1000 * 60, webp: true, optimisationCacheUrl: '' }) expect(await MediaWiki.hasWikimediaDesktopApi(downloader)).toBe(false) @@ -80,7 +80,7 @@ describe('Checking Mediawiki capabilities', () => { test('test capabilities of pokemon.fandom.com with RestApi receipt', async () => { MediaWiki.base = 'https://pokemon.fandom.com/' MediaWiki.wikiPath = '/' - MediaWiki.restApiPath = 'rest.php' + MediaWiki.restApiPath = '/rest.php' const downloader = new Downloader({ uaString: `${config.userAgent} (contact@kiwix.org)`, speed: 1, reqTimeout: 1000 * 60, webp: true, optimisationCacheUrl: '' }) expect(await MediaWiki.hasWikimediaDesktopApi(downloader)).toBe(false)