added test to check the right scraped multimedia files

src/util/rewriteUrls.ts

@@ -97,9 +97,11 @@ function rewriteUrlNoArticleCheck(articleId: string, mw: MediaWiki, dump: Dump,
       lat = parts[4]
       lon = parts[5]
     } else if (rel === 'mw:MediaLink') {
-      const shouldScrape = (href.includes('.pdf') && !dump.nopdf) || ((href.includes('.ogg') || href.includes('.oga')) && !dump.nopic && !dump.novid && !dump.nodet)
+      const scrapePdf = href.includes('.pdf') && !dump.nopdf
+      const scrapeAudioVideo = (href.includes('.ogg') || href.includes('.oga')) && !dump.nopic && !dump.novid && !dump.nodet
+      const scrapeImage = href.includes('.svg') && !dump.nopic
 
-      if (shouldScrape) {
+      if (scrapePdf || scrapeAudioVideo || scrapeImage) {
         try {
           const newHref = getRelativeFilePath(articleId, getMediaBase(href, true), 'I')
           linkNode.setAttribute('href', newHref)
@@ -110,7 +112,7 @@ function rewriteUrlNoArticleCheck(articleId: string, mw: MediaWiki, dump: Dump,
           logger.warn('Error parsing url:', err)
           DU.deleteNode(linkNode)
         }
-      } else if (href.includes('.ogg') || href.includes('.oga')) {
+      } else if (href.includes('.ogg') || href.includes('.oga') || href.includes('.svg')) {
         linkNode.outerHTML = linkNode.innerHTML
       }
       return null

test/e2e/multimediaContent.test.ts

@@ -0,0 +1,61 @@
+import * as mwoffliner from '../../src/mwoffliner.lib.js'
+import { execa } from 'execa'
+import rimraf from 'rimraf'
+import { zimcheckAvailable, zimcheck, zimdumpAvailable, zimdump } from '../util.js'
+import 'dotenv/config'
+import { jest } from '@jest/globals'
+
+jest.setTimeout(20000)
+
+describe('Multimedia', () => {
+  const now = new Date()
+  const testId = `mwo-test-${+now}`
+
+  const parameters = {
+    mwUrl: 'https://en.m.wikipedia.org',
+    adminEmail: '[email protected]',
+    articleList: 'User:Kelson/MWoffliner_CI_reference',
+    outputDirectory: testId,
+    redis: process.env.REDIS,
+    customZimDescription: 'Example of the description',
+  }
+
+  test('check multimedia content from wikipedia test page', async () => {
+    await execa('redis-cli flushall', { shell: true })
+
+    const [dump] = await mwoffliner.execute(parameters)
+
+    expect(dump.status.articles.success).toEqual(1)
+    expect(dump.status.articles.fail).toEqual(0)
+
+    if (await zimcheckAvailable()) {
+      await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    if (await zimdumpAvailable()) {
+      const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
+
+      expect(mediaFiles.split('\n').sort()).toEqual(
+        [
+          'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+          'I/Kiwix_icon.svg',
+          'I/Kiwix_icon.svg.png',
+          'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+          'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+          'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+        ].sort(),
+      )
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    rimraf.sync(`./${testId}`)
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+})