From f39f36b518eeb374762e505a2bfff55d2673ef15 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Tue, 23 Apr 2024 11:03:47 +0500 Subject: [PATCH 1/9] fix: lint errors --- lib/track.js | 2 +- lib/xml.js | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/track.js b/lib/track.js index 70a2579..153dc56 100644 --- a/lib/track.js +++ b/lib/track.js @@ -1,6 +1,6 @@ 'use strict'; -const { getWidth, getBestTrack } = require('./util'); +const { getBestTrack } = require('./util'); const parseMimes = (codecs) => codecs diff --git a/lib/xml.js b/lib/xml.js index 6400198..9db2f41 100644 --- a/lib/xml.js +++ b/lib/xml.js @@ -28,7 +28,7 @@ function parse(text, options = {}) { var closeTag = text.substring(closeStart, pos); if (closeTag.indexOf(tagName) == -1) { - var parsedText = text.substring(0, pos).split('\n'); + const parsedText = text.substring(0, pos).split('\n'); throw new Error( 'Unexpected close tag\nLine: ' + (parsedText.length - 1) + @@ -103,7 +103,7 @@ function parse(text, options = {}) { node.children = []; } } else { - var parsedText = parseText(); + const parsedText = parseText(); if (keepWhitespace) { if (parsedText.length > 0) { children.push(parsedText); @@ -191,12 +191,12 @@ function parse(text, options = {}) { // optional parsing of children if (text.charCodeAt(pos - 1) !== slashCC) { if (tagName == 'script') { - var start = pos + 1; + const start = pos + 1; pos = text.indexOf('', pos); children = [text.slice(start, pos)]; pos += 9; } else if (tagName == 'style') { - var start = pos + 1; + const start = pos + 1; pos = text.indexOf('', pos); children = [text.slice(start, pos)]; pos += 8; @@ -241,10 +241,10 @@ function parse(text, options = {}) { } } - var out = null; + let out = null; if (options.attrValue !== undefined) { options.attrName = options.attrName || 'id'; - var out = []; + out = []; while ((pos = findElements()) !== -1) { pos = text.lastIndexOf('<', pos); @@ -264,10 +264,6 @@ function parse(text, options = {}) { out = filter(out, options.filter); } - if (options.simplify) { - return simplify(Array.isArray(out) ? out : [out]); - } - if (options.setPos) { out.pos = pos; } From e0dfa7ec4e063e1b8f4c2f495ecd2330b1135dee Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Tue, 23 Apr 2024 11:04:02 +0500 Subject: [PATCH 2/9] add dependabot --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..3d3a116 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + open-pull-requests-limit: 10 From 00365361645eab65a111497184513a2ffa5d3916 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Tue, 23 Apr 2024 11:04:30 +0500 Subject: [PATCH 3/9] add ci for tests, remove release workflow --- .github/workflows/ci.yml | 35 ++++++++++++++++++++++++++++++ .github/workflows/release.yml | 41 ----------------------------------- 2 files changed, 35 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..aba1753 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: ci + +on: + push: + paths-ignore: + - 'docs/**' + - '*.md' + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: 20.x + + - name: Install + run: | + npm install + + - name: Lint + run: | + npm run lint + + - name: Run tests + run: | + npm run test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 64e6b9f..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Release - -on: - push: - tags: - - '*.*.*' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - check-latest: true - - - name: Install dependencies - run: npm ci - - - name: Get name and version from package.json - run: | - test -n $(node -p -e "require('./package.json').name") && - test -n $(node -p -e "require('./package.json').version") && - echo PACKAGE_NAME=$(node -p -e "require('./package.json').name") >> $GITHUB_ENV && - echo PACKAGE_VERSION=$(node -p -e "require('./package.json').version") >> $GITHUB_ENV || exit 1 - - - name: Release - uses: softprops/action-gh-release@v2 - with: - draft: true - token: ${{ secrets.GITHUB_TOKEN }} - tag_name: 'v${{ env.PACKAGE_VERSION }}' - generate_release_notes: true - -permissions: - contents: write From 06f8ff78186d952dbeebecceaa6137ca4484e208 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Wed, 24 Apr 2024 22:17:07 +0500 Subject: [PATCH 4/9] update README --- README.md | 13 +++++++++++++ docs/README.ru.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 docs/README.ru.md diff --git a/README.md b/README.md index 8944fe2..160012c 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ Library for parsing MPEG-DASH and HLS manifests. Made with the purpose of obtaining a simplified representation convenient for further downloading of segments. +
+ English • + Pусский +
+ ## Install ```shell @@ -15,9 +20,17 @@ npm i dasha ## Quick start ```js +import fs from 'node:fs/promises'; import { parse } from 'dasha'; const url = 'https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd'; const body = await fetch(url).then((res) => res.text()); const manifest = await parse(body, url); + +for (const track of manifest.tracks.all) { + for (const segment of track.segments) { + const content = await fetch(url).then((res) => res.arrayBuffer()); + await fs.appendFile(`${track.id}.mp4`, content); + } +} ``` diff --git a/docs/README.ru.md b/docs/README.ru.md new file mode 100644 index 0000000..9c9651a --- /dev/null +++ b/docs/README.ru.md @@ -0,0 +1,36 @@ +# dasha + +[![npm version](https://img.shields.io/npm/v/dasha?style=flat&color=white)](https://www.npmjs.com/package/dasha) +[![npm downloads/month](https://img.shields.io/npm/dm/dasha?style=flat&color=white)](https://www.npmjs.com/package/dasha) +[![npm downloads](https://img.shields.io/npm/dt/dasha?style=flat&color=white)](https://www.npmjs.com/package/dasha) + +Библиотека для парсинга MPEG-DASH и HLS манифестов. Создана с целью получения упрощенного представления, удобного для дальнейшей загрузки сегментов. + +
+ English • + Русский +
+ +## Установка + +```shell +npm i dasha +``` + +## Быстрый старт + +```js +import fs from 'node:fs/promises'; +import { parse } from 'dasha'; + +const url = 'https://dash.akamaized.net/dash264/TestCases/1a/sony/SNE_DASH_SD_CASE1A_REVISED.mpd'; +const body = await fetch(url).then((res) => res.text()); +const manifest = await parse(body, url); + +for (const track of manifest.tracks.all) { + for (const segment of track.segments) { + const content = await fetch(url).then((res) => res.arrayBuffer()); + await fs.appendFile(`${track.id}.mp4`, content); + } +} +``` From 41525f7ea0f4b03ddf2f461f63a559574f053973 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 27 Apr 2024 22:05:02 +0500 Subject: [PATCH 5/9] fix: set default segments count if no duration --- lib/dash.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/dash.js b/lib/dash.js index 65b6e63..fb37833 100644 --- a/lib/dash.js +++ b/lib/dash.js @@ -173,10 +173,14 @@ const parseSegmentsFromTemplate = ( const startNumber = Number(segmentTemplate.get('startNumber') || 1); const segmentTimeline = segmentTemplate.get('SegmentTimeline'); resolveSegmentTemplateUrls(segmentTemplate, baseUrl, manifestUrl); - if (!duration) throw new Error('Duration of the Period was unable to be determined.'); const segmentDuration = parseFloat(segmentTemplate.get('duration')); const segmentTimescale = parseFloat(segmentTemplate.get('timescale') || 1); - const segmentsCount = Math.ceil(duration / (segmentDuration / segmentTimescale)); + // TODO: Support live manifests with type=dynamic + const DEFAULT_SEGMENTS_COUNT = 35; + // if (!duration) throw new Error('Duration of the Period was unable to be determined.'); + const segmentsCount = duration + ? Math.ceil(duration / (segmentDuration / segmentTimescale)) + : DEFAULT_SEGMENTS_COUNT; const bandwidth = representation.get('bandwidth'); const id = representation.get('id'); const segments = []; From 45653c016cdafd36428d4967d635a5f01132cd7a Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Wed, 1 May 2024 11:12:59 +0500 Subject: [PATCH 6/9] fix: sort filtered video by bitrate --- lib/track.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/track.js b/lib/track.js index 153dc56..7584dcd 100644 --- a/lib/track.js +++ b/lib/track.js @@ -21,6 +21,7 @@ const createVideoQualityFilter = (videos) => { if (!quality) return [getBestTrack(videos)]; const trackQuality = String(quality).includes('p') ? quality : `${quality}p`; const results = videos.filter((track) => track.quality === trackQuality); + results.sort((a, b) => b.bitrate.bps - a.bitrate.bps); return results.length ? results : [getBestTrack(videos)]; }; }; From cf84cb8fb0f554415a756db90203eac83de21dfc Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Wed, 1 May 2024 11:47:15 +0500 Subject: [PATCH 7/9] feat: add separate filter methods for tracks --- dasha.js | 3 ++- lib/dash.js | 4 ++++ lib/track.js | 22 ++++++++++++++++++++++ types/dasha.d.ts | 7 +++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/dasha.js b/dasha.js index 1c3eb09..6130b64 100644 --- a/dasha.js +++ b/dasha.js @@ -2,6 +2,7 @@ const dash = require('./lib/dash'); const hls = require('./lib/hls'); +const { filterByResolution, filterByQuality, filterByCodecs } = require('./lib/track'); const parse = (text, url, fallbackLanguage) => { if (text.includes(' { else throw new Error('Invalid manifest'); }; -module.exports = { parse }; +module.exports = { parse, filterByResolution, filterByQuality, filterByCodecs }; diff --git a/lib/dash.js b/lib/dash.js index fb37833..d7e31a4 100644 --- a/lib/dash.js +++ b/lib/dash.js @@ -21,6 +21,8 @@ const { createVideoQualityFilter, createAudioLanguageFilter, createSubtitleLanguageFilter, + createVideoCodecFilter, + createAudioCodecFilter, } = require('./track'); const appendUtils = (element) => { @@ -410,7 +412,9 @@ const parseManifest = async (text, url, fallbackLanguage) => { audios, subtitles, withResolution: createResolutionFilter(videos), + withVideoCodecs: createVideoCodecFilter(videos), withVideoQuality: createVideoQualityFilter(videos), + withAudioCodecs: createAudioCodecFilter(audios), withAudioLanguages: createAudioLanguageFilter(audios), withSubtitleLanguages: createSubtitleLanguageFilter(subtitles), }, diff --git a/lib/track.js b/lib/track.js index 7584dcd..bc4d028 100644 --- a/lib/track.js +++ b/lib/track.js @@ -16,6 +16,17 @@ const createResolutionFilter = (videos) => { }; }; +const createCodecFilter = (tracks) => { + return (codecs) => { + if (!codecs?.length) return tracks; + const results = tracks.filter((track) => codecs.includes(track.codec)); + results.sort((a, b) => b.bitrate.bps - a.bitrate.bps); + return results; + }; +}; + +const createVideoCodecFilter = (videos) => createCodecFilter(videos); + const createVideoQualityFilter = (videos) => { return (quality) => { if (!quality) return [getBestTrack(videos)]; @@ -26,6 +37,8 @@ const createVideoQualityFilter = (videos) => { }; }; +const createAudioCodecFilter = (audios) => createCodecFilter(audios); + const createAudioLanguageFilter = (audios) => { return (languages = [], maxTracksPerLanguage) => { if (!languages.length) { @@ -62,10 +75,19 @@ const createSubtitleLanguageFilter = (subtitles) => { }; }; +const filterByResolution = (tracks, resolution) => createResolutionFilter(tracks)(resolution); +const filterByQuality = (tracks, quality) => createVideoQualityFilter(tracks)(quality); +const filterByCodecs = (tracks, codecs) => createCodecFilter(tracks)(codecs); + module.exports = { parseMimes, createResolutionFilter, + createVideoCodecFilter, createVideoQualityFilter, + createAudioCodecFilter, createAudioLanguageFilter, createSubtitleLanguageFilter, + filterByResolution, + filterByQuality, + filterByCodecs, }; diff --git a/types/dasha.d.ts b/types/dasha.d.ts index f84d6c5..7b71784 100644 --- a/types/dasha.d.ts +++ b/types/dasha.d.ts @@ -8,12 +8,19 @@ export interface Manifest { audios: AudioTrack[]; subtitles: SubtitleTrack[]; withResolution(resolution: { width?: string; height?: string }): VideoTrack[]; + withVideoCodecs(codecs: VideoCodec[]): VideoTrack[]; withVideoQuality(quality: number | string): VideoTrack[]; + withAudioCodecs(codecs: AudioCodec[]): AudioTrack[]; withAudioLanguages(languages: string[], maxTracksPerLanguage?: number): AudioTrack[]; withSubtitleLanguages(languages: string[]): SubtitleTrack[]; }; } +export function filterByResolution(resolution: { width?: string; height?: string }): VideoTrack[]; +export function filterByCodecs(tracks: VideoTrack[], codecs: VideoCodec[]): VideoTrack[]; +export function filterByCodecs(tracks: AudioTrack[], codecs: AudioCodec[]): AudioTrack[]; +export function filterByQuality(tracks: VideoTrack[], quality: number | string): VideoTrack[]; + export interface Track { id: string; type: 'video' | 'audio' | 'text'; From 4df468bbb8a7f440b62bd324adf6b6f9843b48bf Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Fri, 3 May 2024 21:24:44 +0500 Subject: [PATCH 8/9] fix: dont fetch init data for segment base, sort tracks by bitrate --- lib/dash.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/dash.js b/lib/dash.js index d7e31a4..c3ee1e1 100644 --- a/lib/dash.js +++ b/lib/dash.js @@ -228,12 +228,13 @@ const parseSegmentFromBase = async (segmentBase, baseUrl) => { const initialization = segmentBase.get('Initialization'); let mediaRange = ''; if (initialization) { - const range = initialization.get('range'); - const headers = range ? { Range: `bytes=${range}` } : undefined; - const response = await fetch(baseUrl, headers); - const initData = await response.arrayBuffer(); - const totalSize = response.headers.get('Content-Range').split('/')[-1]; - if (totalSize) mediaRange = `${initData.byteLength}-${totalSize}`; + // const range = initialization.get('range'); + // const headers = range ? { Range: `bytes=${range}` } : undefined; + // const response = await fetch(baseUrl, headers); + // const initData = await response.arrayBuffer(); + // console.log(response.headers); + // const totalSize = response.headers.get('Content-Range').split('/')[-1]; + // if (totalSize) mediaRange = `${initData.byteLength}-${totalSize}`; } return { url: baseUrl, range: mediaRange }; }; @@ -404,6 +405,11 @@ const parseManifest = async (text, url, fallbackLanguage) => { } } } + + videos.sort((a, b) => b.bitrate.bps - a.bitrate.bps); + audios.sort((a, b) => b.bitrate.bps - a.bitrate.bps); + subtitles.sort((a, b) => b.bitrate.bps - a.bitrate.bps); + return { duration, tracks: { From a643ea79f90d79854ac747f82c8d71c08b4d6856 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sun, 5 May 2024 15:15:20 +0500 Subject: [PATCH 9/9] fix: tests --- test/dasha.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dasha.test.js b/test/dasha.test.js index 4481a57..fd00484 100644 --- a/test/dasha.test.js +++ b/test/dasha.test.js @@ -26,6 +26,6 @@ test('DASH: parse without URL parameter', async () => { const manifest = await parse(text); strictEqual(manifest.tracks.all.length, 8); const initUrl = - 'https://a-vrv.akamaized.net/evs3/8a1b3acce53d49eea0ce2104fae30046/assets/p/c46e06c5fd496e8aec0b6776b97eca3f_,3748583.mp4,3748584.mp4,3748582.mp4,3748580.mp4,3748581.mp4,.urlset/init-f1-v1-x3.mp4?t=exp=1713871228~acl=/evs3/8a1b3acce53d49eea0ce2104fae30046/assets/p/c46e06c5fd496e8aec0b6776b97eca3f_,3748583.mp4,3748584.mp4,3748582.mp4,3748580.mp4,3748581.mp4,.urlset/*~hmac=7dc7daeb338da040c65111a88cbf947505a5897f42d1c433c3858f8d890ed29c'; + 'https://a-vrv.akamaized.net/evs3/8a1b3acce53d49eea0ce2104fae30046/assets/p/c46e06c5fd496e8aec0b6776b97eca3f_,3748583.mp4,3748584.mp4,3748582.mp4,3748580.mp4,3748581.mp4,.urlset/init-f2-v1-x3.mp4?t=exp=1713871228~acl=/evs3/8a1b3acce53d49eea0ce2104fae30046/assets/p/c46e06c5fd496e8aec0b6776b97eca3f_,3748583.mp4,3748584.mp4,3748582.mp4,3748580.mp4,3748581.mp4,.urlset/*~hmac=7dc7daeb338da040c65111a88cbf947505a5897f42d1c433c3858f8d890ed29c'; strictEqual(manifest.tracks.videos[0].segments[0].url, initUrl); });