From 96348a04ba5cb2cfe97242baf01c6b3854fc2946 Mon Sep 17 00:00:00 2001 From: nukeop <12746779+nukeop@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:50:02 +0200 Subject: [PATCH] Update dependency --- package-lock.json | 36 +- packages/core/package.json | 1 + packages/core/src/rest/Youtube.ts | 2 +- packages/core/src/ytdl-core/LICENSE | 23 - packages/core/src/ytdl-core/agent.js | 94 --- packages/core/src/ytdl-core/cache.js | 54 -- packages/core/src/ytdl-core/format-utils.js | 250 ------- packages/core/src/ytdl-core/formats.js | 524 ------------- packages/core/src/ytdl-core/index.d.ts | 774 -------------------- packages/core/src/ytdl-core/index.js | 239 ------ packages/core/src/ytdl-core/info-extras.js | 362 --------- packages/core/src/ytdl-core/info.js | 442 ----------- packages/core/src/ytdl-core/sig.js | 296 -------- packages/core/src/ytdl-core/url-utils.js | 92 --- packages/core/src/ytdl-core/utils.js | 389 ---------- 15 files changed, 37 insertions(+), 3541 deletions(-) delete mode 100644 packages/core/src/ytdl-core/LICENSE delete mode 100644 packages/core/src/ytdl-core/agent.js delete mode 100644 packages/core/src/ytdl-core/cache.js delete mode 100644 packages/core/src/ytdl-core/format-utils.js delete mode 100644 packages/core/src/ytdl-core/formats.js delete mode 100644 packages/core/src/ytdl-core/index.d.ts delete mode 100644 packages/core/src/ytdl-core/index.js delete mode 100644 packages/core/src/ytdl-core/info-extras.js delete mode 100644 packages/core/src/ytdl-core/info.js delete mode 100644 packages/core/src/ytdl-core/sig.js delete mode 100644 packages/core/src/ytdl-core/url-utils.js delete mode 100644 packages/core/src/ytdl-core/utils.js diff --git a/package-lock.json b/package-lock.json index 93ce5ea762..0ca5bf81f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2258,6 +2258,25 @@ "node": ">=10.0.0" } }, + "node_modules/@distube/ytdl-core": { + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.14.4.tgz", + "integrity": "sha512-dHb4GW3qATIjRsS6VIhm3Pop7FdUcDFhsnyQlsPeXW7UhTPuNS0BmraKiTpFbpp0Ky+rxBQjJBfPRFsM+dT1fg==", + "dependencies": { + "http-cookie-agent": "^6.0.5", + "m3u8stream": "^0.8.6", + "miniget": "^4.2.3", + "sax": "^1.4.1", + "tough-cookie": "^4.1.4", + "undici": "five" + }, + "engines": { + "node": ">=14.0" + }, + "funding": { + "url": "https://github.com/distubejs/ytdl-core?sponsor" + } + }, "node_modules/@electron/asar": { "version": "3.2.10", "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.2.10.tgz", @@ -48745,6 +48764,7 @@ "@babel/core": "^7.9.0", "@babel/preset-env": "^7.9.5", "@babel/preset-react": "^7.9.4", + "@distube/ytdl-core": "^4.14.4", "@nuclear/scanner": "^0.6.39", "@supabase/supabase-js": "^1.35.4", "ajv": "^6.12.5", @@ -50997,6 +51017,19 @@ "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", "dev": true }, + "@distube/ytdl-core": { + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.14.4.tgz", + "integrity": "sha512-dHb4GW3qATIjRsS6VIhm3Pop7FdUcDFhsnyQlsPeXW7UhTPuNS0BmraKiTpFbpp0Ky+rxBQjJBfPRFsM+dT1fg==", + "requires": { + "http-cookie-agent": "^6.0.5", + "m3u8stream": "^0.8.6", + "miniget": "^4.2.3", + "sax": "^1.4.1", + "tough-cookie": "^4.1.4", + "undici": "five" + } + }, "@electron/asar": { "version": "3.2.10", "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.2.10.tgz", @@ -53602,6 +53635,7 @@ "@babel/core": "^7.9.0", "@babel/preset-env": "^7.9.5", "@babel/preset-react": "^7.9.4", + "@distube/ytdl-core": "^4.14.4", "@nuclear/scanner": "^0.6.39", "@supabase/supabase-js": "^1.35.4", "@types/fast-levenshtein": "^0.0.2", @@ -86897,4 +86931,4 @@ "dev": true } } -} \ No newline at end of file +} diff --git a/packages/core/package.json b/packages/core/package.json index 986d7debd0..b84c381333 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -22,6 +22,7 @@ "@babel/core": "^7.9.0", "@babel/preset-env": "^7.9.5", "@babel/preset-react": "^7.9.4", + "@distube/ytdl-core": "^4.14.4", "@nuclear/scanner": "^0.6.39", "@supabase/supabase-js": "^1.35.4", "ajv": "^6.12.5", diff --git a/packages/core/src/rest/Youtube.ts b/packages/core/src/rest/Youtube.ts index eb39cb0ac9..c883c9879a 100644 --- a/packages/core/src/rest/Youtube.ts +++ b/packages/core/src/rest/Youtube.ts @@ -1,5 +1,5 @@ import logger from 'electron-timber'; -import ytdl from '../ytdl-core'; +import ytdl from '@distube/ytdl-core'; import ytpl from 'ytpl'; import {search, SearchVideo} from 'youtube-ext'; diff --git a/packages/core/src/ytdl-core/LICENSE b/packages/core/src/ytdl-core/LICENSE deleted file mode 100644 index 8c1bf86017..0000000000 --- a/packages/core/src/ytdl-core/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -Adapted from @distube/ytdl-core (https://github.com/distubejs/ytdl-core) under the following license: - -MIT License - -Copyright (C) 2012-present by fent - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/packages/core/src/ytdl-core/agent.js b/packages/core/src/ytdl-core/agent.js deleted file mode 100644 index bd3492300a..0000000000 --- a/packages/core/src/ytdl-core/agent.js +++ /dev/null @@ -1,94 +0,0 @@ -import { ProxyAgent } from 'undici'; -import { Cookie, CookieJar, canonicalDomain } from 'tough-cookie'; -import { CookieAgent, CookieClient } from 'http-cookie-agent/undici'; - -const convertSameSite = sameSite => { - switch (sameSite) { - case 'strict': - return 'strict'; - case 'lax': - return 'lax'; - case 'no_restriction': - case 'unspecified': - default: - return 'none'; - } -}; - -const convertCookie = cookie => cookie instanceof Cookie ? cookie : new Cookie({ - key: cookie.name, - value: cookie.value, - expires: typeof cookie.expirationDate === 'number' ? new Date(cookie.expirationDate * 1000) : 'Infinity', - domain: canonicalDomain(cookie.domain), - path: cookie.path, - secure: cookie.secure, - httpOnly: cookie.httpOnly, - sameSite: convertSameSite(cookie.sameSite), - hostOnly: cookie.hostOnly -}); - -export const addCookies = (jar, cookies) => { - if (!cookies || !Array.isArray(cookies)) { - throw new Error('cookies must be an array'); - } - if (!cookies.some(c => c.name === 'SOCS')) { - cookies.push({ - domain: '.youtube.com', - hostOnly: false, - httpOnly: false, - name: 'SOCS', - path: '/', - sameSite: 'lax', - secure: true, - session: false, - value: 'CAI' - }); - } - for (const cookie of cookies) { - jar.setCookieSync(convertCookie(cookie), 'https://www.youtube.com'); - } -}; - -export const addCookiesFromString = (jar, cookies) => { - if (!cookies || typeof cookies !== 'string') { - throw new Error('cookies must be a string'); - } - return addCookies(jar, cookies.split(';').map(c => Cookie.parse(c)).filter(Boolean)); -}; - -export const createAgent = (cookies = [], opts = {}) => { - const options = Object.assign({}, opts); - if (!options.cookies) { - const jar = new CookieJar(); - addCookies(jar, cookies); - options.cookies = { jar }; - } - return { - dispatcher: new CookieAgent(options), - localAddress: options.localAddress, - jar: options.cookies.jar - }; -}; - -export const createProxyAgent = (options, cookies = []) => { - if (!cookies) { - cookies = []; - } - if (typeof options === 'string') { - options = { uri: options }; - } - if (options.factory) { - throw new Error('Cannot use factory with createProxyAgent'); - } - const jar = new CookieJar(); - addCookies(jar, cookies); - const proxyOptions = Object.assign({ - factory: (origin, opts) => { - const o = Object.assign({ cookies: { jar } }, opts); - return new CookieClient(origin, o); - } - }, options); - return { dispatcher: new ProxyAgent(proxyOptions), jar, localAddress: options.localAddress }; -}; - -export const defaultAgent = createAgent(); diff --git a/packages/core/src/ytdl-core/cache.js b/packages/core/src/ytdl-core/cache.js deleted file mode 100644 index c694175388..0000000000 --- a/packages/core/src/ytdl-core/cache.js +++ /dev/null @@ -1,54 +0,0 @@ -import { setTimeout } from 'timers'; - -// A cache that expires. -export default class Cache extends Map { - constructor(timeout = 1000) { - super(); - this.timeout = timeout; - } - set(key, value) { - if (this.has(key)) { - clearTimeout(super.get(key).tid); - } - super.set(key, { - tid: setTimeout(this.delete.bind(this, key), this.timeout).unref(), - value - }); - } - get(key) { - let entry = super.get(key); - if (entry) { - return entry.value; - } - return null; - } - getOrSet(key, fn) { - if (this.has(key)) { - return this.get(key); - } else { - let value = fn(); - this.set(key, value); - (async() => { - try { - await value; - } catch (err) { - this.delete(key); - } - })(); - return value; - } - } - delete(key) { - let entry = super.get(key); - if (entry) { - clearTimeout(entry.tid); - super.delete(key); - } - } - clear() { - for (let entry of this.values()) { - clearTimeout(entry.tid); - } - super.clear(); - } -} diff --git a/packages/core/src/ytdl-core/format-utils.js b/packages/core/src/ytdl-core/format-utils.js deleted file mode 100644 index e18d1a244d..0000000000 --- a/packages/core/src/ytdl-core/format-utils.js +++ /dev/null @@ -1,250 +0,0 @@ -import {between} from './utils'; -import FORMATS from './formats'; - - -// Use these to help sort formats, higher index is better. -const audioEncodingRanks = [ - 'mp4a', - 'mp3', - 'vorbis', - 'aac', - 'opus', - 'flac' -]; -const videoEncodingRanks = [ - 'mp4v', - 'avc1', - 'Sorenson H.283', - 'MPEG-4 Visual', - 'VP8', - 'VP9', - 'H.264' -]; - -const getVideoBitrate = format => format.bitrate || 0; -const getVideoEncodingRank = format => - videoEncodingRanks.findIndex(enc => format.codecs && format.codecs.includes(enc)); -const getAudioBitrate = format => format.audioBitrate || 0; -const getAudioEncodingRank = format => - audioEncodingRanks.findIndex(enc => format.codecs && format.codecs.includes(enc)); - - -/** - * Sort formats by a list of functions. - * - * @param {Object} a - * @param {Object} b - * @param {Array.} sortBy - * @returns {number} - */ -const sortFormatsBy = (a, b, sortBy) => { - let res = 0; - for (let fn of sortBy) { - res = fn(b) - fn(a); - if (res !== 0) { - break; - } - } - return res; -}; - - -const sortFormatsByVideo = (a, b) => sortFormatsBy(a, b, [ - format => parseInt(format.qualityLabel), - getVideoBitrate, - getVideoEncodingRank -]); - - -const sortFormatsByAudio = (a, b) => sortFormatsBy(a, b, [ - getAudioBitrate, - getAudioEncodingRank -]); - - -/** - * Sort formats from highest quality to lowest. - * - * @param {Object} a - * @param {Object} b - * @returns {number} - */ -export const sortFormats = (a, b) => sortFormatsBy(a, b, [ - // Formats with both video and audio are ranked highest. - format => +!!format.isHLS, - format => +!!format.isDashMPD, - format => +(format.contentLength > 0), - format => +(format.hasVideo && format.hasAudio), - format => +format.hasVideo, - format => parseInt(format.qualityLabel) || 0, - getVideoBitrate, - getAudioBitrate, - getVideoEncodingRank, - getAudioEncodingRank -]); - - -/** - * Choose a format depending on the given options. - * - * @param {Array.} formats - * @param {Object} options - * @returns {Object} - * @throws {Error} when no format matches the filter/format rules - */ -export const chooseFormat = (formats, options) => { - if (typeof options.format === 'object') { - if (!options.format.url) { - throw Error('Invalid format given, did you use `ytdl.getInfo()`?'); - } - return options.format; - } - - if (options.filter) { - formats = filterFormats(formats, options.filter); - } - - // We currently only support HLS-Formats for livestreams - // So we (now) remove all non-HLS streams - if (formats.some(fmt => fmt.isHLS)) { - formats = formats.filter(fmt => fmt.isHLS || !fmt.isLive); - } - - let format; - const quality = options.quality || 'highest'; - switch (quality) { - case 'highest': - format = formats[0]; - break; - - case 'lowest': - format = formats[formats.length - 1]; - break; - - case 'highestaudio': { - formats = filterFormats(formats, 'audio'); - formats.sort(sortFormatsByAudio); - // Filter for only the best audio format - const bestAudioFormat = formats[0]; - formats = formats.filter(f => sortFormatsByAudio(bestAudioFormat, f) === 0); - // Check for the worst video quality for the best audio quality and pick according - // This does not loose default sorting of video encoding and bitrate - const worstVideoQuality = formats.map(f => parseInt(f.qualityLabel) || 0).sort((a, b) => a - b)[0]; - format = formats.find(f => (parseInt(f.qualityLabel) || 0) === worstVideoQuality); - break; - } - - case 'lowestaudio': - formats = filterFormats(formats, 'audio'); - formats.sort(sortFormatsByAudio); - format = formats[formats.length - 1]; - break; - - case 'highestvideo': { - formats = filterFormats(formats, 'video'); - formats.sort(sortFormatsByVideo); - // Filter for only the best video format - const bestVideoFormat = formats[0]; - formats = formats.filter(f => sortFormatsByVideo(bestVideoFormat, f) === 0); - // Check for the worst audio quality for the best video quality and pick according - // This does not loose default sorting of audio encoding and bitrate - const worstAudioQuality = formats.map(f => f.audioBitrate || 0).sort((a, b) => a - b)[0]; - format = formats.find(f => (f.audioBitrate || 0) === worstAudioQuality); - break; - } - - case 'lowestvideo': - formats = filterFormats(formats, 'video'); - formats.sort(sortFormatsByVideo); - format = formats[formats.length - 1]; - break; - - default: - format = getFormatByQuality(quality, formats); - break; - } - - if (!format) { - throw Error(`No such format found: ${quality}`); - } - return format; -}; - -/** - * Gets a format based on quality or array of quality's - * - * @param {string|[string]} quality - * @param {[Object]} formats - * @returns {Object} - */ -const getFormatByQuality = (quality, formats) => { - let getFormat = itag => formats.find(format => `${format.itag}` === `${itag}`); - if (Array.isArray(quality)) { - return getFormat(quality.find(q => getFormat(q))); - } else { - return getFormat(quality); - } -}; - - -/** - * @param {Array.} formats - * @param {Function} filter - * @returns {Array.} - */ -export const filterFormats = (formats, filter) => { - let fn; - switch (filter) { - case 'videoandaudio': - case 'audioandvideo': - fn = format => format.hasVideo && format.hasAudio; - break; - - case 'video': - fn = format => format.hasVideo; - break; - - case 'videoonly': - fn = format => format.hasVideo && !format.hasAudio; - break; - - case 'audio': - fn = format => format.hasAudio; - break; - - case 'audioonly': - fn = format => !format.hasVideo && format.hasAudio; - break; - - default: - if (typeof filter === 'function') { - fn = filter; - } else { - throw TypeError(`Given filter (${filter}) is not supported`); - } - } - return formats.filter(format => !!format.url && fn(format)); -}; - - -/** - * @param {Object} format - * @returns {Object} - */ -export const addFormatMeta = format => { - format = Object.assign({}, FORMATS[format.itag], format); - format.hasVideo = !!format.qualityLabel; - format.hasAudio = !!format.audioBitrate; - format.container = format.mimeType ? - format.mimeType.split(';')[0].split('/')[1] : null; - format.codecs = format.mimeType ? - between(format.mimeType, 'codecs="', '"') : null; - format.videoCodec = format.hasVideo && format.codecs ? - format.codecs.split(', ')[0] : null; - format.audioCodec = format.hasAudio && format.codecs ? - format.codecs.split(', ').slice(-1)[0] : null; - format.isLive = /\bsource[/=]yt_live_broadcast\b/.test(format.url); - format.isHLS = /\/manifest\/hls_(variant|playlist)\//.test(format.url); - format.isDashMPD = /\/manifest\/dash\//.test(format.url); - return format; -}; diff --git a/packages/core/src/ytdl-core/formats.js b/packages/core/src/ytdl-core/formats.js deleted file mode 100644 index 9c6533d0ea..0000000000 --- a/packages/core/src/ytdl-core/formats.js +++ /dev/null @@ -1,524 +0,0 @@ -/** - * http://en.wikipedia.org/wiki/YouTube#Quality_and_formats - */ -module.exports = { - - 5: { - mimeType: 'video/flv; codecs="Sorenson H.283, mp3"', - qualityLabel: '240p', - bitrate: 250000, - audioBitrate: 64 - }, - - 6: { - mimeType: 'video/flv; codecs="Sorenson H.263, mp3"', - qualityLabel: '270p', - bitrate: 800000, - audioBitrate: 64 - }, - - 13: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: null, - bitrate: 500000, - audioBitrate: null - }, - - 17: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: '144p', - bitrate: 50000, - audioBitrate: 24 - }, - - 18: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '360p', - bitrate: 500000, - audioBitrate: 96 - }, - - 22: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 2000000, - audioBitrate: 192 - }, - - 34: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: '360p', - bitrate: 500000, - audioBitrate: 128 - }, - - 35: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: '480p', - bitrate: 800000, - audioBitrate: 128 - }, - - 36: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: '240p', - bitrate: 175000, - audioBitrate: 32 - }, - - 37: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '1080p', - bitrate: 3000000, - audioBitrate: 192 - }, - - 38: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '3072p', - bitrate: 3500000, - audioBitrate: 192 - }, - - 43: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: '360p', - bitrate: 500000, - audioBitrate: 128 - }, - - 44: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: '480p', - bitrate: 1000000, - audioBitrate: 128 - }, - - 45: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: '720p', - bitrate: 2000000, - audioBitrate: 192 - }, - - 46: { - mimeType: 'audio/webm; codecs="vp8, vorbis"', - qualityLabel: '1080p', - bitrate: null, - audioBitrate: 192 - }, - - 82: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '360p', - bitrate: 500000, - audioBitrate: 96 - }, - - 83: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '240p', - bitrate: 500000, - audioBitrate: 96 - }, - - 84: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 2000000, - audioBitrate: 192 - }, - - 85: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: '1080p', - bitrate: 3000000, - audioBitrate: 192 - }, - - 91: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '144p', - bitrate: 100000, - audioBitrate: 48 - }, - - 92: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '240p', - bitrate: 150000, - audioBitrate: 48 - }, - - 93: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '360p', - bitrate: 500000, - audioBitrate: 128 - }, - - 94: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '480p', - bitrate: 800000, - audioBitrate: 128 - }, - - 95: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 1500000, - audioBitrate: 256 - }, - - 96: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '1080p', - bitrate: 2500000, - audioBitrate: 256 - }, - - 100: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: '360p', - bitrate: null, - audioBitrate: 128 - }, - - 101: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: '360p', - bitrate: null, - audioBitrate: 192 - }, - - 102: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: '720p', - bitrate: null, - audioBitrate: 192 - }, - - 120: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 2000000, - audioBitrate: 128 - }, - - 127: { - mimeType: 'audio/ts; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 96 - }, - - 128: { - mimeType: 'audio/ts; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 96 - }, - - 132: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '240p', - bitrate: 150000, - audioBitrate: 48 - }, - - 133: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '240p', - bitrate: 200000, - audioBitrate: null - }, - - 134: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '360p', - bitrate: 300000, - audioBitrate: null - }, - - 135: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '480p', - bitrate: 500000, - audioBitrate: null - }, - - 136: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '720p', - bitrate: 1000000, - audioBitrate: null - }, - - 137: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '1080p', - bitrate: 2500000, - audioBitrate: null - }, - - 138: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '4320p', - bitrate: 13500000, - audioBitrate: null - }, - - 139: { - mimeType: 'audio/mp4; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 48 - }, - - 140: { - mimeType: 'audio/m4a; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 128 - }, - - 141: { - mimeType: 'audio/mp4; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 256 - }, - - 151: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 50000, - audioBitrate: 24 - }, - - 160: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '144p', - bitrate: 100000, - audioBitrate: null - }, - - 171: { - mimeType: 'audio/webm; codecs="vorbis"', - qualityLabel: null, - bitrate: null, - audioBitrate: 128 - }, - - 172: { - mimeType: 'audio/webm; codecs="vorbis"', - qualityLabel: null, - bitrate: null, - audioBitrate: 192 - }, - - 242: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '240p', - bitrate: 100000, - audioBitrate: null - }, - - 243: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '360p', - bitrate: 250000, - audioBitrate: null - }, - - 244: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '480p', - bitrate: 500000, - audioBitrate: null - }, - - 247: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '720p', - bitrate: 700000, - audioBitrate: null - }, - - 248: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1080p', - bitrate: 1500000, - audioBitrate: null - }, - - 249: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 48 - }, - - 250: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 64 - }, - - 251: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 160 - }, - - 264: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '1440p', - bitrate: 4000000, - audioBitrate: null - }, - - 266: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '2160p', - bitrate: 12500000, - audioBitrate: null - }, - - 271: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1440p', - bitrate: 9000000, - audioBitrate: null - }, - - 272: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '4320p', - bitrate: 20000000, - audioBitrate: null - }, - - 278: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '144p 30fps', - bitrate: 80000, - audioBitrate: null - }, - - 298: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '720p', - bitrate: 3000000, - audioBitrate: null - }, - - 299: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: '1080p', - bitrate: 5500000, - audioBitrate: null - }, - - 300: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: '720p', - bitrate: 1318000, - audioBitrate: 48 - }, - - 302: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '720p HFR', - bitrate: 2500000, - audioBitrate: null - }, - - 303: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1080p HFR', - bitrate: 5000000, - audioBitrate: null - }, - - 308: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1440p HFR', - bitrate: 10000000, - audioBitrate: null - }, - - 313: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '2160p', - bitrate: 13000000, - audioBitrate: null - }, - - 315: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '2160p HFR', - bitrate: 20000000, - audioBitrate: null - }, - - 330: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '144p HDR, HFR', - bitrate: 80000, - audioBitrate: null - }, - - 331: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '240p HDR, HFR', - bitrate: 100000, - audioBitrate: null - }, - - 332: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '360p HDR, HFR', - bitrate: 250000, - audioBitrate: null - }, - - 333: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '240p HDR, HFR', - bitrate: 500000, - audioBitrate: null - }, - - 334: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '720p HDR, HFR', - bitrate: 1000000, - audioBitrate: null - }, - - 335: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1080p HDR, HFR', - bitrate: 1500000, - audioBitrate: null - }, - - 336: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '1440p HDR, HFR', - bitrate: 5000000, - audioBitrate: null - }, - - 337: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: '2160p HDR, HFR', - bitrate: 12000000, - audioBitrate: null - } - -}; diff --git a/packages/core/src/ytdl-core/index.d.ts b/packages/core/src/ytdl-core/index.d.ts deleted file mode 100644 index 2361438855..0000000000 --- a/packages/core/src/ytdl-core/index.d.ts +++ /dev/null @@ -1,774 +0,0 @@ -declare module 'tough-cookie' { - export const version: string; - - export const PrefixSecurityEnum: Readonly<{ - DISABLED: string; - SILENT: string; - STRICT: string; - }>; - - /** - * Parse a cookie date string into a Date. - * Parses according to RFC6265 Section 5.1.1, not Date.parse(). - */ - export function parseDate(string: string): Date; - - /** - * Format a Date into a RFC1123 string (the RFC6265-recommended format). - */ - export function formatDate(date: Date): string; - - /** - * Transforms a domain-name into a canonical domain-name. - * The canonical domain-name is a trimmed, lowercased, stripped-of-leading-dot - * and optionally punycode-encoded domain-name (Section 5.1.2 of RFC6265). - * For the most part, this function is idempotent (can be run again on its output without ill effects). - */ - export function canonicalDomain(str: string): string; - - /** - * Answers "does this real domain match the domain in a cookie?". - * The str is the "current" domain-name and the domStr is the "cookie" domain-name. - * Matches according to RFC6265 Section 5.1.3, but it helps to think of it as a "suffix match". - * - * The canonicalize parameter will run the other two parameters through canonicalDomain or not. - */ - export function domainMatch(str: string, domStr: string, canonicalize?: boolean): boolean; - - /** - * Given a current request/response path, gives the Path apropriate for storing in a cookie. - * This is basically the "directory" of a "file" in the path, but is specified by Section 5.1.4 of the RFC. - * - * The path parameter MUST be only the pathname part of a URI (i.e. excludes the hostname, query, fragment, etc.). - * This is the .pathname property of node's uri.parse() output. - */ - export function defaultPath(path: string): string; - - /** - * Answers "does the request-path path-match a given cookie-path?" as per RFC6265 Section 5.1.4. - * Returns a boolean. - * - * This is essentially a prefix-match where cookiePath is a prefix of reqPath. - */ - export function pathMatch(reqPath: string, cookiePath: string): boolean; - - /** - * alias for Cookie.parse(cookieString[, options]) - */ - export function parse(cookieString: string, options?: Cookie.ParseOptions): Cookie | undefined; - - /** - * alias for Cookie.fromJSON(string) - */ - export function fromJSON(string: string): Cookie; - - export function getPublicSuffix(hostname: string): string | null; - - export function cookieCompare(a: Cookie, b: Cookie): number; - - export function permuteDomain(domain: string, allowSpecialUseDomain?: boolean): string[]; - - export function permutePath(path: string): string[]; - - export class Cookie { - static parse(cookieString: string, options?: Cookie.ParseOptions): Cookie | undefined; - - static fromJSON(strOrObj: string | object): Cookie | null; - - constructor(properties?: Cookie.Properties); - - key: string; - value: string; - expires: Date | 'Infinity'; - maxAge: number | 'Infinity' | '-Infinity'; - domain: string | null; - path: string | null; - secure: boolean; - httpOnly: boolean; - extensions: string[] | null; - creation: Date | null; - creationIndex: number; - - hostOnly: boolean | null; - pathIsDefault: boolean | null; - lastAccessed: Date | null; - sameSite: string; - - toString(): string; - - cookieString(): string; - - setExpires(exp: Date | string): void; - - setMaxAge(number: number): void; - - expiryTime(now?: number): number; - - expiryDate(now?: number): Date; - - TTL(now?: Date): number | typeof Infinity; - - isPersistent(): boolean; - - canonicalizedDomain(): string | null; - - cdomain(): string | null; - - inspect(): string; - - toJSON(): { [key: string]: any }; - - clone(): Cookie; - - validate(): boolean | string; - } - - export namespace Cookie { - interface ParseOptions { - loose?: boolean | undefined; - } - - interface Properties { - key?: string | undefined; - value?: string | undefined; - expires?: Date | 'Infinity' | undefined; - maxAge?: number | 'Infinity' | '-Infinity' | undefined; - domain?: string | undefined; - path?: string | undefined; - secure?: boolean | undefined; - httpOnly?: boolean | undefined; - extensions?: string[] | undefined; - creation?: Date | undefined; - creationIndex?: number | undefined; - - hostOnly?: boolean | undefined; - pathIsDefault?: boolean | undefined; - lastAccessed?: Date | undefined; - sameSite?: string | undefined; - } - - interface Serialized { - [key: string]: any; - } - } - - export class CookieJar { - static deserialize(serialized: CookieJar.Serialized | string, store?: Store): Promise; - static deserialize( - serialized: CookieJar.Serialized | string, - store: Store, - cb: (err: Error | null, object: CookieJar) => void, - ): void; - static deserialize( - serialized: CookieJar.Serialized | string, - cb: (err: Error | null, object: CookieJar) => void, - ): void; - - static deserializeSync(serialized: CookieJar.Serialized | string, store?: Store): CookieJar; - - static fromJSON(string: string): CookieJar; - - constructor(store?: Store, options?: CookieJar.Options); - - setCookie( - cookieOrString: Cookie | string, - currentUrl: string, - options?: CookieJar.SetCookieOptions, - ): Promise; - setCookie( - cookieOrString: Cookie | string, - currentUrl: string, - options: CookieJar.SetCookieOptions, - cb: (err: Error | null, cookie: Cookie) => void, - ): void; - setCookie( - cookieOrString: Cookie | string, - currentUrl: string, - cb: (err: Error | null, cookie: Cookie) => void, - ): void; - - setCookieSync(cookieOrString: Cookie | string, currentUrl: string, options?: CookieJar.SetCookieOptions): Cookie; - - getCookies(currentUrl: string, options?: CookieJar.GetCookiesOptions): Promise; - getCookies( - currentUrl: string, - options: CookieJar.GetCookiesOptions, - cb: (err: Error | null, cookies: Cookie[]) => void, - ): void; - getCookies(currentUrl: string, cb: (err: Error | null, cookies: Cookie[]) => void): void; - - getCookiesSync(currentUrl: string, options?: CookieJar.GetCookiesOptions): Cookie[]; - - getCookieString(currentUrl: string, options?: CookieJar.GetCookiesOptions): Promise; - getCookieString( - currentUrl: string, - options: CookieJar.GetCookiesOptions, - cb: (err: Error | null, cookies: string) => void, - ): void; - getCookieString(currentUrl: string, cb: (err: Error | null, cookies: string) => void): void; - - getCookieStringSync(currentUrl: string, options?: CookieJar.GetCookiesOptions): string; - - getSetCookieStrings(currentUrl: string, options?: CookieJar.GetCookiesOptions): Promise; - getSetCookieStrings( - currentUrl: string, - options: CookieJar.GetCookiesOptions, - cb: (err: Error | null, cookies: string[]) => void, - ): void; - getSetCookieStrings(currentUrl: string, cb: (err: Error | null, cookies: string[]) => void): void; - - getSetCookieStringsSync(currentUrl: string, options?: CookieJar.GetCookiesOptions): string[]; - - serialize(): Promise; - serialize(cb: (err: Error | null, serializedObject: CookieJar.Serialized) => void): void; - - serializeSync(): CookieJar.Serialized; - - toJSON(): CookieJar.Serialized; - - clone(store?: Store): Promise; - clone(store: Store, cb: (err: Error | null, newJar: CookieJar) => void): void; - clone(cb: (err: Error | null, newJar: CookieJar) => void): void; - - cloneSync(store?: Store): CookieJar; - - removeAllCookies(): Promise; - removeAllCookies(cb: (err: Error | null) => void): void; - - removeAllCookiesSync(): void; - } - - export namespace CookieJar { - interface Options { - allowSpecialUseDomain?: boolean | undefined; - looseMode?: boolean | undefined; - rejectPublicSuffixes?: boolean | undefined; - prefixSecurity?: string | undefined; - } - - interface SetCookieOptions { - http?: boolean | undefined; - secure?: boolean | undefined; - now?: Date | undefined; - ignoreError?: boolean | undefined; - } - - interface GetCookiesOptions { - http?: boolean | undefined; - secure?: boolean | undefined; - now?: Date | undefined; - expire?: boolean | undefined; - allPaths?: boolean | undefined; - } - - interface Serialized { - version: string; - storeType: string; - rejectPublicSuffixes: boolean; - cookies: Cookie.Serialized[]; - } - } - - export abstract class Store { - synchronous: boolean; - - findCookie(domain: string, path: string, key: string, cb: (err: Error | null, cookie: Cookie | null) => void): void; - - findCookies( - domain: string, - path: string, - allowSpecialUseDomain: boolean, - cb: (err: Error | null, cookie: Cookie[]) => void, - ): void; - - putCookie(cookie: Cookie, cb: (err: Error | null) => void): void; - - updateCookie(oldCookie: Cookie, newCookie: Cookie, cb: (err: Error | null) => void): void; - - removeCookie(domain: string, path: string, key: string, cb: (err: Error | null) => void): void; - - removeCookies(domain: string, path: string, cb: (err: Error | null) => void): void; - - getAllCookies(cb: (err: Error | null, cookie: Cookie[]) => void): void; - } - - export class MemoryCookieStore extends Store { - findCookie(domain: string, path: string, key: string, cb: (err: Error | null, cookie: Cookie | null) => void): void; - findCookie(domain: string, path: string, key: string): Promise; - - findCookies( - domain: string, - path: string, - allowSpecialUseDomain: boolean, - cb: (err: Error | null, cookie: Cookie[]) => void, - ): void; - findCookies(domain: string, path: string, cb: (err: Error | null, cookie: Cookie[]) => void): void; - findCookies(domain: string, path: string, allowSpecialUseDomain?: boolean): Promise; - - putCookie(cookie: Cookie, cb: (err: Error | null) => void): void; - putCookie(cookie: Cookie): Promise; - - updateCookie(oldCookie: Cookie, newCookie: Cookie, cb: (err: Error | null) => void): void; - updateCookie(oldCookie: Cookie, newCookie: Cookie): Promise; - - removeCookie(domain: string, path: string, key: string, cb: (err: Error | null) => void): void; - removeCookie(domain: string, path: string, key: string): Promise; - - removeCookies(domain: string, path: string, cb: (err: Error | null) => void): void; - removeCookies(domain: string, path: string): Promise; - - getAllCookies(cb: (err: Error | null, cookie: Cookie[]) => void): void; - getAllCookies(): Promise; - } -} - -import { Dispatcher, ProxyAgent, request } from 'undici'; -import { Cookie as CK, CookieJar } from 'tough-cookie'; -import { CookieAgent } from 'http-cookie-agent/undici'; -import { Readable } from 'stream'; - - namespace ytdl { - type Filter = 'audioandvideo' | 'videoandaudio' | 'video' | 'videoonly' | 'audio' | 'audioonly' | ((format: videoFormat) => boolean); - - interface Agent { - dispatcher: Dispatcher; - jar: CookieJar; - localAddress?: string; - } - - interface getInfoOptions { - lang?: string; - requestCallback?: () => {}; - requestOptions?: Parameters[1]; - agent?: Agent; - } - - interface chooseFormatOptions { - quality?: 'lowest' | 'highest' | 'highestaudio' | 'lowestaudio' | 'highestvideo' | 'lowestvideo' | string | number | string[] | number[]; - filter?: Filter; - format?: videoFormat; - } - - interface downloadOptions extends getInfoOptions, chooseFormatOptions { - range?: { - start?: number; - end?: number; - }; - begin?: string | number | Date; - liveBuffer?: number; - highWaterMark?: number; - IPv6Block?: string; - dlChunkSize?: number; - } - - interface videoFormat { - itag: number; - url: string; - mimeType?: string; - bitrate?: number; - audioBitrate?: number; - width?: number; - height?: number; - initRange?: { start: string; end: string }; - indexRange?: { start: string; end: string }; - lastModified: string; - contentLength: string; - quality: 'tiny' | 'small' | 'medium' | 'large' | 'hd720' | 'hd1080' | 'hd1440' | 'hd2160' | 'highres' | string; - qualityLabel: '144p' | '144p 15fps' | '144p60 HDR' | '240p' | '240p60 HDR' | '270p' | '360p' | '360p60 HDR' - | '480p' | '480p60 HDR' | '720p' | '720p60' | '720p60 HDR' | '1080p' | '1080p60' | '1080p60 HDR' | '1440p' - | '1440p60' | '1440p60 HDR' | '2160p' | '2160p60' | '2160p60 HDR' | '4320p' | '4320p60'; - projectionType?: 'RECTANGULAR'; - fps?: number; - averageBitrate?: number; - audioQuality?: 'AUDIO_QUALITY_LOW' | 'AUDIO_QUALITY_MEDIUM'; - colorInfo?: { - primaries: string; - transferCharacteristics: string; - matrixCoefficients: string; - }; - highReplication?: boolean; - approxDurationMs?: string; - targetDurationSec?: number; - maxDvrDurationSec?: number; - audioSampleRate?: string; - audioChannels?: number; - - // Added by ytdl-core - container: 'flv' | '3gp' | 'mp4' | 'webm' | 'ts'; - hasVideo: boolean; - hasAudio: boolean; - codecs: string; - videoCodec?: string; - audioCodec?: string; - - isLive: boolean; - isHLS: boolean; - isDashMPD: boolean; - } - - interface thumbnail { - url: string; - width: number; - height: number; - } - - interface captionTrack { - baseUrl: string; - name: { - simpleText: 'Afrikaans' | 'Albanian' | 'Amharic' | 'Arabic' | 'Armenian' | 'Azerbaijani' | 'Bangla' | 'Basque' - | 'Belarusian' | 'Bosnian' | 'Bulgarian' | 'Burmese' | 'Catalan' | 'Cebuano' | 'Chinese (Simplified)' - | 'Chinese (Traditional)' | 'Corsican' | 'Croatian' | 'Czech' | 'Danish' | 'Dutch' | 'English' - | 'English (auto-generated)' | 'Esperanto' | 'Estonian' | 'Filipino' | 'Finnish' | 'French' | 'Galician' - | 'Georgian' | 'German' | 'Greek' | 'Gujarati' | 'Haitian Creole' | 'Hausa' | 'Hawaiian' | 'Hebrew' | 'Hindi' - | 'Hmong' | 'Hungarian' | 'Icelandic' | 'Igbo' | 'Indonesian' | 'Irish' | 'Italian' | 'Japanese' | 'Javanese' - | 'Kannada' | 'Kazakh' | 'Khmer' | 'Korean' | 'Kurdish' | 'Kyrgyz' | 'Lao' | 'Latin' | 'Latvian' | 'Lithuanian' - | 'Luxembourgish' | 'Macedonian' | 'Malagasy' | 'Malay' | 'Malayalam' | 'Maltese' | 'Maori' | 'Marathi' - | 'Mongolian' | 'Nepali' | 'Norwegian' | 'Nyanja' | 'Pashto' | 'Persian' | 'Polish' | 'Portuguese' | 'Punjabi' - | 'Romanian' | 'Russian' | 'Samoan' | 'Scottish Gaelic' | 'Serbian' | 'Shona' | 'Sindhi' | 'Sinhala' | 'Slovak' - | 'Slovenian' | 'Somali' | 'Southern Sotho' | 'Spanish' | 'Spanish (Spain)' | 'Sundanese' | 'Swahili' - | 'Swedish' | 'Tajik' | 'Tamil' | 'Telugu' | 'Thai' | 'Turkish' | 'Ukrainian' | 'Urdu' | 'Uzbek' | 'Vietnamese' - | 'Welsh' | 'Western Frisian' | 'Xhosa' | 'Yiddish' | 'Yoruba' | 'Zulu' | string; - }; - vssId: string; - languageCode: 'af' | 'sq' | 'am' | 'ar' | 'hy' | 'az' | 'bn' | 'eu' | 'be' | 'bs' | 'bg' | 'my' | 'ca' | 'ceb' - | 'zh-Hans' | 'zh-Hant' | 'co' | 'hr' | 'cs' | 'da' | 'nl' | 'en' | 'eo' | 'et' | 'fil' | 'fi' | 'fr' | 'gl' - | 'ka' | 'de' | 'el' | 'gu' | 'ht' | 'ha' | 'haw' | 'iw' | 'hi' | 'hmn' | 'hu' | 'is' | 'ig' | 'id' | 'ga' | 'it' - | 'ja' | 'jv' | 'kn' | 'kk' | 'km' | 'ko' | 'ku' | 'ky' | 'lo' | 'la' | 'lv' | 'lt' | 'lb' | 'mk' | 'mg' | 'ms' - | 'ml' | 'mt' | 'mi' | 'mr' | 'mn' | 'ne' | 'no' | 'ny' | 'ps' | 'fa' | 'pl' | 'pt' | 'pa' | 'ro' | 'ru' | 'sm' - | 'gd' | 'sr' | 'sn' | 'sd' | 'si' | 'sk' | 'sl' | 'so' | 'st' | 'es' | 'su' | 'sw' | 'sv' | 'tg' | 'ta' | 'te' - | 'th' | 'tr' | 'uk' | 'ur' | 'uz' | 'vi' | 'cy' | 'fy' | 'xh' | 'yi' | 'yo' | 'zu' | string; - kind: string; - rtl?: boolean; - isTranslatable: boolean; - } - - interface audioTrack { - captionTrackIndices: number[]; - } - - interface translationLanguage { - languageCode: captionTrack['languageCode']; - languageName: captionTrack['name']; - } - - interface VideoDetails { - videoId: string; - title: string; - shortDescription: string; - lengthSeconds: string; - keywords?: string[]; - channelId: string; - isOwnerViewing: boolean; - isCrawlable: boolean; - thumbnails: thumbnail[]; - averageRating: number; - allowRatings: boolean; - viewCount: string; - author: string; - isPrivate: boolean; - isUnpluggedCorpus: boolean; - isLiveContent: boolean; - isLive: boolean; - } - - interface Media { - category: string; - category_url: string; - game?: string; - game_url?: string; - year?: number; - song?: string; - artist?: string; - artist_url?: string; - writers?: string; - licensed_by?: string; - thumbnails: thumbnail[]; - } - - interface Author { - id: string; - name: string; - avatar: string; // to remove later - thumbnails?: thumbnail[]; - verified: boolean; - user?: string; - channel_url: string; - external_channel_url?: string; - user_url?: string; - subscriber_count?: number; - } - - interface MicroformatRenderer { - thumbnail: { - thumbnails: thumbnail[]; - }; - embed: { - iframeUrl: string; - flashUrl: string; - width: number; - height: number; - flashSecureUrl: string; - }; - title: { - simpleText: string; - }; - description: { - simpleText: string; - }; - lengthSeconds: string; - ownerProfileUrl: string; - ownerGplusProfileUrl?: string; - externalChannelId: string; - isFamilySafe: boolean; - availableCountries: string[]; - isUnlisted: boolean; - hasYpcMetadata: boolean; - viewCount: string; - category: string; - publishDate: string; - ownerChannelName: string; - liveBroadcastDetails?: { - isLiveNow: boolean; - startTimestamp: string; - endTimestamp?: string; - }; - uploadDate: string; - } - - interface storyboard { - templateUrl: string; - thumbnailWidth: number; - thumbnailHeight: number; - thumbnailCount: number; - interval: number; - columns: number; - rows: number; - storyboardCount: number; - } - - interface Chapter { - title: string; - start_time: number; - } - - interface MoreVideoDetails extends Omit, Omit { - published: number; - video_url: string; - age_restricted: boolean; - likes: number | null; - media: Media; - author: Author; - thumbnails: thumbnail[]; - storyboards: storyboard[]; - chapters: Chapter[]; - description: string | null; - } - - interface videoInfo { - iv_load_policy?: string; - iv_allow_in_place_switch?: string; - iv_endscreen_url?: string; - iv_invideo_url?: string; - iv3_module?: string; - rmktEnabled?: string; - uid?: string; - vid?: string; - focEnabled?: string; - baseUrl?: string; - storyboard_spec?: string; - serialized_ad_ux_config?: string; - player_error_log_fraction?: string; - sffb?: string; - ldpj?: string; - videostats_playback_base_url?: string; - innertube_context_client_version?: string; - t?: string; - fade_in_start_milliseconds: string; - timestamp: string; - ad3_module: string; - relative_loudness: string; - allow_below_the_player_companion: string; - eventid: string; - token: string; - atc: string; - cr: string; - apply_fade_on_midrolls: string; - cl: string; - fexp: string[]; - apiary_host: string; - fade_in_duration_milliseconds: string; - fflags: string; - ssl: string; - pltype: string; - enabled_engage_types: string; - hl: string; - is_listed: string; - gut_tag: string; - apiary_host_firstparty: string; - enablecsi: string; - csn: string; - status: string; - afv_ad_tag: string; - idpj: string; - sfw_player_response: string; - account_playback_token: string; - encoded_ad_safety_reason: string; - tag_for_children_directed: string; - no_get_video_log: string; - ppv_remarketing_url: string; - fmt_list: string[][]; - ad_slots: string; - fade_out_duration_milliseconds: string; - instream_long: string; - allow_html5_ads: string; - core_dbp: string; - ad_device: string; - itct: string; - root_ve_type: string; - excluded_ads: string; - aftv: string; - loeid: string; - cver: string; - shortform: string; - dclk: string; - csi_page_type: string; - ismb: string; - gpt_migration: string; - loudness: string; - ad_tag: string; - of: string; - probe_url: string; - vm: string; - afv_ad_tag_restricted_to_instream: string; - gapi_hint_params: string; - cid: string; - c: string; - oid: string; - ptchn: string; - as_launched_in_country: string; - avg_rating: string; - fade_out_start_milliseconds: string; - midroll_prefetch_size: string; - allow_ratings: string; - thumbnail_url: string; - iurlsd: string; - iurlmq: string; - iurlhq: string; - iurlmaxres: string; - ad_preroll: string; - tmi: string; - trueview: string; - host_language: string; - innertube_api_key: string; - show_content_thumbnail: string; - afv_instream_max: string; - innertube_api_version: string; - mpvid: string; - allow_embed: string; - ucid: string; - plid: string; - midroll_freqcap: string; - ad_logging_flag: string; - ptk: string; - vmap: string; - watermark: string[]; - dbp: string; - ad_flags: string; - html5player: string; - formats: videoFormat[]; - related_videos: relatedVideo[]; - no_embed_allowed?: boolean; - player_response: { - playabilityStatus: { - status: string; - playableInEmbed: boolean; - miniplayer: { - miniplayerRenderer: { - playbackMode: string; - }; - }; - contextParams: string; - }; - streamingData: { - expiresInSeconds: string; - formats: {}[]; - adaptiveFormats: {}[]; - }; - captions?: { - playerCaptionsRenderer: { - baseUrl: string; - visibility: string; - }; - playerCaptionsTracklistRenderer: { - captionTracks: captionTrack[]; - audioTracks: audioTrack[]; - translationLanguages: translationLanguage[]; - defaultAudioTrackIndex: number; - }; - }; - microformat: { - playerMicroformatRenderer: MicroformatRenderer; - }; - videoDetails: VideoDetails; - playerConfig: { - audioConfig: { - loudnessDb: number; - perceptualLoudnessDb: number; - enablePerFormatLoudness: boolean; - }; - streamSelectionConfig: { maxBitrate: string }; - mediaCommonConfig: { dynamicReadaheadConfig: {}[] }; - webPlayerConfig: { webPlayerActionsPorting: {}[] }; - }; - }; - videoDetails: MoreVideoDetails; - } - - interface relatedVideo { - id?: string; - title?: string; - published?: string; - author: Author | 'string'; // to remove the `string` part later - ucid?: string; // to remove later - author_thumbnail?: string; // to remove later - short_view_count_text?: string; - view_count?: string; - length_seconds?: number; - video_thumbnail?: string; // to remove later - thumbnails: thumbnail[]; - richThumbnails: thumbnail[]; - isLive: boolean; - } - - interface Cookie { - name: string; - value: string; - expirationDate?: number; - domain?: string; - path?: string; - secure?: boolean; - httpOnly?: boolean; - hostOnly?: boolean; - sameSite?: string; - } - - function getBasicInfo(url: string, options?: getInfoOptions): Promise; - function getInfo(url: string, options?: getInfoOptions): Promise; - function downloadFromInfo(info: videoInfo, options?: downloadOptions): Readable; - function chooseFormat(format: videoFormat | videoFormat[], options?: chooseFormatOptions): videoFormat | never; - function filterFormats(formats: videoFormat | videoFormat[], filter?: Filter): videoFormat[]; - function validateID(string: string): boolean; - function validateURL(string: string): boolean; - function getURLVideoID(string: string): string | never; - function getVideoID(string: string): string | never; - function createProxyAgent(options: ProxyAgent.Options | string): Agent; - function createProxyAgent(options: ProxyAgent.Options | string, cookies?: (Cookie | CK)[]): Agent; - function createAgent(): Agent; - function createAgent(cookies?: (Cookie | CK)[]): Agent; - function createAgent(cookies?: (Cookie | CK)[], opts?: CookieAgent.Options): Agent; - const version: number; - } - - function ytdl(link: string, options?: ytdl.downloadOptions): Readable; - - export = ytdl; diff --git a/packages/core/src/ytdl-core/index.js b/packages/core/src/ytdl-core/index.js deleted file mode 100644 index e439fefc25..0000000000 --- a/packages/core/src/ytdl-core/index.js +++ /dev/null @@ -1,239 +0,0 @@ -import { PassThrough } from 'stream'; -import {getBasicInfo, getInfo, cache as getInfoCache, watchPageCache} from './info'; -import {playError, applyDefaultHeaders, getRandomIPv6, setPropInsensitive} from './utils'; -import {chooseFormat, filterFormats} from './format-utils'; -import {getURLVideoID, getVideoID, validateID, validateURL} from './url-utils'; -import {cache as sigCache} from './sig'; -import miniget from 'miniget'; -import m3u8stream, { parseTimestamp } from 'm3u8stream'; -import {createAgent, createProxyAgent} from './agent'; - - -/** - * @param {string} link - * @param {!Object} options - * @returns {ReadableStream} - */ -const ytdl = (link, options) => { - const stream = createStream(options); - ytdl.getInfo(link, options).then(info => { - downloadFromInfoCallback(stream, info, options); - }, stream.emit.bind(stream, 'error')); - return stream; -}; -export default ytdl; - -ytdl.getBasicInfo = getBasicInfo; -ytdl.getInfo = getInfo; -ytdl.chooseFormat = chooseFormat; -ytdl.filterFormats = filterFormats; -ytdl.validateID = validateID; -ytdl.validateURL = validateURL; -ytdl.getURLVideoID = getURLVideoID; -ytdl.getVideoID = getVideoID; -ytdl.createAgent = createAgent; -ytdl.createProxyAgent = createProxyAgent; -ytdl.cache = { - sig: sigCache, - info: getInfoCache, - watch: watchPageCache -}; - - -const createStream = options => { - const stream = new PassThrough({ - highWaterMark: (options && options.highWaterMark) || 1024 * 512 - }); - stream._destroy = () => { - stream.destroyed = true; - }; - return stream; -}; - - -const pipeAndSetEvents = (req, stream, end) => { - // Forward events from the request to the stream. - [ - 'abort', 'request', 'response', 'error', 'redirect', 'retry', 'reconnect' - ].forEach(event => { - req.prependListener(event, stream.emit.bind(stream, event)); - }); - req.pipe(stream, { end }); -}; - - -/** - * Chooses a format to download. - * - * @param {stream.Readable} stream - * @param {Object} info - * @param {Object} options - */ -const downloadFromInfoCallback = (stream, info, options) => { - options = options || {}; - - let err = playError(info.player_response, ['UNPLAYABLE', 'LIVE_STREAM_OFFLINE', 'LOGIN_REQUIRED']); - if (err) { - stream.emit('error', err); - return; - } - - if (!info.formats.length) { - stream.emit('error', Error('This video is unavailable')); - return; - } - - let format; - try { - format = chooseFormat(info.formats, options); - } catch (e) { - stream.emit('error', e); - return; - } - stream.emit('info', info, format); - if (stream.destroyed) { - return; - } - - let contentLength, downloaded = 0; - const ondata = chunk => { - downloaded += chunk.length; - stream.emit('progress', chunk.length, downloaded, contentLength); - }; - - applyDefaultHeaders(options); - if (options.IPv6Block) { - options.requestOptions = Object.assign({}, options.requestOptions, { - localAddress: getRandomIPv6(options.IPv6Block) - }); - } - if (options.agent) { - if (options.agent.jar) { - setPropInsensitive( - options.requestOptions.headers, 'cookie', options.agent.jar.getCookieStringSync('https://www.youtube.com') - ); - } - if (options.agent.localAddress) { - options.requestOptions.localAddress = options.agent.localAddress; - } - } - - // Download the file in chunks, in this case the default is 10MB, - // anything over this will cause youtube to throttle the download - const dlChunkSize = typeof options.dlChunkSize === 'number' ? options.dlChunkSize : 1024 * 1024 * 10; - let req; - let shouldEnd = true; - - if (format.isHLS || format.isDashMPD) { - req = m3u8stream(format.url, { - chunkReadahead: +info.live_chunk_readahead, - begin: options.begin || (format.isLive && Date.now()), - liveBuffer: options.liveBuffer, - requestOptions: options.requestOptions, - parser: format.isDashMPD ? 'dash-mpd' : 'm3u8', - id: format.itag - }); - - req.on('progress', (segment, totalSegments) => { - stream.emit('progress', segment.size, segment.num, totalSegments); - }); - pipeAndSetEvents(req, stream, shouldEnd); - } else { - const requestOptions = Object.assign({}, options.requestOptions, { - maxReconnects: 6, - maxRetries: 3, - backoff: { inc: 500, max: 10000 } - }); - - let shouldBeChunked = dlChunkSize !== 0 && (!format.hasAudio || !format.hasVideo); - - if (shouldBeChunked) { - let start = (options.range && options.range.start) || 0; - let end = start + dlChunkSize; - const rangeEnd = options.range && options.range.end; - - contentLength = options.range ? - (rangeEnd ? rangeEnd + 1 : parseInt(format.contentLength)) - start : - parseInt(format.contentLength); - - const getNextChunk = () => { - if (stream.destroyed) { - return; - } - if (!rangeEnd && end >= contentLength) { - end = 0; - } - if (rangeEnd && end > rangeEnd) { - end = rangeEnd; - } - shouldEnd = !end || end === rangeEnd; - - requestOptions.headers = Object.assign({}, requestOptions.headers, { - Range: `bytes=${start}-${end || ''}` - }); - req = miniget(format.url, requestOptions); - req.on('data', ondata); - req.on('end', () => { - if (stream.destroyed) { - return; - } - if (end && end !== rangeEnd) { - start = end + 1; - end += dlChunkSize; - getNextChunk(); - } - }); - pipeAndSetEvents(req, stream, shouldEnd); - }; - getNextChunk(); - } else { - // Audio only and video only formats don't support begin - if (options.begin) { - format.url += `&begin=${parseTimestamp(options.begin)}`; - } - if (options.range && (options.range.start || options.range.end)) { - requestOptions.headers = Object.assign({}, requestOptions.headers, { - Range: `bytes=${options.range.start || '0'}-${options.range.end || ''}` - }); - } - req = miniget(format.url, requestOptions); - req.on('response', res => { - if (stream.destroyed) { - return; - } - contentLength = contentLength || parseInt(res.headers['content-length']); - }); - req.on('data', ondata); - pipeAndSetEvents(req, stream, shouldEnd); - } - } - - stream._destroy = () => { - stream.destroyed = true; - if (req) { - req.destroy(); - req.end(); - } - }; -}; - - -/** - * Can be used to download video after its `info` is gotten through - * `ytdl.getInfo()`. In case the user might want to look at the - * `info` object before deciding to download. - * - * @param {Object} info - * @param {!Object} options - * @returns {ReadableStream} - */ -ytdl.downloadFromInfo = (info, options) => { - const stream = createStream(options); - if (!info.full) { - throw Error('Cannot use `ytdl.downloadFromInfo()` when called with info from `ytdl.getBasicInfo()`'); - } - setImmediate(() => { - downloadFromInfoCallback(stream, info, options); - }); - return stream; -}; diff --git a/packages/core/src/ytdl-core/info-extras.js b/packages/core/src/ytdl-core/info-extras.js deleted file mode 100644 index d1e81e307a..0000000000 --- a/packages/core/src/ytdl-core/info-extras.js +++ /dev/null @@ -1,362 +0,0 @@ -import {parseAbbreviatedNumber, deprecate} from './utils'; -import qs from 'querystring'; -import { parseTimestamp } from 'm3u8stream'; -import { URL } from 'url'; - - -const BASE_URL = 'https://www.youtube.com/watch?v='; -const TITLE_TO_CATEGORY = { - song: { name: 'Music', url: 'https://music.youtube.com/' } -}; - -const getText = obj => obj ? obj.runs ? obj.runs[0].text : obj.simpleText : null; - - -/** - * Get video media. - * - * @param {Object} info - * @returns {Object} - */ -export const getMedia = info => { - let media = {}; - let results = []; - try { - results = info.response.contents.twoColumnWatchNextResults.results.results.contents; - } catch (err) { - // Do nothing - } - - let result = results.find(v => v.videoSecondaryInfoRenderer); - if (!result) { - return {}; - } - - try { - let metadataRows = - (result.metadataRowContainer || result.videoSecondaryInfoRenderer.metadataRowContainer) - .metadataRowContainerRenderer.rows; - for (let row of metadataRows) { - if (row.metadataRowRenderer) { - let title = getText(row.metadataRowRenderer.title).toLowerCase(); - let contents = row.metadataRowRenderer.contents[0]; - media[title] = getText(contents); - let runs = contents.runs; - if (runs && runs[0].navigationEndpoint) { - media[`${title}_url`] = new URL( - runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url, BASE_URL).toString(); - } - if (title in TITLE_TO_CATEGORY) { - media.category = TITLE_TO_CATEGORY[title].name; - media.category_url = TITLE_TO_CATEGORY[title].url; - } - } else if (row.richMetadataRowRenderer) { - let contents = row.richMetadataRowRenderer.contents; - let boxArt = contents - .filter(meta => meta.richMetadataRenderer.style === 'RICH_METADATA_RENDERER_STYLE_BOX_ART'); - for (let { richMetadataRenderer } of boxArt) { - let meta = richMetadataRenderer; - media.year = getText(meta.subtitle); - let type = getText(meta.callToAction).split(' ')[1]; - media[type] = getText(meta.title); - media[`${type}_url`] = new URL( - meta.endpoint.commandMetadata.webCommandMetadata.url, BASE_URL).toString(); - media.thumbnails = meta.thumbnail.thumbnails; - } - let topic = contents - .filter(meta => meta.richMetadataRenderer.style === 'RICH_METADATA_RENDERER_STYLE_TOPIC'); - for (let { richMetadataRenderer } of topic) { - let meta = richMetadataRenderer; - media.category = getText(meta.title); - media.category_url = new URL( - meta.endpoint.commandMetadata.webCommandMetadata.url, BASE_URL).toString(); - } - } - } - } catch (err) { - // Do nothing. - } - - return media; -}; - - -const isVerified = badges => !!(badges && badges.find(b => b.metadataBadgeRenderer.tooltip === 'Verified')); - - -/** - * Get video author. - * - * @param {Object} info - * @returns {Object} - */ -export const getAuthor = info => { - let channelId, thumbnails = [], subscriberCount, verified = false; - try { - let results = info.response.contents.twoColumnWatchNextResults.results.results.contents; - let v = results.find(v2 => - v2.videoSecondaryInfoRenderer && - v2.videoSecondaryInfoRenderer.owner && - v2.videoSecondaryInfoRenderer.owner.videoOwnerRenderer); - let videoOwnerRenderer = v.videoSecondaryInfoRenderer.owner.videoOwnerRenderer; - channelId = videoOwnerRenderer.navigationEndpoint.browseEndpoint.browseId; - thumbnails = videoOwnerRenderer.thumbnail.thumbnails.map(thumbnail => { - thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); - return thumbnail; - }); - subscriberCount = parseAbbreviatedNumber(getText(videoOwnerRenderer.subscriberCountText)); - verified = isVerified(videoOwnerRenderer.badges); - } catch (err) { - // Do nothing. - } - try { - let videoDetails = info.player_response.microformat && info.player_response.microformat.playerMicroformatRenderer; - let id = (videoDetails && videoDetails.channelId) || channelId || info.player_response.videoDetails.channelId; - let author = { - id, - name: videoDetails ? videoDetails.ownerChannelName : info.player_response.videoDetails.author, - user: videoDetails ? videoDetails.ownerProfileUrl.split('/').slice(-1)[0] : null, - channel_url: `https://www.youtube.com/channel/${id}`, - external_channel_url: videoDetails ? `https://www.youtube.com/channel/${videoDetails.externalChannelId}` : '', - user_url: videoDetails ? new URL(videoDetails.ownerProfileUrl, BASE_URL).toString() : '', - thumbnails, - verified, - subscriber_count: subscriberCount - }; - if (thumbnails.length) { - deprecate(author, 'avatar', author.thumbnails[0].url, 'author.avatar', 'author.thumbnails[0].url'); - } - return author; - } catch (err) { - return {}; - } -}; - -const parseRelatedVideo = (details, rvsParams) => { - if (!details) { - return; - } - try { - let viewCount = getText(details.viewCountText); - let shortViewCount = getText(details.shortViewCountText); - let rvsDetails = rvsParams.find(elem => elem.id === details.videoId); - if (!/^\d/.test(shortViewCount)) { - shortViewCount = (rvsDetails && rvsDetails.short_view_count_text) || ''; - } - viewCount = (/^\d/.test(viewCount) ? viewCount : shortViewCount).split(' ')[0]; - let browseEndpoint = details.shortBylineText.runs[0].navigationEndpoint.browseEndpoint; - let channelId = browseEndpoint.browseId; - let name = getText(details.shortBylineText); - let user = (browseEndpoint.canonicalBaseUrl || '').split('/').slice(-1)[0]; - let video = { - id: details.videoId, - title: getText(details.title), - published: getText(details.publishedTimeText), - author: { - id: channelId, - name, - user, - channel_url: `https://www.youtube.com/channel/${channelId}`, - user_url: `https://www.youtube.com/user/${user}`, - thumbnails: details.channelThumbnail.thumbnails.map(thumbnail => { - thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); - return thumbnail; - }), - verified: isVerified(details.ownerBadges), - - [Symbol.toPrimitive]() { - console.warn('`relatedVideo.author` will be removed in a near future release, ' + - 'use `relatedVideo.author.name` instead.'); - return video.author.name; - } - - }, - short_view_count_text: shortViewCount.split(' ')[0], - view_count: viewCount.replace(/,/g, ''), - length_seconds: details.lengthText ? - Math.floor(parseTimestamp(getText(details.lengthText)) / 1000) : - rvsParams && `${rvsParams.length_seconds}`, - thumbnails: details.thumbnail.thumbnails, - richThumbnails: - details.richThumbnail ? - details.richThumbnail.movingThumbnailRenderer.movingThumbnailDetails.thumbnails : [], - isLive: !!(details.badges && details.badges.find(b => b.metadataBadgeRenderer.label === 'LIVE NOW')) - }; - - deprecate(video, 'author_thumbnail', video.author.thumbnails[0].url, - 'relatedVideo.author_thumbnail', 'relatedVideo.author.thumbnails[0].url'); - deprecate(video, 'ucid', video.author.id, 'relatedVideo.ucid', 'relatedVideo.author.id'); - deprecate(video, 'video_thumbnail', video.thumbnails[0].url, - 'relatedVideo.video_thumbnail', 'relatedVideo.thumbnails[0].url'); - return video; - } catch (err) { - // Skip. - } -}; - -/** - * Get related videos. - * - * @param {Object} info - * @returns {Array.} - */ -export const getRelatedVideos = info => { - let rvsParams = [], secondaryResults = []; - try { - rvsParams = info.response.webWatchNextResponseExtensionData.relatedVideoArgs.split(',').map(e => qs.parse(e)); - } catch (err) { - // Do nothing. - } - try { - secondaryResults = info.response.contents.twoColumnWatchNextResults.secondaryResults.secondaryResults.results; - } catch (err) { - return []; - } - let videos = []; - for (let result of secondaryResults || []) { - let details = result.compactVideoRenderer; - if (details) { - let video = parseRelatedVideo(details, rvsParams); - if (video) { - videos.push(video); - } - } else { - let autoplay = result.compactAutoplayRenderer || result.itemSectionRenderer; - if (!autoplay || !Array.isArray(autoplay.contents)) { - continue; - } - for (let content of autoplay.contents) { - let video = parseRelatedVideo(content.compactVideoRenderer, rvsParams); - if (video) { - videos.push(video); - } - } - } - } - return videos; -}; - -/** - * Get like count. - * - * @param {Object} info - * @returns {number} - */ -export const getLikes = info => { - try { - let contents = info.response.contents.twoColumnWatchNextResults.results.results.contents; - let video = contents.find(r => r.videoPrimaryInfoRenderer); - let buttons = video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons; - let accessibilityText = buttons.find(b => b.segmentedLikeDislikeButtonViewModel).segmentedLikeDislikeButtonViewModel - .likeButtonViewModel.likeButtonViewModel.toggleButtonViewModel.toggleButtonViewModel - .defaultButtonViewModel.buttonViewModel.accessibilityText; - return parseInt(accessibilityText.match(/[\d,.]+/)[0].replace(/\D+/g, '')); - } catch (err) { - return null; - } -}; - -/** - * Cleans up a few fields on `videoDetails`. - * - * @param {Object} videoDetails - * @param {Object} info - * @returns {Object} - */ -export const cleanVideoDetails = (videoDetails, info) => { - videoDetails.thumbnails = videoDetails.thumbnail.thumbnails; - delete videoDetails.thumbnail; - deprecate(videoDetails, 'thumbnail', { thumbnails: videoDetails.thumbnails }, - 'videoDetails.thumbnail.thumbnails', 'videoDetails.thumbnails'); - videoDetails.description = videoDetails.shortDescription || getText(videoDetails.description); - delete videoDetails.shortDescription; - deprecate(videoDetails, 'shortDescription', videoDetails.description, - 'videoDetails.shortDescription', 'videoDetails.description'); - - // Use more reliable `lengthSeconds` from `playerMicroformatRenderer`. - videoDetails.lengthSeconds = - (info.player_response.microformat && - info.player_response.microformat.playerMicroformatRenderer.lengthSeconds) || - info.player_response.videoDetails.lengthSeconds; - return videoDetails; -}; - -/** - * Get storyboards info. - * - * @param {Object} info - * @returns {Array.} - */ -export const getStoryboards = info => { - const parts = info.player_response.storyboards && - info.player_response.storyboards.playerStoryboardSpecRenderer && - info.player_response.storyboards.playerStoryboardSpecRenderer.spec && - info.player_response.storyboards.playerStoryboardSpecRenderer.spec.split('|'); - - if (!parts) { - return []; - } - - const url = new URL(parts.shift()); - - return parts.map((part, i) => { - let [ - thumbnailWidth, - thumbnailHeight, - thumbnailCount, - columns, - rows, - interval, - nameReplacement, - sigh - ] = part.split('#'); - - url.searchParams.set('sigh', sigh); - - thumbnailCount = parseInt(thumbnailCount, 10); - columns = parseInt(columns, 10); - rows = parseInt(rows, 10); - - const storyboardCount = Math.ceil(thumbnailCount / (columns * rows)); - - return { - templateUrl: url.toString().replace('$L', i).replace('$N', nameReplacement), - thumbnailWidth: parseInt(thumbnailWidth, 10), - thumbnailHeight: parseInt(thumbnailHeight, 10), - thumbnailCount, - interval: parseInt(interval, 10), - columns, - rows, - storyboardCount - }; - }); -}; - -/** - * Get chapters info. - * - * @param {Object} info - * @returns {Array.} - */ -export const getChapters = info => { - const playerOverlayRenderer = info.response && - info.response.playerOverlays && - info.response.playerOverlays.playerOverlayRenderer; - const playerBar = playerOverlayRenderer && - playerOverlayRenderer.decoratedPlayerBarRenderer && - playerOverlayRenderer.decoratedPlayerBarRenderer.decoratedPlayerBarRenderer && - playerOverlayRenderer.decoratedPlayerBarRenderer.decoratedPlayerBarRenderer.playerBar; - const markersMap = playerBar && - playerBar.multiMarkersPlayerBarRenderer && - playerBar.multiMarkersPlayerBarRenderer.markersMap; - const marker = Array.isArray(markersMap) && markersMap.find(m => m.value && Array.isArray(m.value.chapters)); - if (!marker) { - return []; - } - const chapters = marker.value.chapters; - - return chapters.map(chapter => ({ - title: getText(chapter.chapterRenderer.title), - start_time: chapter.chapterRenderer.timeRangeStartMillis / 1000 - })); -}; diff --git a/packages/core/src/ytdl-core/info.js b/packages/core/src/ytdl-core/info.js deleted file mode 100644 index 6be504f5c0..0000000000 --- a/packages/core/src/ytdl-core/info.js +++ /dev/null @@ -1,442 +0,0 @@ -import sax from 'sax'; -import {applyDefaultAgent, applyDefaultHeaders, applyIPv6Rotations, applyOldLocalAddress, playError, requestUtil, between, cutAfterJS, tryParseBetween, saveDebugFile, setPropInsensitive, generateClientPlaybackNonce} from './utils'; -import {addFormatMeta, sortFormats} from './format-utils'; -// Forces Node JS version of setTimeout for Electron based applications -import { setTimeout } from 'timers'; -import {getVideoID} from './url-utils'; -import * as extras from './info-extras'; -import Cache from './cache'; -import { URL } from 'url'; - - -const BASE_URL = 'https://www.youtube.com/watch?v='; - - -// Cached for storing basic/full info. -export const cache = new Cache(); -export const watchPageCache = new Cache(); - - -// Special error class used to determine if an error is unrecoverable, -// as in, ytdl-core should not try again to fetch the video metadata. -// In this case, the video is usually unavailable in some way. -class UnrecoverableError extends Error {} - - -// List of URLs that show up in `notice_url` for age restricted videos. -const AGE_RESTRICTED_URLS = [ - 'support.google.com/youtube/?p=age_restrictions', - 'youtube.com/t/community_guidelines' -]; - -/** - * Gets info from a video without getting additional formats. - * - * @param {string} id - * @param {Object} options - * @returns {Promise} - */ -const getBasicInfo = async(id, options) => { - applyIPv6Rotations(options); - const retryOptions = Object.assign({}, options.requestOptions); - applyDefaultHeaders(options); - applyDefaultAgent(options); - applyOldLocalAddress(options); - const { jar, dispatcher } = options.agent; - setPropInsensitive( - options.requestOptions.headers, 'cookie', jar.getCookieStringSync('https://www.youtube.com') - ); - options.requestOptions.dispatcher = dispatcher; - const info = await retryFunc(getWatchHTMLPage, [id, options], retryOptions); - - const playErr = playError(info.player_response, ['ERROR'], UnrecoverableError); - if (playErr) { - throw playErr; - } - const privateErr = privateVideoError(info.player_response); - if (privateErr) { - throw privateErr; - } - - - Object.assign(info, { - // formats: parseFormats(info.player_response), - related_videos: extras.getRelatedVideos(info) - }); - - // Add additional properties to info. - const media = extras.getMedia(info); - const additional = { - author: extras.getAuthor(info), - media, - likes: extras.getLikes(info), - age_restricted: !!(media && AGE_RESTRICTED_URLS.some(url => - Object.values(media).some(v => typeof v === 'string' && v.includes(url))) - ), - - // Give the standard link to the video. - video_url: BASE_URL + id, - storyboards: extras.getStoryboards(info), - chapters: extras.getChapters(info) - }; - - info.videoDetails = extras.cleanVideoDetails(Object.assign({}, - info.player_response && info.player_response.microformat && - info.player_response.microformat.playerMicroformatRenderer, - info.player_response && info.player_response.videoDetails, additional), info); - - return info; -}; - -const privateVideoError = player_response => { - const playability = player_response && player_response.playabilityStatus; - if (!playability) { - return null; - } - if (playability.status === 'LOGIN_REQUIRED') { - return new UnrecoverableError(playability.reason || (playability.messages && playability.messages[0])); - } - if (playability.status === 'LIVE_STREAM_OFFLINE') { - return new UnrecoverableError(playability.reason || 'The live stream is offline.'); - } - if (playability.status === 'UNPLAYABLE') { - return new UnrecoverableError(playability.reason || 'This video is unavailable.'); - } - return null; -}; - -const getWatchHTMLURL = (id, options) => - `${BASE_URL + id}&hl=${options.lang || 'en'}&bpctr=${Math.ceil(Date.now() / 1000)}&has_verified=1`; -const getWatchHTMLPageBody = (id, options) => { - const url = getWatchHTMLURL(id, options); - return watchPageCache.getOrSet(url, () => requestUtil(url, options)); -}; - - -const EMBED_URL = 'https://www.youtube.com/embed/'; -const getEmbedPageBody = (id, options) => { - const embedUrl = `${EMBED_URL + id}?hl=${options.lang || 'en'}`; - return requestUtil(embedUrl, options); -}; - - -const getHTML5player = body => { - let html5playerRes = - /|"jsUrl":"([^"]+)"/ - .exec(body); - return html5playerRes ? html5playerRes[1] || html5playerRes[2] : null; -}; - -/** - * Given a function, calls it with `args` until it's successful, - * or until it encounters an unrecoverable error. - * Currently, any error from miniget is considered unrecoverable. Errors such as - * too many redirects, invalid URL, status code 404, status code 502. - * - * @param {Function} func - * @param {Array.} args - * @param {Object} options - * @param {number} options.maxRetries - * @param {Object} options.backoff - * @param {number} options.backoff.inc - */ -const retryFunc = async(func, args, options) => { - let currentTry = 0, result; - if (!options.maxRetries) { - options.maxRetries = 3; - } - if (!options.backoff) { - options.backoff = { inc: 500, max: 5000 }; - } - while (currentTry <= options.maxRetries) { - try { - result = await func(...args); - break; - } catch (err) { - if ((err && err.statusCode < 500) || currentTry >= options.maxRetries) { - throw err; - } - let wait = Math.min(++currentTry * options.backoff.inc, options.backoff.max); - await new Promise(resolve => setTimeout(resolve, wait)); - } - } - return result; -}; - - -const jsonClosingChars = /^[)\]}'\s]+/; -const parseJSON = (source, varName, json) => { - if (!json || typeof json === 'object') { - return json; - } else { - try { - json = json.replace(jsonClosingChars, ''); - return JSON.parse(json); - } catch (err) { - throw Error(`Error parsing ${varName} in ${source}: ${err.message}`); - } - } -}; - - -const findJSON = (source, varName, body, left, right, prependJSON) => { - let jsonStr = between(body, left, right); - if (!jsonStr) { - throw Error(`Could not find ${varName} in ${source}`); - } - return parseJSON(source, varName, cutAfterJS(`${prependJSON}${jsonStr}`)); -}; - - -const findPlayerResponse = (source, info) => { - const player_response = info && ( - (info.args && info.args.player_response) || - info.player_response || info.playerResponse || info.embedded_player_response); - return parseJSON(source, 'player_response', player_response); -}; - -const getWatchHTMLPage = async(id, options) => { - let body = await getWatchHTMLPageBody(id, options); - let info = { page: 'watch' }; - try { - try { - info.player_response = - tryParseBetween(body, 'var ytInitialPlayerResponse = ', '}};', '', '}}') || - tryParseBetween(body, 'var ytInitialPlayerResponse = ', ';var') || - tryParseBetween(body, 'var ytInitialPlayerResponse = ', ';') || - findJSON('watch.html', 'player_response', body, /\bytInitialPlayerResponse\s*=\s*\{/i, '', '{'); - } catch (_e) { - let args = findJSON('watch.html', 'player_response', body, /\bytplayer\.config\s*=\s*{/, '', '{'); - info.player_response = findPlayerResponse('watch.html', args); - } - - info.response = - tryParseBetween(body, 'var ytInitialData = ', '}};', '', '}}') || - tryParseBetween(body, 'var ytInitialData = ', ';') || - tryParseBetween(body, 'window["ytInitialData"] = ', '}};', '', '}}') || - tryParseBetween(body, 'window["ytInitialData"] = ', ';') || - findJSON('watch.html', 'response', body, /\bytInitialData("\])?\s*=\s*\{/i, '', '{'); - info.html5player = getHTML5player(body); - } catch (_) { - throw Error( - 'Error when parsing watch.html, maybe YouTube made a change.\n' + - `Please report this issue with the "${ - saveDebugFile('watch.html', body) - }" file on https://github.com/distubejs/ytdl-core/issues.` - ); - } - return info; -}; - -/** - * @param {Object} player_response - * @returns {Array.} - */ -const parseFormats = player_response => { - let formats = []; - if (player_response && player_response.streamingData) { - formats = formats - .concat(player_response.streamingData.formats || []) - .concat(player_response.streamingData.adaptiveFormats || []); - } - return formats; -}; - - -/** - * Gets info from a video additional formats and deciphered URLs. - * - * @param {string} id - * @param {Object} options - * @returns {Promise} - */ -const getInfo = async(id, options) => { - const info = await getBasicInfo(id, options); - const iosPlayerResponse = await fetchIosJsonPlayer(id, options); - info.formats = parseFormats(iosPlayerResponse); - const hasManifest = - iosPlayerResponse && iosPlayerResponse.streamingData && ( - iosPlayerResponse.streamingData.dashManifestUrl || - iosPlayerResponse.streamingData.hlsManifestUrl - ); - let funcs = []; - if (info.formats.length) { - // Stream from ios player doesn't need to be deciphered. - // info.html5player = info.html5player || - // getHTML5player(await getWatchHTMLPageBody(id, options)) || getHTML5player(await getEmbedPageBody(id, options)); - // if (!info.html5player) { - // throw Error('Unable to find html5player file'); - // } - // const html5player = new URL(info.html5player, BASE_URL).toString(); - // funcs.push(sig.decipherFormats(info.formats, html5player, options)); - funcs.push(info.formats); - } - if (hasManifest && iosPlayerResponse.streamingData.dashManifestUrl) { - let url = iosPlayerResponse.streamingData.dashManifestUrl; - funcs.push(getDashManifest(url, options)); - } - if (hasManifest && iosPlayerResponse.streamingData.hlsManifestUrl) { - let url = iosPlayerResponse.streamingData.hlsManifestUrl; - funcs.push(getM3U8(url, options)); - } - - let results = await Promise.all(funcs); - info.formats = Object.values(Object.assign({}, ...results)); - info.formats = info.formats.map(addFormatMeta); - info.formats.sort(sortFormats); - - - info.full = true; - return info; -}; - -// TODO: Clean up this code to impliment Android player. -const IOS_CLIENT_VERSION = '19.28.1', - IOS_DEVICE_MODEL = 'iPhone16,2', - IOS_USER_AGENT_VERSION = '17_5_1', - IOS_OS_VERSION = '17.5.1.21F90'; - -const fetchIosJsonPlayer = async(videoId, options) => { - const cpn = generateClientPlaybackNonce(16); - const payload = { - videoId, - cpn, - contentCheckOk: true, - racyCheckOk: true, - context: { - client: { - clientName: 'IOS', - clientVersion: IOS_CLIENT_VERSION, - deviceMake: 'Apple', - deviceModel: IOS_DEVICE_MODEL, - platform: 'MOBILE', - osName: 'iOS', - osVersion: IOS_OS_VERSION, - hl: 'en', - gl: 'US', - utcOffsetMinutes: -240 - }, - request: { - internalExperimentFlags: [], - useSsl: true - }, - user: { - lockedSafetyMode: false - } - } - }; - - const { jar, dispatcher } = options.agent; - const opts = { - requestOptions: { - method: 'POST', - dispatcher, - query: { - prettyPrint: false, - t: generateClientPlaybackNonce(12), - id: videoId - }, - headers: { - 'Content-Type': 'application/json', - cookie: jar.getCookieStringSync('https://www.youtube.com'), - 'User-Agent': `com.google.ios.youtube/${IOS_CLIENT_VERSION}(${ - IOS_DEVICE_MODEL - }; U; CPU iOS ${IOS_USER_AGENT_VERSION} like Mac OS X; en_US)`, - 'X-Goog-Api-Format-Version': '2' - }, - body: JSON.stringify(payload) - } - }; - const response = await requestUtil('https://youtubei.googleapis.com/youtubei/v1/player', opts); - if (videoId !== response.videoDetails.videoId) { - throw Error('Video ID mismatch'); - } - return response; -}; - - -/** - * Gets additional DASH formats. - * - * @param {string} url - * @param {Object} options - * @returns {Promise>} - */ -const getDashManifest = (url, options) => new Promise((resolve, reject) => { - let formats = {}; - const parser = sax.parser(false); - parser.onerror = reject; - let adaptationSet; - parser.onopentag = node => { - if (node.name === 'ADAPTATIONSET') { - adaptationSet = node.attributes; - } else if (node.name === 'REPRESENTATION') { - const itag = parseInt(node.attributes.ID); - if (!isNaN(itag)) { - formats[url] = Object.assign({ - itag, - url, - bitrate: parseInt(node.attributes.BANDWIDTH), - mimeType: `${adaptationSet.MIMETYPE}; codecs="${node.attributes.CODECS}"` - }, node.attributes.HEIGHT ? { - width: parseInt(node.attributes.WIDTH), - height: parseInt(node.attributes.HEIGHT), - fps: parseInt(node.attributes.FRAMERATE) - } : { - audioSampleRate: node.attributes.AUDIOSAMPLINGRATE - }); - } - } - }; - parser.onend = () => { - resolve(formats); - }; - requestUtil(new URL(url, BASE_URL).toString(), options).then(res => { - parser.write(res); - parser.close(); - }).catch(reject); -}); - - -/** - * Gets additional formats. - * - * @param {string} url - * @param {Object} options - * @returns {Promise>} - */ -const getM3U8 = async(url, options) => { - url = new URL(url, BASE_URL); - const body = await requestUtil(url.toString(), options); - let formats = {}; - body - .split('\n') - .filter(line => /^https?:\/\//.test(line)) - .forEach(line => { - const itag = parseInt(line.match(/\/itag\/(\d+)\//)[1]); - formats[line] = { itag, url: line }; - }); - return formats; -}; - - -// // Cache get info functions. -// // In case a user wants to get a video's info before downloading. -const cachedGetBasicInfo = async (link, options = {}) => { - let id = await getVideoID(link); - const key = ['getBasicInfo', id, options.lang].join('-'); - return cache.getOrSet(key, () => getBasicInfo(id, options)); -}; - -const cachedGetInfo = async (link, options = {}) => { - let id = await getVideoID(link); - const key = ['getInfo', id, options.lang].join('-'); - return cache.getOrSet(key, () => getInfo(id, options)); -}; - -// Export the functions -export { cachedGetBasicInfo as getBasicInfo, cachedGetInfo as getInfo }; - - -// Export a few helpers. -export {validateID, validateURL, getURLVideoID, getVideoID} from './url-utils'; diff --git a/packages/core/src/ytdl-core/sig.js b/packages/core/src/ytdl-core/sig.js deleted file mode 100644 index d7b351ab90..0000000000 --- a/packages/core/src/ytdl-core/sig.js +++ /dev/null @@ -1,296 +0,0 @@ -import querystring from 'querystring'; -import Cache from './cache'; -import {requestUtil, saveDebugFile} from './utils'; -import vm from 'vm'; -import { URL } from 'url'; - -// A shared cache to keep track of html5player js functions. -export const cache = new Cache(1); - -/** - * Extract signature deciphering and n parameter transform functions from html5player file. - * - * @param {string} html5playerfile - * @param {Object} options - * @returns {Promise>} - */ -export const getFunctions = (html5playerfile, options) => cache.getOrSet(html5playerfile, async() => { - const body = await requestUtil(html5playerfile, options); - const functions = extractFunctions(body); - cache.set(html5playerfile, functions); - return functions; -}); - -// NewPipeExtractor regexps -const DECIPHER_NAME_REGEXPS = [ - '\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)', - '\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)', - // eslint-disable-next-line max-len - '(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*""\\s*\\)', - '([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(""\\)\\s*;' -]; - -const SCVR = '[a-zA-Z0-9$_]'; -const FNR = `${SCVR}+`; -const AAR = '\\[(\\d+)]'; -const N_TRANSFORM_NAME_REGEXPS = [ - // NewPipeExtractor regexps - `${SCVR}+="nn"\\[\\+${ - SCVR}+\\.${SCVR}+],${ - SCVR}+=${SCVR - }+\\.get\\(${SCVR}+\\)\\)&&\\(${ - SCVR}+=(${SCVR - }+)\\[(\\d+)]`, - `${SCVR}+="nn"\\[\\+${ - SCVR}+\\.${SCVR}+],${ - SCVR}+=${SCVR}+\\.get\\(${ - SCVR}+\\)\\).+\\|\\|(${SCVR - }+)\\(""\\)`, - `\\(${SCVR}=String\\.fromCharCode\\(110\\),${ - SCVR}=${SCVR}\\.get\\(${ - SCVR}\\)\\)&&\\(${SCVR - }=(${FNR})(?:${AAR})?\\(${ - SCVR}\\)`, - `\\.get\\("n"\\)\\)&&\\(${SCVR - }=(${FNR})(?:${AAR})?\\(${ - SCVR}\\)`, - // Skick regexps - '(\\w+).length\\|\\|\\w+\\(""\\)', - '\\w+.length\\|\\|(\\w+)\\(""\\)' -]; - -// LavaPlayer regexps -const VARIABLE_PART = '[a-zA-Z_\\$][a-zA-Z_0-9]*'; -const VARIABLE_PART_DEFINE = `\\"?${VARIABLE_PART}\\"?`; -const BEFORE_ACCESS = '(?:\\[\\"|\\.)'; -const AFTER_ACCESS = '(?:\\"\\]|)'; -const VARIABLE_PART_ACCESS = BEFORE_ACCESS + VARIABLE_PART + AFTER_ACCESS; -const REVERSE_PART = ':function\\(a\\)\\{(?:return )?a\\.reverse\\(\\)\\}'; -const SLICE_PART = ':function\\(a,b\\)\\{return a\\.slice\\(b\\)\\}'; -const SPLICE_PART = ':function\\(a,b\\)\\{a\\.splice\\(0,b\\)\\}'; -const SWAP_PART = ':function\\(a,b\\)\\{' + - 'var c=a\\[0\\];a\\[0\\]=a\\[b%a\\.length\\];a\\[b(?:%a.length|)\\]=c(?:;return a)?\\}'; - -const DECIPHER_REGEXP = `function(?: ${VARIABLE_PART})?\\(a\\)\\{` + - 'a=a\\.split\\(""\\);\\s*' + - `((?:(?:a=)?${VARIABLE_PART}${VARIABLE_PART_ACCESS}\\(a,\\d+\\);)+)` + - 'return a\\.join\\(""\\)' + - '\\}'; - -const HELPER_REGEXP = `var (${VARIABLE_PART})=\\{((?:(?:${ - VARIABLE_PART_DEFINE}${REVERSE_PART}|${ - VARIABLE_PART_DEFINE}${SLICE_PART}|${ - VARIABLE_PART_DEFINE}${SPLICE_PART}|${ - VARIABLE_PART_DEFINE}${SWAP_PART}),?\\n?)+)\\};`; - -const N_TRANSFORM_REGEXP = 'function\\(\\s*(\\w+)\\s*\\)\\s*\\{' + - 'var\\s*(\\w+)=(?:\\1\\.split\\(""\\)|String\\.prototype\\.split\\.call\\(\\1,""\\)),' + - '\\s*(\\w+)=(\\[.*?]);\\s*\\3\\[\\d+]' + - '(.*?try)(\\{.*?})catch\\(\\s*(\\w+)\\s*\\)\\s*\\' + - '{\\s*return"enhanced_except_([A-z0-9-]+)"\\s*\\+\\s*\\1\\s*}' + - '\\s*return\\s*(\\2\\.join\\(""\\)|Array\\.prototype\\.join\\.call\\(\\2,""\\))};'; - -const DECIPHER_ARGUMENT = 'sig'; -const N_ARGUMENT = 'ncode'; - -const matchRegex = (regex, str) => { - const match = str.match(new RegExp(regex, 's')); - if (!match) { - throw new Error(`Could not match ${regex}`); - } - return match; -}; - -const matchFirst = (regex, str) => matchRegex(regex, str)[0]; - -const matchGroup1 = (regex, str) => matchRegex(regex, str)[1]; - -const getFuncName = (body, regexps) => { - let fn; - for (const regex of regexps) { - try { - fn = matchGroup1(regex, body); - try { - fn = matchGroup1(`${fn.replace(/\$/g, '\\$')}=\\[([a-zA-Z0-9$\\[\\]]{2,})\\]`, body); - } catch (err) { - // Function name is not inside an array - } - break; - } catch (err) { - continue; - } - } - if (!fn || fn.includes('[')) { - throw Error(); - } - return fn; -}; - -const DECIPHER_FUNC_NAME = 'DisTubeDecipherFunc'; -const extractDecipherFunc = body => { - try { - const helperObject = matchFirst(HELPER_REGEXP, body); - const decipherFunc = matchFirst(DECIPHER_REGEXP, body); - const resultFunc = `var ${DECIPHER_FUNC_NAME}=${decipherFunc};`; - const callerFunc = `${DECIPHER_FUNC_NAME}(${DECIPHER_ARGUMENT});`; - return helperObject + resultFunc + callerFunc; - } catch (e) { - return null; - } -}; - -const extractDecipherWithName = body => { - try { - const decipherFuncName = getFuncName(body, DECIPHER_NAME_REGEXPS); - const funcPattern = `(${decipherFuncName.replace(/\$/g, '\\$')}=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})`; - const decipherFunc = `var ${matchGroup1(funcPattern, body)};`; - const helperObjectName = matchGroup1(';([A-Za-z0-9_\\$]{2,})\\.\\w+\\(', decipherFunc); - const helperPattern = `(var ${helperObjectName.replace(/\$/g, '\\$')}=\\{[\\s\\S]+?\\}\\};)`; - const helperObject = matchGroup1(helperPattern, body); - const callerFunc = `${decipherFuncName}(${DECIPHER_ARGUMENT});`; - return helperObject + decipherFunc + callerFunc; - } catch (e) { - return null; - } -}; - -const getExtractFunctions = (extractFunctions, body) => { - for (const extractFunction of extractFunctions) { - try { - const func = extractFunction(body); - if (!func) { - continue; - } - return new vm.Script(func); - } catch (err) { - continue; - } - } - return null; -}; - -let decipherWarning = false; -// This is required function to get the stream url, but we can continue if user doesn't need stream url. -const extractDecipher = body => { - // Faster: extractDecipherWithName - const decipherFunc = getExtractFunctions([extractDecipherWithName, extractDecipherFunc], body); - if (!decipherFunc && !decipherWarning) { - console.warn('\x1b[33mWARNING:\x1B[0m Could not parse decipher function.\n' + - `Please report this issue with the "${ - saveDebugFile('base.js', body) - }" file on https://github.com/distubejs/ytdl-core/issues.\nStream URL will be missing.`); - decipherWarning = true; - } - return decipherFunc; -}; - -const N_TRANSFORM_FUNC_NAME = 'DisTubeNTransformFunc'; -const extractNTransformFunc = body => { - try { - const nFunc = matchFirst(N_TRANSFORM_REGEXP, body); - const resultFunc = `var ${N_TRANSFORM_FUNC_NAME}=${nFunc}`; - const callerFunc = `${N_TRANSFORM_FUNC_NAME}(${N_ARGUMENT});`; - return resultFunc + callerFunc; - } catch (e) { - return null; - } -}; - -const extractNTransformWithName = body => { - try { - const nFuncName = getFuncName(body, N_TRANSFORM_NAME_REGEXPS); - const funcPattern = `(${ - nFuncName.replace(/\$/g, '\\$') - // eslint-disable-next-line max-len - }=\\s*function([\\S\\s]*?\\}\\s*return (([\\w$]+?\\.join\\(""\\))|(Array\\.prototype\\.join\\.call\\([\\w$]+?,[\\n\\s]*(("")|(\\("",""\\)))\\)))\\s*\\}))`; - const nTransformFunc = `var ${matchGroup1(funcPattern, body)};`; - const callerFunc = `${nFuncName}(${N_ARGUMENT});`; - return nTransformFunc + callerFunc; - } catch (e) { - return null; - } -}; - -let nTransformWarning = false; -const extractNTransform = body => { - // Faster: extractNTransformFunc - const nTransformFunc = getExtractFunctions([extractNTransformFunc, extractNTransformWithName], body); - if (!nTransformFunc && !nTransformWarning) { - // This is optional, so we can continue if it's not found, but it will bottleneck the download. - console.warn('\x1b[33mWARNING:\x1B[0m Could not parse n transform function.\n' + - `Please report this issue with the "${ - saveDebugFile('base.js', body) - }" file on https://github.com/distubejs/ytdl-core/issues.`); - nTransformWarning = true; - } - return nTransformFunc; -}; - -/** - * Extracts the actions that should be taken to decipher a signature - * and transform the n parameter - * - * @param {string} body - * @returns {Array.} - */ -export const extractFunctions = body => [ - extractDecipher(body), - extractNTransform(body) -]; - -/** - * Apply decipher and n-transform to individual format - * - * @param {Object} format - * @param {vm.Script} decipherScript - * @param {vm.Script} nTransformScript - */ -export const setDownloadURL = (format, decipherScript, nTransformScript) => { - if (!decipherScript) { - return; - } - const decipher = url => { - const args = querystring.parse(url); - if (!args.s) { - return args.url; - } - const components = new URL(decodeURIComponent(args.url)); - const context = {}; - context[DECIPHER_ARGUMENT] = decodeURIComponent(args.s); - components.searchParams.set(args.sp || 'sig', decipherScript.runInNewContext(context)); - return components.toString(); - }; - const nTransform = url => { - const components = new URL(decodeURIComponent(url)); - const n = components.searchParams.get('n'); - if (!n || !nTransformScript) { - return url; - } - const context = {}; - context[N_ARGUMENT] = n; - components.searchParams.set('n', nTransformScript.runInNewContext(context)); - return components.toString(); - }; - const cipher = !format.url; - const url = format.url || format.signatureCipher || format.cipher; - format.url = nTransform(cipher ? decipher(url) : url); - delete format.signatureCipher; - delete format.cipher; -}; - -/** - * Applies decipher and n parameter transforms to all format URL's. - * - * @param {Array.} formats - * @param {string} html5player - * @param {Object} options - */ -export const decipherFormats = async(formats, html5player, options) => { - const decipheredFormats = {}; - const [decipherScript, nTransformScript] = await getFunctions(html5player, options); - formats.forEach(format => { - setDownloadURL(format, decipherScript, nTransformScript); - decipheredFormats[format.url] = format; - }); - return decipheredFormats; -}; diff --git a/packages/core/src/ytdl-core/url-utils.js b/packages/core/src/ytdl-core/url-utils.js deleted file mode 100644 index 3ec66a6bff..0000000000 --- a/packages/core/src/ytdl-core/url-utils.js +++ /dev/null @@ -1,92 +0,0 @@ -import { URL } from 'url'; -/** - * Get video ID. - * - * There are a few type of video URL formats. - * - https://www.youtube.com/watch?v=VIDEO_ID - * - https://m.youtube.com/watch?v=VIDEO_ID - * - https://youtu.be/VIDEO_ID - * - https://www.youtube.com/v/VIDEO_ID - * - https://www.youtube.com/embed/VIDEO_ID - * - https://music.youtube.com/watch?v=VIDEO_ID - * - https://gaming.youtube.com/watch?v=VIDEO_ID - * - * @param {string} link - * @return {string} - * @throws {Error} If unable to find a id - * @throws {TypeError} If videoid doesn't match specs - */ -const validQueryDomains = new Set([ - 'youtube.com', - 'www.youtube.com', - 'm.youtube.com', - 'music.youtube.com', - 'gaming.youtube.com' -]); -const validPathDomains = /^https?:\/\/(youtu\.be\/|(www\.)?youtube\.com\/(embed|v|shorts|live)\/)/; -export const getURLVideoID = link => { - const parsed = new URL(link.trim()); - let id = parsed.searchParams.get('v'); - if (validPathDomains.test(link.trim()) && !id) { - const paths = parsed.pathname.split('/'); - id = parsed.host === 'youtu.be' ? paths[1] : paths[2]; - } else if (parsed.hostname && !validQueryDomains.has(parsed.hostname)) { - throw Error('Not a YouTube domain'); - } - if (!id) { - throw Error(`No video id found: "${link}"`); - } - id = id.substring(0, 11); - if (!validateID(id)) { - throw TypeError(`Video id (${id}) does not match expected ` + - `format (${idRegex.toString()})`); - } - return id; -}; - - -/** - * Gets video ID either from a url or by checking if the given string - * matches the video ID format. - * - * @param {string} str - * @returns {string} - * @throws {Error} If unable to find a id - * @throws {TypeError} If videoid doesn't match specs - */ -const urlRegex = /^https?:\/\//; -export const getVideoID = str => { - if (validateID(str)) { - return str; - } else if (urlRegex.test(str.trim())) { - return getURLVideoID(str); - } else { - throw Error(`No video id found: ${str}`); - } -}; - - -/** - * Returns true if given id satifies YouTube's id format. - * - * @param {string} id - * @return {boolean} - */ -const idRegex = /^[a-zA-Z0-9-_]{11}$/; -export const validateID = id => idRegex.test(id.trim()); - - -/** - * Checks wether the input string includes a valid id. - * - * @param {string} string - * @returns {boolean} - */ -export const validateURL = string => { - try { - getURLVideoID(string); - return true; - } catch (e) { - return false; - } -}; diff --git a/packages/core/src/ytdl-core/utils.js b/packages/core/src/ytdl-core/utils.js deleted file mode 100644 index 8dc48482fa..0000000000 --- a/packages/core/src/ytdl-core/utils.js +++ /dev/null @@ -1,389 +0,0 @@ -import {request} from 'undici'; -import { writeFileSync } from 'fs'; -import {defaultAgent, createAgent, addCookiesFromString} from './agent'; - - -/** - * Extract string inbetween another. - * - * @param {string} haystack - * @param {string} left - * @param {string} right - * @returns {string} - */ -export const between = (haystack, left, right) => { - let pos; - if (left instanceof RegExp) { - const match = haystack.match(left); - if (!match) { - return ''; - } - pos = match.index + match[0].length; - } else { - pos = haystack.indexOf(left); - if (pos === -1) { - return ''; - } - pos += left.length; - } - haystack = haystack.slice(pos); - pos = haystack.indexOf(right); - if (pos === -1) { - return ''; - } - haystack = haystack.slice(0, pos); - return haystack; -}; - -export const tryParseBetween = (body, left, right, prepend = '', append = '') => { - try { - let data = between(body, left, right); - if (!data) { - return null; - } - return JSON.parse(`${prepend}${data}${append}`); - } catch (e) { - return null; - } -}; - -/** - * Get a number from an abbreviated number string. - * - * @param {string} string - * @returns {number} - */ -export const parseAbbreviatedNumber = string => { - const match = string - .replace(',', '.') - .replace(' ', '') - .match(/([\d,.]+)([MK]?)/); - if (match) { - let [, num, multi] = match; - num = parseFloat(num); - return Math.round(multi === 'M' ? num * 1000000 : - multi === 'K' ? num * 1000 : num); - } - return null; -}; - -/** - * Escape sequences for cutAfterJS - * @param {string} start the character string the escape sequence - * @param {string} end the character string to stop the escape seequence - * @param {undefined|Regex} startPrefix a regex to check against the preceding 10 characters - */ -const ESCAPING_SEQUENZES = [ - // Strings - { start: '"', end: '"' }, - { start: '\'', end: '\'' }, - { start: '`', end: '`' }, - // RegeEx - { start: '/', end: '/', startPrefix: /(^|[[{:;,/])\s?$/ } -]; - -/** - * Match begin and end braces of input JS, return only JS - * - * @param {string} mixedJson - * @returns {string} -*/ -export const cutAfterJS = mixedJson => { - // Define the general open and closing tag - let open, close; - if (mixedJson[0] === '[') { - open = '['; - close = ']'; - } else if (mixedJson[0] === '{') { - open = '{'; - close = '}'; - } - - if (!open) { - throw new Error(`Can't cut unsupported JSON (need to begin with [ or { ) but got: ${mixedJson[0]}`); - } - - // States if the loop is currently inside an escaped js object - let isEscapedObject = null; - - // States if the current character is treated as escaped or not - let isEscaped = false; - - // Current open brackets to be closed - let counter = 0; - - let i; - // Go through all characters from the start - for (i = 0; i < mixedJson.length; i++) { - // End of current escaped object - if (!isEscaped && isEscapedObject !== null && mixedJson[i] === isEscapedObject.end) { - isEscapedObject = null; - continue; - // Might be the start of a new escaped object - } else if (!isEscaped && isEscapedObject === null) { - for (const escaped of ESCAPING_SEQUENZES) { - if (mixedJson[i] !== escaped.start) { - continue; - } - // Test startPrefix against last 10 characters - if (!escaped.startPrefix || mixedJson.substring(i - 10, i).match(escaped.startPrefix)) { - isEscapedObject = escaped; - break; - } - } - // Continue if we found a new escaped object - if (isEscapedObject !== null) { - continue; - } - } - - // Toggle the isEscaped boolean for every backslash - // Reset for every regular character - isEscaped = mixedJson[i] === '\\' && !isEscaped; - - if (isEscapedObject !== null) { - continue; - } - - if (mixedJson[i] === open) { - counter++; - } else if (mixedJson[i] === close) { - counter--; - } - - // All brackets have been closed, thus end of JSON is reached - if (counter === 0) { - // Return the cut JSON - return mixedJson.substring(0, i + 1); - } - } - - // We ran through the whole string and ended up with an unclosed bracket - throw Error('Can\'t cut unsupported JSON (no matching closing bracket found)'); -}; - -/** - * Checks if there is a playability error. - * - * @param {Object} player_response - * @param {Array.} statuses - * @param {Error} ErrorType - * @returns {!Error} - */ -export const playError = (player_response, statuses, ErrorType = Error) => { - let playability = player_response && player_response.playabilityStatus; - if (playability && statuses.includes(playability.status)) { - return new ErrorType(playability.reason || (playability.messages && playability.messages[0])); - } - return null; -}; - -// Undici request -export const requestUtil = async(url, options = {}) => { - const { requestOptions } = options; - const req = await request(url, requestOptions); - const code = req.statusCode.toString(); - if (code.startsWith('2')) { - if (req.headers['content-type'].includes('application/json')) { - return req.body.json(); - } - return req.body.text(); - } - if (code.startsWith('3')) { - return requestUtil(req.headers.location, options); - } - const e = new Error(`Status code: ${code}`); - e.statusCode = req.statusCode; - throw e; -}; - -/** - * Temporary helper to help deprecating a few properties. - * - * @param {Object} obj - * @param {string} prop - * @param {Object} value - * @param {string} oldPath - * @param {string} newPath - */ -export const deprecate = (obj, prop, value, oldPath, newPath) => { - Object.defineProperty(obj, prop, { - get: () => { - console.warn(`\`${oldPath}\` will be removed in a near future release, ` + - `use \`${newPath}\` instead.`); - return value; - } - }); -}; - -/** - * Gets random IPv6 Address from a block - * - * @param {string} ip the IPv6 block in CIDR-Notation - * @returns {string} - */ -export const getRandomIPv6 = ip => { - // Start with a fast Regex-Check - if (!isIPv6(ip)) { - throw Error('Invalid IPv6 format'); - } - // Start by splitting and normalizing addr and mask - const [rawAddr, rawMask] = ip.split('/'); - let base10Mask = parseInt(rawMask); - if (!base10Mask || base10Mask > 128 || base10Mask < 24) { - throw Error('Invalid IPv6 subnet'); - } - const base10addr = normalizeIP(rawAddr); - // Get random addr to pad with - // using Math.random since we're not requiring high level of randomness - const randomAddr = new Array(8).fill(1).map(() => Math.floor(Math.random() * 0xffff)); - - // Merge base10addr with randomAddr - const mergedAddr = randomAddr.map((randomItem, idx) => { - // Calculate the amount of static bits - const staticBits = Math.min(base10Mask, 16); - // Adjust the bitmask with the staticBits - base10Mask -= staticBits; - // Calculate the bitmask - // lsb makes the calculation way more complicated - const mask = 0xffff - ((2 ** (16 - staticBits)) - 1); - // Combine base10addr and random - return (base10addr[idx] & mask) + (randomItem & (mask ^ 0xffff)); - }); - // Return new addr - return mergedAddr.map(x => x.toString('16')).join(':'); -}; - -// eslint-disable-next-line max-len -const IPV6_REGEX = /^(([0-9a-f]{1,4}:)(:[0-9a-f]{1,4}){1,6}|([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}|([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}|([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}|([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}|([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4})|([0-9a-f]{1,4}:){1,7}(([0-9a-f]{1,4})|:))\/(1[0-1]\d|12[0-8]|\d{1,2})$/; -/** - * Quick check for a valid IPv6 - * The Regex only accepts a subset of all IPv6 Addresses - * - * @param {string} ip the IPv6 block in CIDR-Notation to test - * @returns {boolean} true if valid - */ -export const isIPv6 = ip => IPV6_REGEX.test(ip); - -/** - * Normalise an IP Address - * - * @param {string} ip the IPv6 Addr - * @returns {number[]} the 8 parts of the IPv6 as Integers - */ -export const normalizeIP = ip => { - // Split by fill position - const parts = ip.split('::').map(x => x.split(':')); - // Normalize start and end - const partStart = parts[0] || []; - const partEnd = parts[1] || []; - partEnd.reverse(); - // Placeholder for full ip - const fullIP = new Array(8).fill(0); - // Fill in start and end parts - for (let i = 0; i < Math.min(partStart.length, 8); i++) { - fullIP[i] = parseInt(partStart[i], 16) || 0; - } - for (let i = 0; i < Math.min(partEnd.length, 8); i++) { - fullIP[7 - i] = parseInt(partEnd[i], 16) || 0; - } - return fullIP; -}; - -export const saveDebugFile = (name, body) => { - const filename = `${+new Date()}-${name}`; - writeFileSync(filename, body); - return filename; -}; - -const findPropKeyInsensitive = (obj, prop) => - Object.keys(obj).find(p => p.toLowerCase() === prop.toLowerCase()) || null; - -export const getPropInsensitive = (obj, prop) => { - const key = findPropKeyInsensitive(obj, prop); - return key && obj[key]; -}; - -export const setPropInsensitive = (obj, prop, value) => { - const key = findPropKeyInsensitive(obj, prop); - obj[key || prop] = value; - return key; -}; - -let oldCookieWarning = true; -let oldDispatcherWarning = true; -export const applyDefaultAgent = options => { - if (!options.agent) { - const { jar } = defaultAgent; - const c = getPropInsensitive(options.requestOptions.headers, 'cookie'); - if (c) { - jar.removeAllCookiesSync(); - addCookiesFromString(jar, c); - if (oldCookieWarning) { - oldCookieWarning = false; - console.warn( - '\x1b[33mWARNING:\x1B[0m Using old cookie format, ' + - 'please use the new one instead. (https://github.com/distubejs/ytdl-core#cookies-support)' - ); - } - } - if (options.requestOptions.dispatcher && oldDispatcherWarning) { - oldDispatcherWarning = false; - console.warn( - '\x1b[33mWARNING:\x1B[0m Your dispatcher is overridden by `ytdl.Agent`. ' + - 'To implement your own, check out the documentation. ' + - '(https://github.com/distubejs/ytdl-core#how-to-implement-ytdlagent-with-your-own-dispatcher)' - ); - } - options.agent = defaultAgent; - } -}; - -let oldLocalAddressWarning = true; -export const applyOldLocalAddress = options => { - if ( - !options.requestOptions || - !options.requestOptions.localAddress || - options.requestOptions.localAddress === options.agent.localAddress - ) { - return; - } - options.agent = createAgent(undefined, { localAddress: options.requestOptions.localAddress }); - if (oldLocalAddressWarning) { - oldLocalAddressWarning = false; - console.warn( - '\x1b[33mWARNING:\x1B[0m Using old localAddress option, ' + - 'please add it to the agent options instead. (https://github.com/distubejs/ytdl-core#ip-rotation)' - ); - } -}; - -let oldIpRotationsWarning = true; -export const applyIPv6Rotations = options => { - if (options.IPv6Block) { - options.requestOptions = Object.assign({}, options.requestOptions, { - localAddress: getRandomIPv6(options.IPv6Block) - }); - if (oldIpRotationsWarning) { - oldIpRotationsWarning = false; - oldLocalAddressWarning = false; - console.warn( - '\x1b[33mWARNING:\x1B[0m IPv6Block option is deprecated, ' + - 'please create your own ip rotation instead. (https://github.com/distubejs/ytdl-core#ip-rotation)' - ); - } - } -}; - -export const applyDefaultHeaders = options => { - options.requestOptions = Object.assign({}, options.requestOptions); - options.requestOptions.headers = Object.assign({}, { - // eslint-disable-next-line max-len - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Safari/537.36' - }, options.requestOptions.headers); -}; - -export const generateClientPlaybackNonce = length => { - const CPN_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; - return Array.from({ length }, () => CPN_CHARS[Math.floor(Math.random() * CPN_CHARS.length)]).join(''); -};