diff --git a/Dockerfile b/Dockerfile
index 013446e..99ae491 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,12 @@
-FROM alpine
+FROM buildkite/puppeteer
+
+RUN apt update && \
+ apt install -y \
+ ffmpeg && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g spotify-dl --unsafe-perm
-RUN apk add \
- npm \
- ffmpeg
-RUN npm install -g spotify-dl
## uncomment this for local testing
# COPY ./ /usr/local/lib/node_modules/spotify-dl/
WORKDIR /download
diff --git a/README.md b/README.md
index 0e78d73..6a9568a 100644
--- a/README.md
+++ b/README.md
@@ -71,22 +71,28 @@ $ spotifydl https://open.spotify.com/track/xyz
```
#### Options
-| Flag | Usage |
-| ---- | ------------------------------------------------------------ |
-| -o | takes valid output path argument |
-| --es | takes extra search string/term to be used for youtube search |
-| --oo | enforces all downloaded songs in the output dir |
-| --st | download spotify saved tracks |
-| --ss | download spotify saved shows |
-| --sp | download spotify saved playlists |
-| --sa | download spotify saved albums |
-| -u | spotify username (only needed in non tty) |
-| -p | spotify password (only needed in non tty) |
-| -cf | takes valid output file name path argument |
-| -v | returns current version |
-| -h | outputs help text |
+| Flag | Long Flag | Usage |
+| ----- | ----------------- | --------------------------------------------------------------------- |
+| --o | --output | takes valid output path argument |
+| --es | --extra-search | takes extra search string/term to be used for youtube search |
+| --oo | --output-only | enforces all downloaded songs in the output dir |
+| --st | --saved-tracks | download spotify saved tracks |
+| --ss | --saved-songs | download spotify saved shows |
+| --sp | --saved-playlists | download spotify saved playlists |
+| --sa | --saved-albums | download spotify saved albums |
+| --u | --username | spotify username (only needed in non tty) |
+| --p | --password | spotify password (only needed in non tty) |
+| --cf | --cache-file | takes valid output file name path argument |
+| --dr | --download-report | output a download report of what files failed |
+| --cof | --cookie-file | takes valid file name path argument to a txt file for youtube cookies |
+| --v | --version | returns current version |
+| --h | --help | outputs help text |
+## Notes
+
+If you receive a 429 error please provide a cookies file given the `--cof` flag, to generate a cookies file please refer to [Chrome](https://chrome.google.com/webstore/detail/njabckikapfpffapmjgojcnbfjonfjfg) or [Firefox](https://github.com/rotemdan/ExportCookies)
+
## Docker
```sh
docker run -it --user=$(id -u):$(id -g) -v $(pwd):/download --rm spotify-dl
diff --git a/config.js b/config.js
index ad667e5..2b7494b 100644
--- a/config.js
+++ b/config.js
@@ -4,6 +4,8 @@ module.exports = {
},
flags: {
cacheFile: '.spdlcache',
+ cookieFile: 'cookies.txt',
+ downloadReport: true,
output: process.cwd(),
extraSearch: '',
password: '',
diff --git a/lib/api.js b/lib/api.js
index 5dd9e8d..cf87cd3 100644
--- a/lib/api.js
+++ b/lib/api.js
@@ -104,12 +104,10 @@ module.exports = {
if (autoLogin) {
browser = await puppeteer.launch({
+ headless: true,
args: [
- // Required for Docker version of Puppeteer
'--no-sandbox',
'--disable-setuid-sandbox',
- // This will write shared memory files into /tmp instead of /dev/shm,
- // because Docker’s default for /dev/shm is 64MB
'--disable-dev-shm-usage',
],
});
@@ -122,13 +120,20 @@ module.exports = {
await page.click('#login-button');
await page.waitForSelector('#auth-accept');
await page.click('#auth-accept');
- } catch (_e) {
- console.log('Please find a screenshot of why the auto login failed at' +
- './failure.png');
+ } catch (e) {
+ logFailure(e.message);
+ const screenshotPath = './failure.png';
await page.screenshot({
- path: './failure.png',
+ path: screenshotPath,
fullPage: true,
});
+ throw new Error(
+ [
+ 'Could not generate token',
+ 'Please find a screenshot of why the auto login failed at ',
+ `${screenshotPath}`,
+ ].join(' '),
+ );
}
} else {
open(authURL);
@@ -194,7 +199,7 @@ module.exports = {
parseTrack: function (track) {
return {
name: track.name,
- artist_name: track.artists.map(artist => artist.name)[0],
+ artists: track.artists.map(artist => artist.name),
album_name: track.album.name,
release_date: track.album.release_date,
cover_url: track.album.images.map(image => image.url)[0],
@@ -204,7 +209,7 @@ module.exports = {
parseEpisode: function (episode) {
return {
name: episode.name,
- artist_name: episode.show.publisher,
+ artists: [episode.show.publisher],
album_name: episode.show.name,
release_date: episode.release_date,
cover_url: episode.images.map(image => image.url)[0],
diff --git a/lib/downloader.js b/lib/downloader.js
index 59b8c6b..bbf35ce 100644
--- a/lib/downloader.js
+++ b/lib/downloader.js
@@ -3,6 +3,8 @@ const ytdl = require('ytdl-core');
const { youtubeDLConfig } = require('../config');
const ffmpeg = require('fluent-ffmpeg');
const { SponsorBlock } = require('sponsorblock-api');
+const { cliInputs } = require('./setup');
+const fs = require('fs');
const sponsorBlock = new SponsorBlock(1234);
const {
SPONSOR_BLOCK: {
@@ -17,6 +19,7 @@ const {
},
FFMPEG: {
ASET,
+ TIMEOUT_MINUTES,
},
} = require('../util/constants');
const {
@@ -77,6 +80,30 @@ const progressFunction = (_, downloaded, total) => {
}
};
+const getYoutubeDLConfig = () => {
+ const config = youtubeDLConfig;
+ const { cookieFile } = cliInputs();
+ if (fs.existsSync(cookieFile)) {
+ const cookieFileContents = fs
+ .readFileSync(cookieFile, 'utf-8')
+ .split('\n')
+ .reduce((cookie, line) => {
+ const segments = line.split(/[\t]+|[ ]+/);
+ if (segments.length == 7) {
+ cookie += `${segments[5]}=${segments[6]}; `;
+ }
+ return cookie;
+ }, '')
+ .trim();
+ config.requestOptions = {
+ headers: {
+ Cookie: cookieFileContents,
+ },
+ };
+ }
+ return config;
+};
+
/**
* This function downloads the given youtube video
* in best audio format as mp3 file
@@ -91,10 +118,11 @@ const downloader = async (youtubeLinks, output) => {
const link = youtubeLinks[attemptCount];
logStart(`Trying youtube link ${attemptCount + 1}...`);
const complexFilter = await sponsorComplexFilter(link);
+
const doDownload = (resolve, reject) => {
- const download = ytdl(link, youtubeDLConfig);
+ const download = ytdl(link, getYoutubeDLConfig());
download.on('progress', progressFunction);
- const ffmpegCommand = ffmpeg();
+ const ffmpegCommand = ffmpeg({ timeout: TIMEOUT_MINUTES * 60 });
if (complexFilter) {
ffmpegCommand
.complexFilter(complexFilter)
diff --git a/lib/metadata.js b/lib/metadata.js
index e2808d4..a4c28ab 100644
--- a/lib/metadata.js
+++ b/lib/metadata.js
@@ -25,7 +25,7 @@ const mergeMetadata = async (output, songData) => {
}
await downloadAndSaveCover(coverURL, coverFileName);
const metadata = {
- artist: songData.artist_name,
+ artist: songData.artists,
album: songData.album_name,
title: songData.name,
date: songData.release_date,
diff --git a/lib/setup.js b/lib/setup.js
index 40a4154..cf52924 100644
--- a/lib/setup.js
+++ b/lib/setup.js
@@ -79,17 +79,17 @@ module.exports = {
help: {
alias: 'h',
helpText: [
- '--help or -h',
+ '--help or --h',
'* returns help',
- 'eg. $ spotifydl -h',
+ 'eg. $ spotifydl --h',
],
},
version: {
alias: 'v',
helpText: [
- '--version or -v',
+ '--version or --v',
'* returns the current version',
- 'eg. $ spotifydl -v',
+ 'eg. $ spotifydl --v',
],
},
cacheFile: {
@@ -99,7 +99,30 @@ module.exports = {
helpText: [
'--cache-file "" or --cf ""',
'-takes relative or absolute file path argument',
- 'eg. $ spotifydl -cf ~/songs.txt ',
+ 'eg. $ spotifydl --cf ~/songs.txt ',
+ ],
+ },
+ cookieFile: {
+ alias: 'cof',
+ type: 'string',
+ default: flagsConfig.cookieFile,
+ helpText: [
+ '--cookie-file "" or --cof ""',
+ '-takes relative or absolute file path argument',
+ '- defaults to cookies.txt',
+ 'eg. $ spotifydl --cof ~/cookies.txt ',
+ ],
+ },
+ downloadReport: {
+ alias: 'dr',
+ type: 'boolean',
+ default: flagsConfig.downloadReport,
+ helpText: [
+ '--download-report or --dr',
+ '-displays an output at the end of all failed items',
+ 'NOTE: uses alot of ram',
+ '-defaults to false',
+ 'eg. $ spotifydl --dr false ',
],
},
output: {
@@ -107,8 +130,8 @@ module.exports = {
type: 'string',
default: flagsConfig.output,
helpText: [
- '--output "" or -o ""', '-takes valid path argument',
- 'eg. $ spotifydl -o ~/songs ',
+ '--output "" or --o ""', '-takes valid path argument',
+ 'eg. $ spotifydl --o ~/songs ',
],
},
extraSearch: {
@@ -129,11 +152,11 @@ module.exports = {
default: flagsConfig.username,
isRequired: loginRequired,
helpText: [
- '--username "" or -u ""',
+ '--username "" or --u ""',
'* takes string for spotify username',
'* optional when tty',
- '* required when using -sa, -sp and -st in non tty',
- 'eg. $ spotifydl -u "username"',
+ '* required when using --sa, --sp and --st in non tty',
+ 'eg. $ spotifydl --u "username"',
],
},
password: {
@@ -142,11 +165,11 @@ module.exports = {
default: flagsConfig.password,
isRequired: loginRequired,
helpText: [
- '--password "" or -p ""',
+ '--password "" or --p ""',
'* takes string for spotify password',
'* optional when tty',
- '* required when using -sa, -sp and -st in non tty',
- 'eg. $ spotifydl -p "password"',
+ '* required when using --sa, --sp and --st in non tty',
+ 'eg. $ spotifydl --p "password"',
],
},
savedAlbums: {
@@ -157,8 +180,8 @@ module.exports = {
'--saved-albums or --sa',
'* downloads a users saved albums',
'* username and password required for non TTY',
- 'eg. $ spotifydl -u "username" -p "password" -sa',
- 'eg. $ spotifydl -sa',
+ 'eg. $ spotifydl --u "username" --p "password" --sa',
+ 'eg. $ spotifydl --sa',
],
},
savedShows: {
@@ -169,8 +192,8 @@ module.exports = {
'--saved-shows or --ss',
'* downloads a users saved shows',
'* username and password required for non TTY',
- 'eg. $ spotifydl -u "username" -p "password" -ss',
- 'eg. $ spotifydl -ss',
+ 'eg. $ spotifydl --u "username" --p "password" --ss',
+ 'eg. $ spotifydl --ss',
],
},
savedPlaylists: {
@@ -181,8 +204,8 @@ module.exports = {
'--saved-playlists or --sp',
'* downloads a users saved playlists',
'* username and password required for non TTY',
- 'eg. $ spotifydl -u "username" -p "password" -sp',
- 'eg. $ spotifydl -sp',
+ 'eg. $ spotifydl --u "username" --p "password" --sp',
+ 'eg. $ spotifydl --sp',
],
},
savedTracks: {
@@ -193,8 +216,8 @@ module.exports = {
'--saved-tracks or --st',
'* downloads a users saved tracks',
'* username and password required for non TTY',
- 'eg. $ spotifydl -st',
- 'eg. $ spotifydl -u "username" -p "password" -st',
+ 'eg. $ spotifydl --st',
+ 'eg. $ spotifydl --u "username" --p "password" --st',
],
},
outputOnly: {
@@ -204,7 +227,7 @@ module.exports = {
helpText: [
'--outputOnly or --oo',
'* saves all songs directly to the output dir',
- 'eg. $ spotifydl -oo',
+ 'eg. $ spotifydl --oo',
],
},
};
@@ -223,10 +246,10 @@ module.exports = {
$ spotifydl https://open.spotify.com/playlist/4hOKQuZbraPDIfaGbM3lKI
$ spotifydl https://open.spotify.com/album/32Epx6wQXSulDr24Ez6vTE
$ spotifydl https://open.spotify.com/artist/3vn7rk7VNMfDhuZNB9sDYP
- $ spotifydl -u username -p password -sa
- $ spotifydl -sp
- $ spotifydl -st
- $ spotifydl -cf
+ $ spotifydl --u username --p password --sa
+ $ spotifydl --sp
+ $ spotifydl --st
+ $ spotifydl --cf
Options
${helpText}
@@ -266,11 +289,14 @@ module.exports = {
process.exit(1);
}
+
return {
inputs: inputs,
extraSearch: inputFlags.extraSearch,
output: inputFlags.output,
cacheFile: inputFlags.cacheFile,
+ cookieFile: inputFlags.cookieFile,
+ downloadReport: inputFlags.downloadReport,
outputOnly: inputFlags.outputOnly,
username: inputFlags.username,
password: inputFlags.password,
diff --git a/package-lock.json b/package-lock.json
index a158a77..30f2b15 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,7 +21,7 @@
"spotify-web-api-node": "^5.0.0",
"string-similarity": "^4.0.4",
"yt-search": "^2.7.5",
- "ytdl-core": "^4.8.0"
+ "ytdl-core": "^4.9.1"
},
"bin": {
"spotifydl": "cli.js"
@@ -9600,9 +9600,9 @@
}
},
"node_modules/ytdl-core": {
- "version": "4.8.3",
- "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.8.3.tgz",
- "integrity": "sha512-cWCBeX4FCgjcKmuVK384MT582RIAakpUSeMF/NPVmhO8cWiG+LeQLnBordvLolb0iXYzfUvalgmycYAE5Sy6Xw==",
+ "version": "4.9.1",
+ "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.9.1.tgz",
+ "integrity": "sha512-6Jbp5RDhUEozlaJQAR+l8oV8AHsx3WUXxSyPxzE6wOIAaLql7Hjiy0ZM58wZoyj1YEenlEPjEqcJIjKYKxvHtQ==",
"dependencies": {
"m3u8stream": "^0.8.3",
"miniget": "^4.0.0",
@@ -16919,9 +16919,9 @@
}
},
"ytdl-core": {
- "version": "4.8.3",
- "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.8.3.tgz",
- "integrity": "sha512-cWCBeX4FCgjcKmuVK384MT582RIAakpUSeMF/NPVmhO8cWiG+LeQLnBordvLolb0iXYzfUvalgmycYAE5Sy6Xw==",
+ "version": "4.9.1",
+ "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.9.1.tgz",
+ "integrity": "sha512-6Jbp5RDhUEozlaJQAR+l8oV8AHsx3WUXxSyPxzE6wOIAaLql7Hjiy0ZM58wZoyj1YEenlEPjEqcJIjKYKxvHtQ==",
"requires": {
"m3u8stream": "^0.8.3",
"miniget": "^4.0.0",
diff --git a/package.json b/package.json
index df53da6..fb9e8db 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
"spotify-web-api-node": "^5.0.0",
"string-similarity": "^4.0.4",
"yt-search": "^2.7.5",
- "ytdl-core": "^4.8.0"
+ "ytdl-core": "^4.9.1"
},
"xo": {
"space": true,
diff --git a/util/constants.js b/util/constants.js
index d567048..b02c32b 100644
--- a/util/constants.js
+++ b/util/constants.js
@@ -30,6 +30,7 @@ module.exports = {
},
FFMPEG: {
ASET: 'asetpts=PTS-STARTPTS',
+ TIMEOUT_MINUTES: 30,
},
MAX_LIMIT_DEFAULT: 50,
SERVER: {
diff --git a/util/get-songdata.js b/util/get-songdata.js
index 45656ed..3ab2bbd 100644
--- a/util/get-songdata.js
+++ b/util/get-songdata.js
@@ -29,7 +29,7 @@ class SpotifyExtractor {
const albumInfo = await spotify.extractAlbum(albumIds[x]);
// hardcode to artist being requested
albumInfo.items.forEach(item => {
- item.artist_name = artistResult.name;
+ item.artists = [artistResult.name, ...item.artists];
});
albumInfos.push(albumInfo);
}
diff --git a/util/runner.js b/util/runner.js
index 4ff5dff..e0095e6 100644
--- a/util/runner.js
+++ b/util/runner.js
@@ -13,13 +13,13 @@ const mergeMetadata = require('../lib/metadata');
const { cliInputs } = require('../lib/setup');
const SpotifyExtractor = require('./get-songdata');
const { logSuccess, logInfo, logFailure } = require('./log-helper');
-const { inputs, extraSearch, output, outputOnly } = cliInputs();
+const { inputs, extraSearch, output, outputOnly, downloadReport } = cliInputs();
module.exports = {
itemOutputDir: item => {
const outputDir = path.normalize(output);
return outputOnly ? outputDir : path.join(
outputDir,
- filter.cleanOutputPath(item.artist_name),
+ filter.cleanOutputPath(item.artists[0]),
filter.cleanOutputPath(item.album_name),
);
},
@@ -38,7 +38,7 @@ module.exports = {
const itemId = nextItem.id;
const itemName = nextItem.name;
const albumName = nextItem.album_name;
- const artistName = nextItem.artist_name;
+ const artistName = nextItem.artists[0];
logInfo(
[
`${currentCount}/${itemsCount}`,
@@ -58,9 +58,11 @@ module.exports = {
},
);
+ const fileNameCleaned = filter.cleanOutputPath(itemName) || '_';
+
const output = path.resolve(
itemDir,
- `${filter.cleanOutputPath(itemName)}.mp3`,
+ `${fileNameCleaned}.mp3`,
);
const downloadSuccessful = await downloader(ytLinks, output);
if (downloadSuccessful) {
@@ -88,47 +90,44 @@ module.exports = {
});
return await this.downloadLoop(list);
},
- downloadLists: async function (lists) {
- const listResults = [];
- for (const [x, list] of lists.entries()) {
- logInfo(`Starting download of list ${x + 1}/${lists.length}`);
- listResults.push(await this.downloadList(list));
- }
- logInfo('Download Report:');
- listResults.forEach(result => {
- const listItems = result.items;
- const itemLength = listItems.length;
- const failedItems = listItems.filter(item => item.failed);
- const failedItemLength = failedItems.length;
- logInfo(
- [
- 'Successfully downloaded',
- `${itemLength - failedItemLength}/${itemLength}`,
- `for ${result.name} (${result.type})`,
- ].join(' '),
- );
- if (failedItemLength) {
- logFailure(
+ generateReport: async function (listResults) {
+ if (listResults.length) {
+ logInfo('Download Report:');
+ listResults.forEach(result => {
+ const listItems = result.items;
+ const itemLength = listItems.length;
+ const failedItems = listItems.filter(item => item.failed);
+ const failedItemLength = failedItems.length;
+ logInfo(
[
- 'Failed items:',
- ...failedItems.map(item => {
- return [
- `Item: (${item.name})`,
- `Album: ${item.album_name}`,
- `Artist: ${item.artist_name}`,
- `ID: (${item.id})`,
- ].join(' ');
- }),
- ].join('\n'),
+ 'Successfully downloaded',
+ `${itemLength - failedItemLength}/${itemLength}`,
+ `for ${result.name} (${result.type})`,
+ ].join(' '),
);
- }
- });
- logSuccess('Finished!');
+ if (failedItemLength) {
+ logFailure(
+ [
+ 'Failed items:',
+ ...failedItems.map(item => {
+ return [
+ `Item: (${item.name})`,
+ `Album: ${item.album_name}`,
+ `Artist: ${item.artists[0]}`,
+ `ID: (${item.id})`,
+ ].join(' ');
+ }),
+ ].join('\n'),
+ );
+ }
+ });
+ }
},
run: async function () {
const spotifyExtractor = new SpotifyExtractor();
- const lists = [];
+ const listResults = [];
for (const input of inputs) {
+ const lists = [];
logInfo(`Starting processing of ${input.type} (${input.url})`);
const URL = input.url;
switch (input.type) {
@@ -138,7 +137,7 @@ module.exports = {
items: [
track,
],
- name: `${track.name} ${track.artist_name}`,
+ name: `${track.name} ${track.artists[0]}`,
type: input.type,
});
break;
@@ -215,7 +214,7 @@ module.exports = {
items: [
{
name: URL,
- artist_name: '',
+ artists: [''],
album_name: URL,
release_date: null,
//todo can we get the youtube image?
@@ -234,7 +233,16 @@ module.exports = {
'Please visit github and make a request to support this type');
}
}
+
+ for (const [x, list] of lists.entries()) {
+ logInfo(`Starting download of list ${x + 1}/${lists.length}`);
+ const downloadResult = await this.downloadList(list);
+ if (downloadReport) {
+ listResults.push(downloadResult);
+ }
+ }
}
- await this.downloadLists(lists);
+ await this.generateReport(listResults);
+ logSuccess('Finished!');
},
};
\ No newline at end of file