Merge pull request #128 from phyzical/feature/add-cookies-file-support

Feature/add cookies file support
SwapnilSoni1999 · Aug 17, 2021 · 9b59e54 · 9b59e54
2 parents 31aea44 + 0289f94
commit 9b59e54
Show file tree

Hide file tree

Showing 12 changed files with 187 additions and 108 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,12 @@
-FROM alpine
+FROM buildkite/puppeteer
+
+RUN apt update && \
+    apt install -y  \
+    ffmpeg && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g spotify-dl --unsafe-perm
 
-RUN apk add \
-    npm \
-    ffmpeg
-RUN npm install -g spotify-dl
 ## uncomment this for local testing
 # COPY ./ /usr/local/lib/node_modules/spotify-dl/ 
 WORKDIR /download

diff --git a/README.md b/README.md
@@ -71,22 +71,28 @@ $ spotifydl https://open.spotify.com/track/xyz
 ```
 
 #### Options
-| Flag | Usage                                                        |
-| ---- | ------------------------------------------------------------ |
-| -o   | takes valid output path argument                             |
-| --es | takes extra search string/term to be used for youtube search |
-| --oo | enforces all downloaded songs in the output dir              |
-| --st | download spotify saved tracks                                |
-| --ss | download spotify saved shows                                 |
-| --sp | download spotify saved playlists                             |
-| --sa | download spotify saved albums                                |
-| -u   | spotify username (only needed in non tty)                    |
-| -p   | spotify password (only needed in non tty)                    |
-| -cf  | takes valid output file name path argument                   |
-| -v   | returns current version                                      |
-| -h   | outputs help text                                            |
+| Flag  | Long Flag         | Usage                                                                 |
+| ----- | ----------------- | --------------------------------------------------------------------- |
+| --o   | --output          | takes valid output path argument                                      |
+| --es  | --extra-search    | takes extra search string/term to be used for youtube search          |
+| --oo  | --output-only     | enforces all downloaded songs in the output dir                       |
+| --st  | --saved-tracks    | download spotify saved tracks                                         |
+| --ss  | --saved-songs     | download spotify saved shows                                          |
+| --sp  | --saved-playlists | download spotify saved playlists                                      |
+| --sa  | --saved-albums    | download spotify saved albums                                         |
+| --u   | --username        | spotify username (only needed in non tty)                             |
+| --p   | --password        | spotify password (only needed in non tty)                             |
+| --cf  | --cache-file      | takes valid output file name path argument                            |
+| --dr  | --download-report | output a download report of what files failed                         |
+| --cof | --cookie-file     | takes valid file name path argument to a txt file for youtube cookies |
+| --v   | --version         | returns current version                                               |
+| --h   | --help            | outputs help text                                                     |
 <hr>
 
+## Notes
+
+If you receive a 429 error please provide a cookies file given the `--cof` flag, to generate a cookies file please refer to [Chrome](https://chrome.google.com/webstore/detail/njabckikapfpffapmjgojcnbfjonfjfg) or [Firefox](https://github.com/rotemdan/ExportCookies)
+
 ## Docker
 ```sh
 docker run -it --user=$(id -u):$(id -g) -v $(pwd):/download --rm spotify-dl <options-to-spotify-dl defaults to --help>

diff --git a/config.js b/config.js
@@ -4,6 +4,8 @@ module.exports = {
   },
   flags: {
     cacheFile: '.spdlcache',
+    cookieFile: 'cookies.txt',
+    downloadReport: true,
     output: process.cwd(),
     extraSearch: '',
     password: '',

diff --git a/lib/api.js b/lib/api.js
@@ -104,12 +104,10 @@ module.exports = {
 
     if (autoLogin) {
       browser = await puppeteer.launch({
+        headless: true,
         args: [
-          // Required for Docker version of Puppeteer
           '--no-sandbox',
           '--disable-setuid-sandbox',
-          // This will write shared memory files into /tmp instead of /dev/shm,
-          // because Docker’s default for /dev/shm is 64MB
           '--disable-dev-shm-usage',
         ],
       });
@@ -122,13 +120,20 @@ module.exports = {
         await page.click('#login-button');
         await page.waitForSelector('#auth-accept');
         await page.click('#auth-accept');
-      } catch (_e) {
-        console.log('Please find a screenshot of why the auto login failed at' +
-          './failure.png');
+      } catch (e) {
+        logFailure(e.message);
+        const screenshotPath = './failure.png';
         await page.screenshot({
-          path: './failure.png',
+          path: screenshotPath,
           fullPage: true,
         });
+        throw new Error(
+          [
+            'Could not generate token',
+            'Please find a screenshot of why the auto login failed at ',
+            `${screenshotPath}`,
+          ].join(' '),
+        );
       }
     } else {
       open(authURL);
@@ -194,7 +199,7 @@ module.exports = {
   parseTrack: function (track) {
     return {
       name: track.name,
-      artist_name: track.artists.map(artist => artist.name)[0],
+      artists: track.artists.map(artist => artist.name),
       album_name: track.album.name,
       release_date: track.album.release_date,
       cover_url: track.album.images.map(image => image.url)[0],
@@ -204,7 +209,7 @@ module.exports = {
   parseEpisode: function (episode) {
     return {
       name: episode.name,
-      artist_name: episode.show.publisher,
+      artists: [episode.show.publisher],
       album_name: episode.show.name,
       release_date: episode.release_date,
       cover_url: episode.images.map(image => image.url)[0],

diff --git a/lib/downloader.js b/lib/downloader.js
@@ -3,6 +3,8 @@ const ytdl = require('ytdl-core');
 const { youtubeDLConfig } = require('../config');
 const ffmpeg = require('fluent-ffmpeg');
 const { SponsorBlock } = require('sponsorblock-api');
+const { cliInputs } = require('./setup');
+const fs = require('fs');
 const sponsorBlock = new SponsorBlock(1234);
 const {
   SPONSOR_BLOCK: {
@@ -17,6 +19,7 @@ const {
   },
   FFMPEG: {
     ASET,
+    TIMEOUT_MINUTES,
   },
 } = require('../util/constants');
 const {
@@ -77,6 +80,30 @@ const progressFunction = (_, downloaded, total) => {
   }
 };
 
+const getYoutubeDLConfig = () => {
+  const config = youtubeDLConfig;
+  const { cookieFile } = cliInputs();
+  if (fs.existsSync(cookieFile)) {
+    const cookieFileContents = fs
+      .readFileSync(cookieFile, 'utf-8')
+      .split('\n')
+      .reduce((cookie, line) => {
+        const segments = line.split(/[\t]+|[ ]+/);
+        if (segments.length == 7) {
+          cookie += `${segments[5]}=${segments[6]}; `;
+        }
+        return cookie;
+      }, '')
+      .trim();
+    config.requestOptions = {
+      headers: {
+        Cookie: cookieFileContents,
+      },
+    };
+  }
+  return config;
+};
+
 /**
  * This function downloads the given youtube video 
  * in best audio format as mp3 file
@@ -91,10 +118,11 @@ const downloader = async (youtubeLinks, output) => {
     const link = youtubeLinks[attemptCount];
     logStart(`Trying youtube link ${attemptCount + 1}...`);
     const complexFilter = await sponsorComplexFilter(link);
+
     const doDownload = (resolve, reject) => {
-      const download = ytdl(link, youtubeDLConfig);
+      const download = ytdl(link, getYoutubeDLConfig());
       download.on('progress', progressFunction);
-      const ffmpegCommand = ffmpeg();
+      const ffmpegCommand = ffmpeg({ timeout: TIMEOUT_MINUTES * 60 });
       if (complexFilter) {
         ffmpegCommand
           .complexFilter(complexFilter)

diff --git a/lib/metadata.js b/lib/metadata.js
@@ -25,7 +25,7 @@ const mergeMetadata = async (output, songData) => {
   }
   await downloadAndSaveCover(coverURL, coverFileName);
   const metadata = {
-    artist: songData.artist_name,
+    artist: songData.artists,
     album: songData.album_name,
     title: songData.name,
     date: songData.release_date,

diff --git a/lib/setup.js b/lib/setup.js
@@ -79,17 +79,17 @@ module.exports = {
       help: {
         alias: 'h',
         helpText: [
-          '--help or -h',
+          '--help or --h',
           '* returns help',
-          'eg. $ spotifydl -h',
+          'eg. $ spotifydl --h',
         ],
       },
       version: {
         alias: 'v',
         helpText: [
-          '--version or -v',
+          '--version or --v',
           '* returns the current version',
-          'eg. $ spotifydl -v',
+          'eg. $ spotifydl --v',
         ],
       },
       cacheFile: {
@@ -99,16 +99,39 @@ module.exports = {
         helpText: [
           '--cache-file "<file-path>" or --cf "<file-path>"',
           '-takes relative or absolute file path argument',
-          'eg. $ spotifydl -cf ~/songs.txt <url>',
+          'eg. $ spotifydl --cf ~/songs.txt <url>',
+        ],
+      },
+      cookieFile: {
+        alias: 'cof',
+        type: 'string',
+        default: flagsConfig.cookieFile,
+        helpText: [
+          '--cookie-file "<file-path>" or --cof "<file-path>"',
+          '-takes relative or absolute file path argument',
+          '- defaults to cookies.txt',
+          'eg. $ spotifydl --cof ~/cookies.txt <url>',
+        ],
+      },
+      downloadReport: {
+        alias: 'dr',
+        type: 'boolean',
+        default: flagsConfig.downloadReport,
+        helpText: [
+          '--download-report or --dr',
+          '-displays an output at the end of all failed items',
+          'NOTE: uses alot of ram',
+          '-defaults to false',
+          'eg. $ spotifydl --dr false <url>',
         ],
       },
       output: {
         alias: 'o',
         type: 'string',
         default: flagsConfig.output,
         helpText: [
-          '--output "<path>" or -o "<path>"', '-takes valid path argument',
-          'eg. $ spotifydl -o ~/songs <url>',
+          '--output "<path>" or --o "<path>"', '-takes valid path argument',
+          'eg. $ spotifydl --o ~/songs <url>',
         ],
       },
       extraSearch: {
@@ -129,11 +152,11 @@ module.exports = {
         default: flagsConfig.username,
         isRequired: loginRequired,
         helpText: [
-          '--username "<username>" or -u "<username>"',
+          '--username "<username>" or --u "<username>"',
           '* takes string for spotify username',
           '* optional when tty',
-          '* required when using -sa, -sp and -st in non tty',
-          'eg. $ spotifydl -u "username"',
+          '* required when using --sa, --sp and --st in non tty',
+          'eg. $ spotifydl --u "username"',
         ],
       },
       password: {
@@ -142,11 +165,11 @@ module.exports = {
         default: flagsConfig.password,
         isRequired: loginRequired,
         helpText: [
-          '--password "<password>" or -p "<password>"',
+          '--password "<password>" or --p "<password>"',
           '* takes string for spotify password',
           '* optional when tty',
-          '* required when using -sa, -sp and -st in non tty',
-          'eg. $ spotifydl -p "password"',
+          '* required when using --sa, --sp and --st in non tty',
+          'eg. $ spotifydl --p "password"',
         ],
       },
       savedAlbums: {
@@ -157,8 +180,8 @@ module.exports = {
           '--saved-albums or --sa',
           '* downloads a users saved albums',
           '* username and password required for non TTY',
-          'eg. $ spotifydl -u "username" -p "password" -sa',
-          'eg. $ spotifydl -sa',
+          'eg. $ spotifydl --u "username" --p "password" --sa',
+          'eg. $ spotifydl --sa',
         ],
       },
       savedShows: {
@@ -169,8 +192,8 @@ module.exports = {
           '--saved-shows or --ss',
           '* downloads a users saved shows',
           '* username and password required for non TTY',
-          'eg. $ spotifydl -u "username" -p "password" -ss',
-          'eg. $ spotifydl -ss',
+          'eg. $ spotifydl --u "username" --p "password" --ss',
+          'eg. $ spotifydl --ss',
         ],
       },
       savedPlaylists: {
@@ -181,8 +204,8 @@ module.exports = {
           '--saved-playlists or --sp',
           '* downloads a users saved playlists',
           '* username and password required for non TTY',
-          'eg. $ spotifydl -u "username" -p "password" -sp',
-          'eg. $ spotifydl -sp',
+          'eg. $ spotifydl --u "username" --p "password" --sp',
+          'eg. $ spotifydl --sp',
         ],
       },
       savedTracks: {
@@ -193,8 +216,8 @@ module.exports = {
           '--saved-tracks or --st',
           '* downloads a users saved tracks',
           '* username and password required for non TTY',
-          'eg. $ spotifydl -st',
-          'eg. $ spotifydl -u "username" -p "password" -st',
+          'eg. $ spotifydl --st',
+          'eg. $ spotifydl --u "username" --p "password" --st',
         ],
       },
       outputOnly: {
@@ -204,7 +227,7 @@ module.exports = {
         helpText: [
           '--outputOnly or --oo',
           '* saves all songs directly to the output dir',
-          'eg. $ spotifydl -oo',
+          'eg. $ spotifydl --oo',
         ],
       },
     };
@@ -223,10 +246,10 @@ module.exports = {
           $ spotifydl https://open.spotify.com/playlist/4hOKQuZbraPDIfaGbM3lKI
           $ spotifydl https://open.spotify.com/album/32Epx6wQXSulDr24Ez6vTE
           $ spotifydl https://open.spotify.com/artist/3vn7rk7VNMfDhuZNB9sDYP
-          $ spotifydl -u username -p password -sa
-          $ spotifydl -sp
-          $ spotifydl -st
-          $ spotifydl <link> -cf <cache-file>
+          $ spotifydl --u username --p password --sa
+          $ spotifydl --sp
+          $ spotifydl --st
+          $ spotifydl <link> --cf <cache-file>
     
       Options
       ${helpText}    
@@ -266,11 +289,14 @@ module.exports = {
       process.exit(1);
     }
 
+
     return {
       inputs: inputs,
       extraSearch: inputFlags.extraSearch,
       output: inputFlags.output,
       cacheFile: inputFlags.cacheFile,
+      cookieFile: inputFlags.cookieFile,
+      downloadReport: inputFlags.downloadReport,
       outputOnly: inputFlags.outputOnly,
       username: inputFlags.username,
       password: inputFlags.password,