Merge branch 'fixes'

client_test.go

@@ -83,7 +83,7 @@ func TestGetVideoWithoutManifestURL(t *testing.T) {
 	assert, require := assert.New(t), require.New(t)
 
 	video, err := testClient.GetVideo(dwlURL)
-	require.NoError(err)
+	require.NoError(err, "get video")
 	require.NotNil(video)
 
 	assert.NotEmpty(video.Thumbnails)
@@ -95,9 +95,11 @@ func TestGetVideoWithoutManifestURL(t *testing.T) {
 	assert.Equal("rFejpH_tAHM", video.ID)
 	assert.Equal("dotGo 2015 - Rob Pike - Simplicity is Complicated", video.Title)
 	assert.Equal("dotconferences", video.Author)
-	assert.Equal(1392*time.Second, video.Duration)
+	assert.GreaterOrEqual(video.Duration, 1390*time.Second)
 	assert.Contains(video.Description, "Go is often described as a simple language.")
-	assert.Equal("2015-12-02 00:00:00 +0000 UTC", video.PublishDate.String())
+
+	// Publishing date doesn't seem to be present in android client
+	// assert.Equal("2015-12-02 00:00:00 +0000 UTC", video.PublishDate.String())
 }
 
 func TestGetVideoWithManifestURL(t *testing.T) {
@@ -175,8 +177,10 @@ func TestGetBigPlaylist(t *testing.T) {
 	assert.NotEmpty(playlist.Description)
 	assert.NotEmpty(playlist.Author)
 
-	assert.Greater(len(playlist.Videos), 100)
-	assert.NotEmpty(playlist.Videos[100].ID)
+	assert.Greater(len(playlist.Videos), 300)
+	assert.NotEmpty(playlist.Videos[300].ID)
+
+	t.Logf("Playlist Title: %s, Video Count: %d", playlist.Title, len(playlist.Videos))
 }
 
 func TestClient_httpGetBodyBytes(t *testing.T) {

cmd/youtubedr/downloader.go

@@ -11,10 +11,11 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/kkdai/youtube/v2"
-	ytdl "github.com/kkdai/youtube/v2/downloader"
 	"github.com/spf13/pflag"
 	"golang.org/x/net/http/httpproxy"
+
+	"github.com/kkdai/youtube/v2"
+	ytdl "github.com/kkdai/youtube/v2/downloader"
 )
 
 var (
@@ -94,7 +95,7 @@ func getVideoWithFormat(id string) (*youtube.Video, *youtube.Format, error) {
 		}
 
 	case outputQuality != "":
-		format = formats.WithAudioChannels().FindByQuality(outputQuality)
+		format = formats.FindByQuality(outputQuality)
 		if format == nil {
 			return nil, nil, fmt.Errorf("unable to find format with quality %s", outputQuality)
 		}

decipher.go

@@ -283,12 +283,3 @@ func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, e
 	}
 	return ops, nil
 }
-
-func (config playerConfig) getSignatureTimestamp() (string, error) {
-	result := signatureRegexp.FindSubmatch(config)
-	if result == nil {
-		return "", ErrSignatureTimestampNotFound
-	}
-
-	return string(result[1]), nil
-}

downloader/downloader_test.go

@@ -7,9 +7,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kkdai/youtube/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/kkdai/youtube/v2"
 )
 
 var testDownloader = func() (dl Downloader) {
@@ -39,7 +40,7 @@ func TestDownload_FirstStream(t *testing.T) {
 	assert.Equal(`youtube-dl test video "'/\ä↭𝕐`, video.Title)
 	assert.Equal(`Philipp Hagemeister`, video.Author)
 	assert.Equal(10*time.Second, video.Duration)
-	assert.Len(video.Formats, 18)
+	assert.GreaterOrEqual(len(video.Formats), 18)
 
 	if assert.Greater(len(video.Formats), 0) {
 		assert.NoError(testDownloader.Download(ctx, video, &video.Formats[0], ""))

fetch_testdata_helper.go

@@ -1,3 +1,4 @@
+//go:build fetch
 // +build fetch
 
 package youtube

format_list.go

@@ -9,6 +9,8 @@ import (
 type FormatList []Format
 
 // FindByQuality returns the first format matching Quality or QualityLabel
+//
+// Examples: tiny, small, medium, large, 720p, hd720, hd1080
 func (list FormatList) FindByQuality(quality string) *Format {
 	for i := range list {
 		if list[i].Quality == quality || list[i].QualityLabel == quality {

itag_test.go

@@ -14,5 +14,5 @@ func TestYoutube_GetItagInfo(t *testing.T) {
 	url := "https://www.youtube.com/watch?v=rFejpH_tAHM"
 	video, err := client.GetVideo(url)
 	require.NoError(err)
-	require.Len(video.Formats, 24)
+	require.GreaterOrEqual(len(video.Formats), 24)
 }

player_parse.go

@@ -15,11 +15,7 @@ type playerConfig []byte
 
 var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`)
 
-// we may use \d{5} instead of \d+ since currently its 5 digits, but i can't be sure it will be 5 digits always
-var signatureRegexp = regexp.MustCompile(`(?m)(?:^|,)(?:signatureTimestamp:)(\d+)`)
-
 func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) {
-
 	embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID)
 	embedBody, err := c.httpGetBodyBytes(ctx, embedURL)
 	if err != nil {

playlist.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"runtime/debug"
 	"strconv"
 	"time"
 
@@ -68,8 +69,9 @@ func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body [
 	}
 
 	defer func() {
+		stack := debug.Stack()
 		if r := recover(); r != nil {
-			err = fmt.Errorf("JSON parsing error: %v", r)
+			err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack)
 		}
 	}()
 
@@ -80,27 +82,70 @@ func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body [
 		return ErrPlaylistStatus{Reason: message}
 	}
 
-	p.Title = j.GetPath("metadata", "playlistMetadataRenderer", "title").MustString()
-	p.Description = j.GetPath("metadata", "playlistMetadataRenderer", "description").MustString()
+	// Metadata can be located in multiple places depending on client type
+	var metadata *sjson.Json
+	if node, ok := j.CheckGet("metadata"); ok {
+		metadata = node
+	} else if node, ok := j.CheckGet("header"); ok {
+		metadata = node
+	} else {
+		return fmt.Errorf("no playlist header / metadata found")
+	}
+
+	metadata = metadata.Get("playlistHeaderRenderer")
+
+	p.Title = sjsonGetText(metadata, "title")
+	p.Description = sjsonGetText(metadata, "description", "descriptionText")
 	p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
 		GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
 		GetIndex(0).Get("text").MustString()
-	vJSON, err := j.GetPath("contents", "twoColumnBrowseResultsRenderer", "tabs").GetIndex(0).
-		GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0).
-		GetPath("itemSectionRenderer", "contents").GetIndex(0).
-		GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
+
+	if len(p.Author) == 0 {
+		p.Author = sjsonGetText(metadata, "owner", "ownerText")
+	}
+
+	contents, ok := j.CheckGet("contents")
+	if !ok {
+		return fmt.Errorf("contents not found in json body")
+	}
+
+	// contents can have different keys with same child structure
+	firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0).
+		GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0)
+
+	// This extra nested item is only set with the web client
+	if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) {
+		firstPart = n
+	}
+
+	vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
+	if err != nil {
+		return err
+	}
+
+	if len(vJSON) <= 4 {
+		return fmt.Errorf("no video data found in JSON")
+	}
 
 	entries, continuation, err := extractPlaylistEntries(vJSON)
 	if err != nil {
 		return err
 	}
 
+	if len(continuation) == 0 {
+		continuation = getContinuation(firstPart.Get("playlistVideoListRenderer"))
+	}
+
+	if len(entries) == 0 {
+		return fmt.Errorf("no videos found in playlist")
+	}
+
 	p.Videos = entries
 
 	for continuation != "" {
-		data := prepareInnertubePlaylistData(continuation, true, webClient)
+		data := prepareInnertubePlaylistData(continuation, true, *client.client)
 
-		body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+webClient.key, data)
+		body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data)
 		if err != nil {
 			return err
 		}
@@ -110,9 +155,14 @@ func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body [
 			return err
 		}
 
-		vJSON, err := j.GetPath("onResponseReceivedActions").GetIndex(0).
-			GetPath("appendContinuationItemsAction", "continuationItems").MarshalJSON()
+		next := j.GetPath("onResponseReceivedActions").GetIndex(0).
+			GetPath("appendContinuationItemsAction", "continuationItems")
+
+		if !isValidJSON(next) {
+			next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents")
+		}
 
+		vJSON, err := next.MarshalJSON()
 		if err != nil {
 			return err
 		}
@@ -122,7 +172,13 @@ func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body [
 			return err
 		}
 
-		p.Videos, continuation = append(p.Videos, entries...), token
+		if len(token) > 0 {
+			continuation = token
+		} else {
+			continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation"))
+		}
+
+		p.Videos = append(p.Videos, entries...)
 	}
 
 	return err