diff --git a/changelog.txt b/changelog.txt index 7b1c1de..05e36c0 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,178 @@ 更新履歴 +20221122.42 +・録画済みのデータベース(sqlite3)の各種情報を表示するコマンド(-dbinfo)追加 + $ livedl -dbinfo -- 'database filename(.sqlite3) with fullpath' + - youtubeのデーターベースはcomment情報のみ表示 + - データベース情報表示、データベースextractの際DBをreadonlyで開くように修正 + - データベースファイルの存在チェックを追加 +・Refactoring + - 旧配信(RTMP)、実験放送のロジックを削除 + - 不要なオプションの削除(options.go) + -nico-hls-only + -nico-rtmp-only + -nico-rtmp-max-conn + -nico-rtmp-index [,] + - 不要な変数(wsapi、broadcastId他)の削除、ソース整形(nico_hls.go、nico.go) + - ログインチェック及びページエラー処理を現状のニコ生に合わせて修正 getProps() + - -nico-loginを指定した場合-nico-login-only=onにしてconf.dbに保存するよう修正 +・録画時強制予約機能(-nico-force-reservation)修正 + - タイムシフト録画時に予約されていない場合自動的に予約する(プレミア) + - 予約しているタイムシフト放送を録画時に自動的にチケット使用する + - TS予約/チケット使用APIのendpointを変更 +・httpbase内の全関数をcookiejar対応に変更 +・httpbase.SetTimeout()追加 + +20221108.41 +・直接ログインの２段階認証(MFA)対応 +・上記に伴うlogin APIのendpoint、cookie取得方法の変更 +・firefoxからのcookie取得機能追加 + -nico-cookies firefox[:profile|cookiefile] + e.g. + - profile default-release のcookieを取得 + ./livedl -nico-cookies firefox + - profile NicoTaro のcookieを取得 + ./livedl -nico-cookies firefox:NicoTaro + - 直接cookiefileを指定 + ./livedl -nico-cookies firefox:'C:/Users/*******/AppData/Roaming/Mozilla/Firefox/Profiles/*****/cookies.sqlite' +※Mac/Linuxで `cookies from browser failed: firefox profiles not found`が 表示される場合は報告おねがいします +※直接cookiefile指定の場合は必ず'か"で囲ってください +※プロファイルにspaceを含む場合は'か"で囲ってください + +20220905.40 +・ニコ生のコメントのvposを補正 + -nico-adjust-vpos=on + コメント書き出し時にvposの値を補正する + vposの値が-1000より小さい場合はコメント出力しない + -nico-adjust-vpos=off + コメント書き出し時にvposの値をそのまま出力する(デフォルト) + ※ExtractChunks()もコメントvposを補正するように修正 + ※ニコ生の生放送を録画する際、再接続してもkvsテーブルは更新されません +・Youtubeのコメントにemojiを出力する/しない + -yt-emoji=on + コメントにemojiを表示する(デフォルト) + -yt-emoji=off + コメントにemojiを表示しない +・音声のみ録画対応 +　-nico-limit-bw に audio_high または audio_only を指定してください +・-http-timeout の設定を保存するように修正 +・live2.* -> live.* に修正 +・その他主にコメント関連の修正 + - livedl.exeとsqlite3ファイルが別のフォルダーにある場合、コメント出力時にxmlファイルに + -数字が付かなかったのを修正 + - ニコ生の生放送の最初に取得するコメント数を1000から100に変更した(サーバー側の仕様による) + +20211017.39 +・livedlのあるディレクトリ以外から実行する時カレントディレクトリにconf.dbが作成されるのを修正 +https://egg.5ch.net/test/read.cgi/software/1595715643/922 +例: C:\bin\livedl\livedl.exe を D:\home\tmp をカレントディレクトリとして実行した場合、conf.dbは D:\home\tmp に作成されてしまう + +仕様：conf.dbは実行するlivedlと同じディレクトリに作成する + ただし、オプション -no-chdir が指定された場合はカレントディレクトリにconf.dbを作成する + (livedl実行ファイルがユーザ書き込み権限のないディレクトリにある場合を想定) + +20210607.38 +・livedl で YouTube Live のアーカイブコメントの取得開始時刻を指定するオプション +https://egg.5ch.net/test/read.cgi/software/1595715643/789 + +使用例： +　livedl -yt-comment-start 3:21:06 https://～ + 特殊例 0：続きからコメント取得 1：最初からコメント取得 + +・livedl で-yt-no-streamlink=on -yt-no-youtube-dl=on が指定されたとき、YouTube Live のコメントを永久に取得し続けるパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/567 + +・livedl を YouTube Live の直近の仕様変更に対応 +https://egg.5ch.net/test/read.cgi/software/1595715643/559 + +・金額のフォーマットの要望ないみたいだからこっちで勝手に決めさせてもらったよ +https://egg.5ch.net/test/read.cgi/software/1595715643/543 + +Youtube Liveのコメントにamount属性を追加 + +・livedl で YouTubeLive リプレイのコメントが取れるよう直したよ +https://egg.5ch.net/test/read.cgi/software/1595715643/523 + +Youtube Liveのコメントが途中で切れるのを修正 +Youtube liveのアーカイブダウンロード中に`json decode error'となり中断するのを修正 +(404エラーになる場合は少しwaitする) + +20210524.37 +・2021/01/01までに5chで公開されたniconico関係の全てのパッチを適用 + +・livedl で一部コメントが保存されないのを修正するパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/457 +　vposが0またはdate_usecが0の場合その要素自体が存在しないのでエラーになりコメントが保存されないのを修正 + +・livedl で YouTube Live を扱えるようにするためのパッチ（リビジョン1） +patch は livedl.youtube-r1.patch のみ適用 +https://egg.5ch.net/test/read.cgi/software/1595715643/402 +https://egg.5ch.net/test/read.cgi/software/1595715643/406 + +・livedl で HTTP のタイムアウト時間を変更できるようにするパッチ +https://egg.5ch.net/test/read.cgi/software/1595715643/272 + +・XMLコメントのname属性(出演者が名前付きのコメントする時に使用)を保存するように修正 +https://egg.5ch.net/test/read.cgi/software/1595715643/174 +patch は livedl.comment-name-attribute-r1.patch.gz のみ適用 +https://egg.5ch.net/test/read.cgi/software/1595715643/194 + +・指定時間でタイムシフト録画を停止するためのパッチ（＋α） +https://egg.5ch.net/test/read.cgi/software/1595715643/163 + +オプション +　-nico-ts-start ＜num＞ +　　タイムシフトの録画を指定した再生時間（秒）から開始する +　-nico-ts-stop ＜num＞ +　　タイムシフトの録画を指定した再生時間（秒）で停止する +　上記2つは ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + +　-nico-ts-start-min ＜num＞ +　　タイムシフトの録画を指定した再生時間（分）から開始する +　-nico-ts-stop-min ＜num＞ +　　タイムシフトの録画を指定した再生時間（分）で停止する +　上記2つは ＜時＞:＜分＞ の形式でも指定可能 + +・セルフ追っかけ再生機能その他 +https://egg.5ch.net/test/read.cgi/software/1595715643/57 +　例：http://127.0.0.1:12345/m3u8/2/1200/index.m3u8 +　　現在のシーケンス番号から1200セグメント（リアルタイムの場合30分）戻ったところを再生 + +・追加オプション +　-nico-conv-seqno-start ＜num＞ +　　MP4への変換を指定したセグメント番号から開始する +　-nico-conv-seqno-end ＜num＞ +　　MP4への変換を指定したセグメント番号で終了する +　-nico-conv-force-concat +　　MP4への変換で画質変更または抜けがあっても分割しないように設定 +　-nico-conv-force-concat=on +　　(+) 上記を有効に設定 +　-nico-conv-force-concat=off +　　(+) 上記を無効に設定(デフォルト) + +・　-d2h +　　[実験的] 録画済みのdb(.sqlite3)を視聴するためのHLSサーバを立てる(-db-to-hls) +　　　開始シーケンス番号は（変換ではないが） -nico-conv-seqno-start で指定 +　　　　使用例：$ livedl lvXXXXXXXXX.sqlite3 -d2h -nico-hls-port 12345 -nico-conv-seqno-start 2780 + +20210516.36 +・fix タイムシフト録画時に'getwaybackkey: waybackkey not found' と表示されコメントが保存されない(nnn-revo2012 PR#54) +　- コメントサーバー仕様変更に対応(threadId、waybackkey廃止など)(2020/07/27) +　　5chで公開されたID:jM/9Q+5+0作成のpatchを適用 +　　https://egg.5ch.net/test/read.cgi/software/1570634489/932 +　- livedl で waybackkey の取得方法を変更するパッチ +　　https://egg.5ch.net/test/read.cgi/software/1595715643/424 + +・ニコニコ仕様変更を反映（#49 5chボランティア (sangwon-jung-work PR#51) +　- 放送情報取得時のwebsocketプロトコルが変わって録画できなくなったのを修正(2020/06/02) +　　5chで公開されたID:jM/9Q+5+0作成のpatchを適用 +　　http://egg.5ch.net/test/read.cgi/software/1570634489/535 +　- fix broadcastId not found error, add debug log + +・20181215.35以降の修正を追加 +・TS録画時にセグメント抜けが起こるのを修正 (PR#47) +・http -> httpsに修正 (PR#39) + 20181215.35 ・-nico-ts-start-minオプションの追加 ・win32bit版のビルドを追加 diff --git a/src/buildno/buildno.go b/src/buildno/buildno.go index c527d39..c966e55 100644 --- a/src/buildno/buildno.go +++ b/src/buildno/buildno.go @@ -1,5 +1,5 @@ package buildno -var BuildDate = "20181215" -var BuildNo = "35" +var BuildDate = "20221122" +var BuildNo = "42" diff --git a/src/httpbase/httpbase.go b/src/httpbase/httpbase.go index 770b0eb..cc86073 100644 --- a/src/httpbase/httpbase.go +++ b/src/httpbase/httpbase.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net/http" "net/url" + "net/http/cookiejar" "os" "strings" "time" @@ -136,8 +137,13 @@ func SetProxy(rawurl string) (err error) { Client.Transport.(*http.Transport).Proxy = http.ProxyURL(u) return } +func SetTimeout(timeout int) (err error) { + err = nil + Client.Timeout = time.Duration(timeout) * time.Second + return +} -func httpBase(method, uri string, header map[string]string, body io.Reader) (resp *http.Response, err, neterr error) { +func httpBase(method, uri string, header map[string]string, jar *cookiejar.Jar, body io.Reader) (resp *http.Response, err, neterr error) { req, err := http.NewRequest(method, uri, body) if err != nil { return @@ -149,6 +155,10 @@ func httpBase(method, uri string, header map[string]string, body io.Reader) (res req.Header.Set(k, v) } + if (jar != nil) { + Client.Jar = jar + } + resp, neterr = Client.Do(req) if neterr != nil { if strings.Contains(neterr.Error(), "x509: certificate signed by unknown") { @@ -159,17 +169,17 @@ func httpBase(method, uri string, header map[string]string, body io.Reader) (res } return } -func Get(uri string, header map[string]string) (*http.Response, error, error) { - return httpBase("GET", uri, header, nil) +func Get(uri string, header map[string]string, jar *cookiejar.Jar) (*http.Response, error, error) { + return httpBase("GET", uri, header, jar, nil) } -func PostForm(uri string, header map[string]string, val url.Values) (*http.Response, error, error) { +func PostForm(uri string, header map[string]string, jar *cookiejar.Jar, val url.Values) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } header["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8" - return httpBase("POST", uri, header, strings.NewReader(val.Encode())) + return httpBase("POST", uri, header, jar, strings.NewReader(val.Encode())) } -func reqJson(method, uri string, header map[string]string, data interface{}) ( +func reqJson(method, uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) ( *http.Response, error, error) { encoded, err := json.Marshal(data) if err != nil { @@ -181,22 +191,22 @@ func reqJson(method, uri string, header map[string]string, data interface{}) ( } header["Content-Type"] = "application/json" - return httpBase(method, uri, header, bytes.NewReader(encoded)) + return httpBase(method, uri, header, jar, bytes.NewReader(encoded)) } -func PostJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { - return reqJson("POST", uri, header, data) +func PostJson(uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) (*http.Response, error, error) { + return reqJson("POST", uri, header, jar, data) } -func PutJson(uri string, header map[string]string, data interface{}) (*http.Response, error, error) { - return reqJson("PUT", uri, header, data) +func PutJson(uri string, header map[string]string, jar *cookiejar.Jar, data interface{}) (*http.Response, error, error) { + return reqJson("PUT", uri, header, jar, data) } -func PostData(uri string, header map[string]string, data io.Reader) (*http.Response, error, error) { +func PostData(uri string, header map[string]string, jar *cookiejar.Jar, data io.Reader) (*http.Response, error, error) { if header == nil { header = make(map[string]string) } - return httpBase("POST", uri, header, data) + return httpBase("POST", uri, header, jar, data) } -func GetBytes(uri string, header map[string]string) (code int, buff []byte, err, neterr error) { - resp, err, neterr := Get(uri, header) +func GetBytes(uri string, header map[string]string, jar *cookiejar.Jar) (code int, buff []byte, err, neterr error) { + resp, err, neterr := Get(uri, header, jar) if err != nil { return } diff --git a/src/livedl.go b/src/livedl.go index ca06fe0..76c7e33 100644 --- a/src/livedl.go +++ b/src/livedl.go @@ -45,17 +45,36 @@ func main() { baseDir = filepath.Dir(pa) } - opt := options.ParseArgs() + // check option only -no-chdir + args := os.Args[1:] + nochdir := false + r := regexp.MustCompile(`\A(?i)--?no-?chdir\z`) + for _, s := range args { + if r.MatchString(s) { + nochdir = true + break + } + } // chdir if not disabled - if !opt.NoChdir { + if !nochdir { fmt.Printf("chdir: %s\n", baseDir) if e := os.Chdir(baseDir); e != nil { fmt.Println(e) return } + } else { + fmt.Printf("no chdir\n") + pwd, e := os.Getwd() + if e != nil { + fmt.Println(e) + return + } + fmt.Printf("read %s\n", filepath.FromSlash(pwd+"/conf.db")) } + opt := options.ParseArgs() + // http if opt.HttpRootCA != "" { if err := httpbase.SetRootCA(opt.HttpRootCA); err != nil { @@ -129,7 +148,7 @@ func main() { } case "YOUTUBE": - err := youtube.Record(opt.YoutubeId, opt.YtNoStreamlink, opt.YtNoYoutubeDl) + err := youtube.Record(opt.YoutubeId, opt.YtNoStreamlink, opt.YtNoYoutubeDl, opt.YtCommentStart) if err != nil { fmt.Println(err) } @@ -141,17 +160,17 @@ func main() { os.Exit(1) } if hlsPlaylistEnd && opt.NicoAutoConvert { - done, nMp4s, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb) + done, nMp4s, skipped, err := zip2mp4.ConvertDB(dbname, opt.ConvExt, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd) if err != nil { fmt.Println(err) os.Exit(1) } if done { - if nMp4s == 1 { + if nMp4s == 1 && (! skipped) { if 1 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } - } else if 1 < nMp4s { + } else if 1 < nMp4s || (nMp4s == 1 && skipped) { if 2 <= opt.NicoAutoDeleteDBMode { os.Remove(dbname) } @@ -172,20 +191,45 @@ func main() { case "DB2MP4": if strings.HasSuffix(opt.DBFile, ".yt.sqlite3") { - zip2mp4.YtComment(opt.DBFile) + zip2mp4.YtComment(opt.DBFile, opt.YtEmoji) } else if opt.ExtractChunks { - if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb); err != nil { + if _, err := zip2mp4.ExtractChunks(opt.DBFile, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { fmt.Println(err) os.Exit(1) } } else { - if _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb); err != nil { + if _, _, _, err := zip2mp4.ConvertDB(opt.DBFile, opt.ConvExt, opt.NicoSkipHb, opt.NicoAdjustVpos, opt.NicoConvForceConcat, opt.NicoConvSeqnoStart, opt.NicoConvSeqnoEnd); err != nil { fmt.Println(err) os.Exit(1) } } + + case "DB2HLS": + if opt.NicoHlsPort == 0 { + fmt.Println("HLS port not specified") + os.Exit(1) + } + if err := zip2mp4.ReplayDB(opt.DBFile, opt.NicoHlsPort, opt.NicoConvSeqnoStart); err != nil { + fmt.Println(err) + os.Exit(1) + } + + case "DBINFO": + if strings.HasSuffix(opt.DBFile, ".yt.sqlite3") { + if _, err := youtube.ShowDbInfo(opt.DBFile); err != nil { + fmt.Println(err) + os.Exit(1) + } + + } else { + if _, err := niconico.ShowDbInfo(opt.DBFile, opt.ConvExt); err != nil { + fmt.Println(err) + os.Exit(1) + } + } + } return diff --git a/src/niconico/jikken.gox b/src/niconico/jikken.gox deleted file mode 100644 index f1f78d2..0000000 --- a/src/niconico/jikken.gox +++ /dev/null @@ -1,270 +0,0 @@ - - -package niconico - -import ( - "fmt" - "os" - "time" - "os/signal" - "syscall" - "net/http" - "io/ioutil" - "log" - "encoding/json" - "bytes" - "../options" - "../obj" - "../files" -) - - -func getActionTrackId() (actionTrackId string, err error) { - uri := "https://public.api.nicovideo.jp/v1/action-track-ids.json" - req, _ := http.NewRequest("POST", uri, nil) - - req.Header.Set("Content-Type", "application/json") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - var props interface{} - if err = json.Unmarshal(bs, &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - data, ok := obj.FindString(props, "data") - if (! ok) { - err = fmt.Errorf("actionTrackId not found") - } - actionTrackId = data - - return -} - -func jikkenWatching(opt options.Option, actionTrackId string, isArchive bool) (props interface{}, err error) { - - str, _ := json.Marshal(OBJ{ - "actionTrackId": actionTrackId, - "isBroadcaster": false, - "isLowLatencyStream": true, - "streamCapacity": "superhigh", - "streamProtocol": "https", - "streamQuality": "auto", // high, auto - }) - if err != nil { - log.Println(err) - return - } - - data := bytes.NewReader(str) - - var uri string - if isArchive { - uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching-archive", opt.NicoLiveId) - } else { - uri = fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) - } - req, _ := http.NewRequest("POST", uri, data) - - //if opt.NicoSession != "" { - req.Header.Set("Cookie", "user_session=" + opt.NicoSession) - //} - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://cas.nicovideo.jp") - req.Header.Set("X-Connection-Environment", "ethernet") - req.Header.Set("X-Frontend-Id", "91") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - if err = json.Unmarshal([]byte(bs), &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - return -} - - -func jikkenPut(opt options.Option, actionTrackId string) (forbidden, notOnAir bool, err error) { - str, _ := json.Marshal(OBJ{ - "actionTrackId": actionTrackId, - "isBroadcaster": false, - }) - if err != nil { - log.Println(err) - } - fmt.Printf("\n%s\n\n", str) - - data := bytes.NewReader(str) - - uri := fmt.Sprintf("https://api.cas.nicovideo.jp/v1/services/live/programs/%s/watching", opt.NicoLiveId) - req, _ := http.NewRequest("PUT", uri, data) - - //if opt.NicoSession != "" { - req.Header.Set("Cookie", "user_session=" + opt.NicoSession) - //} - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://cas.nicovideo.jp") - req.Header.Set("X-Frontend-Id", "91") - - client := new(http.Client) - resp, e := client.Do(req) - if e != nil { - err = e - return - } - defer resp.Body.Close() - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println(err) - } - - var props interface{} - if err = json.Unmarshal([]byte(bs), &props); err != nil { - return - } - - //obj.PrintAsJson(props) - - if code, ok := obj.FindString(props, "meta", "errorCode"); ok { - switch code { - case "FORBIDDEN": - forbidden = true - return - case "PROGRAM_NOT_ONAIR": - notOnAir = true - return - } - } - - return -} - - -func jikkenHousou(nicoliveProgramId, title, userId, nickname, communityId string, opt options.Option, isArchive bool) (err error) { - - chInterrupt := make(chan os.Signal, 10) - signal.Notify(chInterrupt, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) - - actionTrackId, err := getActionTrackId() - if err != nil { - log.Println(err) - } - - media := &NicoMedia{} - - defer func() { - if media.zipWriter != nil { - media.zipWriter.Close() - } - }() - - title = files.ReplaceForbidden(title) - nickname = files.ReplaceForbidden(nickname) - media.fileName = fmt.Sprintf("%s-%s-%s.zip", nicoliveProgramId, nickname, title) - - - var nLast int - L_main: for { - select { - case <-chInterrupt: - break L_main - default: - } - props, e := jikkenWatching(opt, actionTrackId, isArchive) - if e != nil { - err = e - log.Println(err) - return - } - - if uri, ok := obj.FindString(props, "data", "streamServer", "url"); ok { - //fmt.Println(uri) - - is403, e := media.SetPlaylist(uri) - if is403 { - break L_main - } - if e != nil { - err = e - log.Println(e) - return - } - } - - L_loc: for i := 0; true; i++ { - select { - case <-chInterrupt: - break L_main - default: - } - - is403, e := media.GetMedias() - if is403 { - n := media.getNumChunk() - if n != nLast { - nLast = n - break L_loc - } else { - break L_main - } - } - if e != nil { - log.Println(e) - return - } - if i > 60 { - forbidden, notOnAir, e := jikkenPut(opt, actionTrackId) - if e != nil { - err = e - log.Println(e) - return - } - if notOnAir { - break L_main - } - if forbidden { - break L_loc - } - i = 0 - } - select { - case <-chInterrupt: - break L_main - case <-time.After(1 * time.Second): - } - } - } - if media.zipWriter != nil { - media.zipWriter.Close() - } - - signal.Stop(chInterrupt) - - return -} diff --git a/src/niconico/nico.go b/src/niconico/nico.go index bec3ee4..aee19a3 100644 --- a/src/niconico/nico.go +++ b/src/niconico/nico.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "net/http" + "net/http/cookiejar" _ "net/http/pprof" "net/url" "os" @@ -21,7 +22,16 @@ import ( "github.com/himananiito/livedl/options" ) +func joinCookie(cookies []*http.Cookie) (result string) { + result = "" + for _, v := range cookies { + result += v.String() + "; " + } + return result +} + func NicoLogin(opt options.Option) (err error) { + id, pass, _, _ := options.LoadNicoAccount(opt.NicoLoginAlias) if id == "" || pass == "" { @@ -29,10 +39,19 @@ func NicoLogin(opt options.Option) (err error) { return } + jar, err := cookiejar.New(nil) + if err != nil { + return + } + resp, err, neterr := httpbase.PostForm( - "https://account.nicovideo.jp/api/v1/login", - nil, - url.Values{"mail_tel": {id}, "password": {pass}, "site": {"nicoaccountsdk"}}, + "https://account.nicovideo.jp/login/redirector?show_button_twitter=1&site=niconico&show_button_facebook=1&next_url=%2F", + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar, + url.Values{"mail_tel": {id}, "password": {pass}}, ) if err != nil { return @@ -43,18 +62,112 @@ func NicoLogin(opt options.Option) (err error) { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return + // cookieによって判定 + if opt.NicoDebug { + fmt.Fprintf(os.Stderr, "%v\n", resp.Request.Response.Header) + fmt.Fprintln(os.Stderr, "StatusCode:", resp.StatusCode) } - - if ma := regexp.MustCompile(`(.+?)`).FindSubmatch(body); len(ma) > 0 { + //fmt.Println("StatusCode:", resp.Request.Response.StatusCode) // 302 + set_cookie_url, _ := url.Parse("https://www.nicovideo.jp/") + cookie := joinCookie(jar.Cookies(set_cookie_url)) + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "cookie:", cookie) + } + var body []byte + if ma := regexp.MustCompile(`mfa_session=`).FindStringSubmatch(cookie); len(ma) > 0 { + //2段階認証処理 + fmt.Println("login MFA(2FA)") + loc := resp.Request.Response.Header.Values("Location")[0] + //fmt.Fprintln(os.Stderr, "Location:",loc) + resp, err, neterr = httpbase.Get( + loc, + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar) + if err != nil { + err = fmt.Errorf("login MFA error") + return + } + if neterr != nil { + err = neterr + return + } + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + err = fmt.Errorf("login MFA read error 1") + return + } + //fmt.Printf("%s", body) + str := string(body) + if ma = regexp.MustCompile(`/mfa\?site=niconico`).FindStringSubmatch(str); len(ma) <= 0 { + err = fmt.Errorf("login MFA read error 2") + return + } + //actionから抜き出して loc にセット + if ma = regexp.MustCompile(`form action=\"([^\"]+)\"`).FindStringSubmatch(str); len(ma) <= 0 { + err = fmt.Errorf("login MFA read error 3") + return + } + loc = "https://account.nicovideo.jp" + ma[1] + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "Location:", loc) + } + //6 digits code を入力 + otp := "" + retry := 3 + for retry > 0 { + fmt.Println("Enter 6 digits code (CANCEL: c/q/x):") + fmt.Scan(&otp) // データを格納する変数のアドレスを指定 + //p = Pattern.compile("^[0-9]{6}$"); + if ma = regexp.MustCompile(`[cqxCQX]{1}`).FindStringSubmatch(otp); len(ma) > 0 { + err = fmt.Errorf("login MFA : cancel") + return + } + if ma = regexp.MustCompile(`^[0-9]{6}$`).FindStringSubmatch(otp); len(ma) > 0 { + retry = 99 + break + } + retry-- + } + //fmt.Println("code:",otp) + if retry <= 0 { + err = fmt.Errorf("login MFA : wrong digits code") + return + } + resp, err, neterr = httpbase.PostForm( + loc, + map[string]string{ + "Origin": "https://account.nicovideo.jp", + "Referer": "https://account.nicovideo.jp/login", + }, + jar, + url.Values{"otp": {otp}}, + ) + if err != nil { + err = fmt.Errorf("login MFA POST error") + return + } + if neterr != nil { + err = neterr + return + } + //結果が302 + cookie = joinCookie(jar.Cookies(set_cookie_url)) + //fmt.Fprintln("StatusCode:", resp.Request.Response.StatusCode) // 302 + if opt.NicoDebug { + fmt.Fprintln(os.Stderr, "StatusCode:", resp.StatusCode) + fmt.Fprintln(os.Stderr, "MFA cookie:", cookie) + } + } + //Cookieからuser_sessionの値を読み込む + if ma := regexp.MustCompile(`user_session=(user_session_.+?);`).FindStringSubmatch(cookie); len(ma) > 0 { + fmt.Println("session_key: ", string(ma[1])) options.SetNicoSession(opt.NicoLoginAlias, string(ma[1])) - fmt.Println("login success") } else { err = fmt.Errorf("login failed: session_key not found") - return } return } @@ -63,46 +176,34 @@ func Record(opt options.Option) (hlsPlaylistEnd bool, dbName string, err error) for i := 0; i < 2; i++ { // load session info - if opt.NicoSession == "" || i > 0 { - _, _, opt.NicoSession, _ = options.LoadNicoAccount(opt.NicoLoginAlias) - } - - if !opt.NicoRtmpOnly { - var done bool - var notLogin bool - var reserved bool - done, hlsPlaylistEnd, notLogin, reserved, dbName, err = NicoRecHls(opt) - if done { - return - } + if opt.NicoCookies != "" { + opt.NicoSession, err = NicoBrowserCookies(opt) if err != nil { return } - if notLogin { - fmt.Println("not_login") - if err = NicoLogin(opt); err != nil { - return - } - continue - } - if reserved { - continue - } + } else if opt.NicoSession == "" || i > 0 { + _, _, opt.NicoSession, _ = options.LoadNicoAccount(opt.NicoLoginAlias) } - if !opt.NicoHlsOnly { - notLogin, e := NicoRecRtmp(opt) - if e != nil { - err = e + var done bool + var notLogin bool + var reserved bool + done, hlsPlaylistEnd, notLogin, reserved, dbName, err = NicoRecHls(opt) + if done { + return + } + if err != nil { + return + } + if notLogin { + fmt.Println("not_login") + if err = NicoLogin(opt); err != nil { return } - if notLogin { - fmt.Println("not_login") - if err = NicoLogin(opt); err != nil { - return - } - continue - } + continue + } + if reserved { + continue } break @@ -126,10 +227,6 @@ func TestRun(opt options.Option) (err error) { }() } - opt.NicoRtmpIndex = map[int]bool{ - 0: true, - } - var nextId func() string if opt.NicoLiveId == "" { @@ -139,7 +236,7 @@ func TestRun(opt options.Option) (err error) { opt.NicoTestTimeout = 12 } - resp, e, nete := httpbase.Get("https://live.nicovideo.jp/api/getalertinfo", nil) + resp, e, nete := httpbase.Get("https://live.nicovideo.jp/api/getalertinfo", nil, nil) if e != nil { err = e return diff --git a/src/niconico/nico_cookies.go b/src/niconico/nico_cookies.go new file mode 100644 index 0000000..11433d6 --- /dev/null +++ b/src/niconico/nico_cookies.go @@ -0,0 +1,166 @@ +package niconico + +import ( + "bufio" + "database/sql" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/himananiito/livedl/options" +) + +func readFirefoxProfiles(file string) (profiles map[string]string, err error) { + + profiles = make(map[string]string) + data, err := os.Open(file) + if err != nil { + err = fmt.Errorf("cookies from browser failed: profile.ini can't read") + return + } + defer data.Close() + + var key string + flag := false + scanner := bufio.NewScanner(data) + for scanner.Scan() { + //fmt.Println(scanner.Text()) + if strings.Index(scanner.Text(), "[Profile") >= 0 { + flag = true + continue + } + if flag { + if ma := regexp.MustCompile(`^Name=(.+)$`).FindStringSubmatch(scanner.Text()); len(ma) > 0 { + key = ma[1] + continue + } + if ma := regexp.MustCompile(`^Path=(.+)$`).FindStringSubmatch(scanner.Text()); len(ma) > 0 { + profiles[key] = ma[1] + flag = false + continue + } + } + } + return +} + +func NicoBrowserCookies(opt options.Option) (sessionkey string, err error) { + + sessionkey = "" + var profname string + var dbfile string + profiles := make(map[string]string) + + fmt.Println("NicoCookies:", opt.NicoCookies) + if ma := regexp.MustCompile(`^([^:]+):?(.*)$`).FindStringSubmatch(opt.NicoCookies); len(ma) > 0 { + profname = ma[2] + } + if len(profname) < 1 { + profname = "default-release" + } + //fmt.Println("Profile:",profname) + + if strings.Index(profname, "cookies.sqlite") < 0 { + //profiles.iniを開く + switch runtime.GOOS { + case "windows": + dbfile = os.Getenv("APPDATA") + "/Mozilla/Firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + case "darwin": + dbfile = os.Getenv("HOME") + "/Library/Application Support/Firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + dbfile = os.Getenv("HOME") + "/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + } + default: + dbfile = os.Getenv("HOME") + "/snap/firefox/common/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + dbfile = os.Getenv("HOME") + "/.mozilla/firefox" + _, err = os.Stat(dbfile) + if os.IsNotExist(err) { + err = fmt.Errorf("cookies from browser failed: firefox profiles not found") + return + } + } + } + + profiles, err = readFirefoxProfiles(dbfile + "/profiles.ini") + if len(profiles) <= 0 { + err = fmt.Errorf("cookies from browser failed: profiles not found") + return + } + if _, ok := profiles[profname]; !ok { + err = fmt.Errorf("cookies from browser failed: profiles not found") + return + } + if file, _ := filepath.Glob(dbfile + "/" + profiles[profname] + "/cookies.sqlite"); len(file) > 0 { + dbfile = file[0] + } + //fmt.Println(dbfile) + + if strings.Index(dbfile, "cookies.sqlite") < 0 { + err = fmt.Errorf("cookies from browser failed: cookies not found") + return + } + } else { + dbfile = profname + } + fmt.Println("cookiefile:", dbfile) + + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + err = fmt.Errorf("cookies from browser failed: cookie not found") + return + } + query := `SELECT name, value FROM moz_cookies WHERE host='.nicovideo.jp'` + rows, err := db.Query(query) + if err != nil { + log.Println(err) + db.Close() + return + } + defer rows.Close() + + result := "" + for rows.Next() { + var name string + var value string + var dest = []interface{}{ + &name, + &value, + } + err = rows.Scan(dest...) + if err != nil { + log.Println(err) + db.Close() + return + } + result += name + "=" + value + "; " + } + db.Close() + + //Cookieからuser_sessionの値を読み込む + if ma := regexp.MustCompile(`user_session=(user_session_.+?);`).FindStringSubmatch(result); len(ma) > 0 { + fmt.Println("session_key: ", string(ma[1])) + sessionkey = string(ma[1]) + fmt.Println("cookies from browser get success") + } else { + err = fmt.Errorf("cookies from browser failed: session_key not found") + } + + return +} diff --git a/src/niconico/nico_db.go b/src/niconico/nico_db.go index a92996e..fba5f49 100644 --- a/src/niconico/nico_db.go +++ b/src/niconico/nico_db.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "path/filepath" "strings" "time" @@ -26,6 +25,7 @@ var SelComment = `SELECT user_id, content, IFNULL(mail, "") AS mail, + %s IFNULL(premium, 0) AS premium, IFNULL(score, 0) AS score, thread, @@ -34,6 +34,17 @@ var SelComment = `SELECT FROM comment ORDER BY date2` +func SelMediaF(seqnoStart, seqnoEnd int64) (ret string) { + ret = `SELECT + seqno, bandwidth, size, data FROM media + WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL` + ret += ` AND seqno >= ` + fmt.Sprint(seqnoStart) + ret += ` AND seqno <= ` + fmt.Sprint(seqnoEnd) + ret += ` ORDER BY seqno` + + return +} + func (hls *NicoHls) dbOpen() (err error) { db, err := sql.Open("sqlite3", hls.dbName) if err != nil { @@ -102,6 +113,7 @@ func (hls *NicoHls) dbCreate() (err error) { user_id TEXT NOT NULL, content TEXT NOT NULL, mail TEXT, + name TEXT, premium INTEGER, score INTEGER, thread TEXT, @@ -216,6 +228,36 @@ func (hls *NicoHls) dbKVSet(k string, v interface{}) { }) } +func (hls *NicoHls) dbKVExist(k string) (res int){ + hls.dbMtx.Lock() + defer hls.dbMtx.Unlock() + query := `SELECT COUNT(*) FROM kvs WHERE k = ?` + hls.db.QueryRow(query, k).Scan(&res) + return +} + +func DbKVGet(db *sql.DB) (data map[string]interface{}) { + data = make(map[string]interface{}) + rows, err := db.Query(`SELECT k,v FROM kvs`) + if err != nil { + log.Println(err) + return + } + defer rows.Close() + + for rows.Next() { + var k string + var v interface{} + err := rows.Scan(&k, &v) + if err != nil { + log.Println(err) + } + data[k] = v + } + + return +} + func (hls *NicoHls) dbInsertReplaceOrIgnore(table string, data map[string]interface{}, replace bool) { var keys []string var qs []string @@ -272,7 +314,7 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { var endTime float64 hls.db.QueryRow(`SELECT v FROM kvs WHERE k = "endTime"`).Scan(&endTime) - when = endTime + when = endTime + 3600 } else { when = float64(date2) / (1000 * 1000) } @@ -280,9 +322,69 @@ func (hls *NicoHls) dbGetFromWhen() (res_from int, when float64) { return } -func WriteComment(db *sql.DB, fileName string, skipHb bool) { +func dbadjustVpos(opentime, offset, date, vpos int64, providerType string) (ret int64) { + ret = vpos + if providerType != "official" { + ret = (date - opentime) * 100 - offset + } else { + ret = vpos - offset + } + return ret +} + +func dbGetCommentRevision(db *sql.DB) (commentRevision int) { + commentRevision = 0 + var nameCount int64 + db.QueryRow(`SELECT COUNT(name) FROM pragma_table_info('comment') WHERE name = 'name'`).Scan(&nameCount) + if nameCount > 0 { + commentRevision = 1 + } + return +} + +func WriteComment(db *sql.DB, fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd, seqOffset int64) { - rows, err := db.Query(SelComment) + var fSelComment = func(revision int) string { + var selAppend string + if revision >= 1 { + selAppend += `IFNULL(name, "") AS name,` + } + return fmt.Sprintf(SelComment, selAppend) + } + + commentRevision := dbGetCommentRevision(db) + fmt.Println("commentRevision: ", commentRevision) + + //adjustVposの場合はkvsテーブルから読み込み + var openTime int64 + var providerType string + var offset int64 + kvs := DbKVGet(db) + if adjustVpos == true { + var t float64 + var sts string + var serverTime int64 + t = kvs["serverTime"].(float64) + serverTime = int64(t) + t = kvs["openTime"].(float64) + openTime = int64(t) + sts = kvs["status"].(string) + if sts == "ENDED" { + offset = seqnoStart * 500 //timeshift + } else { + offset = (serverTime/10) - (openTime*100) + (seqOffset*150) //on_air + } + providerType = kvs["providerType"].(string) + //fmt.Println("serverTime: ", serverTime) + fmt.Println("status: ", sts) + } + + fmt.Println("adjustVpos: ", adjustVpos) + fmt.Println("providerType: ", providerType) + //fmt.Println("openTime: ", openTime) + //fmt.Println("offset: ", offset) + + rows, err := db.Query(fSelComment(commentRevision)) if err != nil { log.Println(err) return @@ -290,15 +392,12 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") - - dir := filepath.Dir(fileName) - base := filepath.Base(fileName) - base, err = files.GetFileNameNext(base) + fileName, err = files.GetFileNameNext(fileName) + fmt.Println("xml file: ", fileName) if err != nil { fmt.Println(err) os.Exit(1) } - fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) @@ -316,12 +415,13 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { var user_id string var content string var mail string + var name string var premium int64 var score int64 var thread string var origin string var locale string - err = rows.Scan( + var dest0 = []interface{} { &vpos, &date, &date_usec, @@ -330,12 +430,19 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { &user_id, &content, &mail, + } + var dest1 = []interface{} { &premium, &score, &thread, &origin, &locale, - ) + } + if commentRevision >= 1 { + dest0 = append(dest0, &name) + } + var dest = append(dest0, dest1...) + err = rows.Scan(dest...) if err != nil { log.Println(err) return @@ -346,8 +453,13 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { continue } - if vpos < 0 { - continue + // adjustVposの場合はvpos再計算 + // vposが-1000(-10秒)より小さい場合は出力しない + if adjustVpos == true { + vpos = dbadjustVpos(openTime, offset, date, vpos, providerType) + if vpos <= -1000 { + continue + } } line := fmt.Sprintf( @@ -371,6 +483,12 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { mail = strings.Replace(mail, "<", "<", -1) line += fmt.Sprintf(` mail="%s"`, mail) } + if name != "" { + name = strings.Replace(name, `"`, """, -1) + name = strings.Replace(name, "&", "&", -1) + name = strings.Replace(name, "<", "<", -1) + line += fmt.Sprintf(` name="%s"`, name) + } if origin != "" { origin = strings.Replace(origin, `"`, """, -1) origin = strings.Replace(origin, "&", "&", -1) @@ -399,6 +517,75 @@ func WriteComment(db *sql.DB, fileName string, skipHb bool) { fmt.Fprintf(f, "%s\r\n", ``) } +func ShowDbInfo(fileName, ext string) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") + if err != nil { + return + } + defer db.Close() + + fmt.Println("----- DATABASE info. -----") + fmt.Println("sqlite3 file :", fileName) + for _, tbl := range []string{"kvs", "media", "comment"} { + if !dbIsExistTable(db, tbl) { + fmt.Println("table", tbl, "not found") + } else { + fmt.Println("table", tbl, "exist") + } + } + + fmt.Println("----- broadcast info. -----") + kvs := DbKVGet(db) + if len(kvs) > 0 { + id := kvs["nicoliveProgramId"].(string) + title := kvs["title"].(string) + sts := kvs["status"].(string) + ptype := kvs["providerType"].(string) + open := int64(kvs["openTime"].(float64)) + begin := int64(kvs["beginTime"].(float64)) + end := int64(kvs["endTime"].(float64)) + username := kvs["userName"].(string) + + fmt.Println("id: ", id) + fmt.Println("title: ", title) + fmt.Println("username: ", username) + fmt.Println("providerType: ", ptype) + fmt.Println("status: ", sts) + fmt.Println("openTime: ", time.Unix(open, 0)) + if ptype == "official" { + fmt.Println("beginTime: ", time.Unix(begin, 0)) + } + fmt.Println("endTime: ", time.Unix(end, 0)) + } else { + fmt.Println("kvs data not found") + } + commentRevision := dbGetCommentRevision(db) + fmt.Println("commentRevision: ", commentRevision) + + media_all := DbGetCountMedia(db , 0) + media_err := DbGetCountMedia(db , 2) + media_sseq := DbGetFirstSeqNo(db , 0) + media_eseq := DbGetLastSeqNo(db , 0) + comm_data := DbGetCountComment(db) + + fmt.Println("----- media info. -----") + fmt.Println("start seqno: ", media_sseq) + fmt.Println("end seqno: ", media_eseq) + fmt.Println("data: ", media_all, "(media:", media_all - media_err, "err:", media_err, ")") + + fmt.Println("----- comment info. -----") + fmt.Println("data: ", comm_data) + + done = true + + return +} + // ts func (hls *NicoHls) dbGetLastMedia(i int) (res []byte) { hls.dbMtx.Lock() @@ -406,9 +593,63 @@ func (hls *NicoHls) dbGetLastMedia(i int) (res []byte) { hls.db.QueryRow("SELECT data FROM media WHERE seqno = ?", i).Scan(&res) return } -func (hls *NicoHls) dbGetLastSeqNo() (res int64) { +// +func (hls *NicoHls) dbGetLastSeqNo(flg int) (res int64) { hls.dbMtx.Lock() defer hls.dbMtx.Unlock() - hls.db.QueryRow("SELECT seqno FROM media ORDER BY seqno DESC LIMIT 1").Scan(&res) + var sql string + if flg == 1 { + sql = "SELECT seqno FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL ORDER BY seqno DESC LIMIT 1" + } else { + sql = "SELECT seqno FROM media ORDER BY seqno DESC LIMIT 1" + } + hls.db.QueryRow(sql).Scan(&res) + return +} +func DbGetLastSeqNo(db *sql.DB, flg int) (res int64) { + var sql string + if flg == 1 { + sql = "SELECT seqno FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL ORDER BY seqno DESC LIMIT 1" + } else { + sql = "SELECT seqno FROM media ORDER BY seqno DESC LIMIT 1" + } + db.QueryRow(sql).Scan(&res) + return +} +func DbGetFirstSeqNo(db *sql.DB, flg int) (res int64) { + var sql string + if flg == 1 { + sql = "SELECT seqno FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL ORDER BY seqno ASC LIMIT 1" + } else { + sql = "SELECT seqno FROM media ORDER BY seqno ASC LIMIT 1" + } + db.QueryRow(sql).Scan(&res) + return +} +func DbGetCountMedia(db *sql.DB, flg int) (res int64) { + var sql string + if flg == 1 { + sql = "SELECT COUNT(seqno) FROM media WHERE IFNULL(notfound, 0) == 0 AND data IS NOT NULL" + } else if flg == 2 { + sql = "SELECT COUNT(seqno) FROM media WHERE IFNULL(notfound, 0) != 0 OR data IS NULL" + } else { + sql = "SELECT COUNT(seqno) FROM media" + } + db.QueryRow(sql).Scan(&res) + return +} +func DbGetCountComment(db *sql.DB) (res int64) { + db.QueryRow("SELECT COUNT(date) FROM comment").Scan(&res) + return +} +func dbIsExistTable(db *sql.DB, table_name string) (ret bool) { + var res int + ret = false + if len(table_name) > 0 { + db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE TYPE='table' AND name=?", table_name).Scan(&res) + if res > 0 { + ret = true + } + } return } diff --git a/src/niconico/nico_hls.go b/src/niconico/nico_hls.go index 09b09f6..7e86412 100644 --- a/src/niconico/nico_hls.go +++ b/src/niconico/nico_hls.go @@ -47,14 +47,11 @@ type playlist struct { position float64 } type NicoHls struct { - wsapi int - startDelay int playlist playlist - nicoliveProgramId string - webSocketUrl string - myUserId string + webSocketUrl string + myUserId string commentStarted bool mtxCommentStarted sync.Mutex @@ -77,6 +74,7 @@ type NicoHls struct { isTimeshift bool timeshiftStart float64 + timeshiftStop int fastTimeshift bool ultrafastTimeshift bool @@ -112,24 +110,12 @@ func debug_Now() string { } func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err error) { - nicoliveProgramId, ok := prop["nicoliveProgramId"].(string) - if !ok { - err = fmt.Errorf("nicoliveProgramId is not string") - return - } - webSocketUrl, ok := prop["//webSocketUrl"].(string) if !ok { err = fmt.Errorf("webSocketUrl is not string") return } - wsapi := 2 - if m := regexp.MustCompile(`/wsapi/v1/`).FindStringSubmatch(webSocketUrl); len(m) > 0 { - wsapi = 1 - log.Println("wsapi: 1") - } - myUserId, _ := prop["//myId"].(string) if myUserId == "" { myUserId = "NaN" @@ -140,15 +126,6 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err timeshift = true } - if wsapi == 2 && false && !timeshift { - if m := regexp.MustCompile(`/watch/([^?]+)`).FindStringSubmatch(webSocketUrl); len(m) > 0 { - nicoliveProgramId = m[1] - } - webSocketUrl = strings.Replace(webSocketUrl, "/wsapi/v2/", "/wsapi/v1/", 1) - wsapi = 1 - log.Println("wsapi: 1") - } - var pid string if nicoliveProgramId, ok := prop["nicoliveProgramId"]; ok { pid, _ = nicoliveProgramId.(string) @@ -254,23 +231,30 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err files.MkdirByFileName(dbName) - hls = &NicoHls{ - wsapi: wsapi, + var quality string + var limitBw int + if m := regexp.MustCompile(`^(audio_only|audio_high)$`).FindStringSubmatch(opt.NicoLimitBw); len(m) > 0 { + quality = m[0] + limitBw = 0 + } else { + quality = "abr" + limitBw, _ = strconv.Atoi(opt.NicoLimitBw) + } - nicoliveProgramId: nicoliveProgramId, - webSocketUrl: webSocketUrl, - myUserId: myUserId, + hls = &NicoHls{ + webSocketUrl: webSocketUrl, + myUserId: myUserId, - quality: "abr", + quality: quality, dbName: dbName, isTimeshift: timeshift, fastTimeshift: opt.NicoFastTs || opt.NicoUltraFastTs, - ultrafastTimeshift: opt.NicoUltraFastTs, + ultrafastTimeshift: false, NicoSession: opt.NicoSession, - limitBw: opt.NicoLimitBw, - limitBwOrig: opt.NicoLimitBw, + limitBw: limitBw, + limitBwOrig: limitBw, nicoDebug: opt.NicoDebug, gmPlst: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), @@ -279,6 +263,7 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err gmMain: gorman.WithChecker(func(c int) { hls.checkReturnCode(c) }), timeshiftStart: opt.NicoTsStart, + timeshiftStop: opt.NicoTsStop, } hls.fastTimeshiftOrig = hls.fastTimeshift @@ -305,10 +290,14 @@ func NewHls(opt options.Option, prop map[string]interface{}) (hls *NicoHls, err // 放送情報をdbに入れる。自身のユーザ情報は入れない // dbに入れたくないデータはキーの先頭を//としている - for k, v := range prop { - if !strings.HasPrefix(k, "//") { - hls.dbKVSet(k, v) + // 生放送の場合はデータが既にあればupdateしない + if hls.isTimeshift || hls.dbKVExist("serverTime") == 0 { + for k, v := range prop { + if !strings.HasPrefix(k, "//") { + hls.dbKVSet(k, v) + } } + //fmt.Println("Write dbKVSet") } return @@ -332,8 +321,11 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { return } //fmt.Printf("%#v\n", attrMap) - if vpos_f, ok := attrMap["vpos"].(float64); ok { - vpos := int64(vpos_f) + if tag == "chat" { + var vpos int64 + if d, ok := attrMap["vpos"].(float64); ok { + vpos = int64(d) + } var date int64 if d, ok := attrMap["date"].(float64); ok { date = int64(d) @@ -362,15 +354,16 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { } hls.dbInsert("comment", map[string]interface{}{ - "vpos": attrMap["vpos"], - "date": attrMap["date"], - "date_usec": attrMap["date_usec"], + "vpos": vpos, + "date": date, + "date_usec": date_usec, "date2": date2, "no": attrMap["no"], "anonymity": attrMap["anonymity"], "user_id": attrMap["user_id"], "content": attrMap["content"], "mail": attrMap["mail"], + "name": attrMap["name"], "premium": attrMap["premium"], "score": attrMap["score"], "thread": thread, @@ -379,10 +372,14 @@ func (hls *NicoHls) commentHandler(tag string, attr interface{}) (err error) { "hash": hash, }) } else { - if d, ok := attrMap["thread"].(float64); ok { - hls.dbKVSet("comment/thread", fmt.Sprintf("%.f", d)) - } else if s, ok := attrMap["thread"].(string); ok { - hls.dbKVSet("comment/thread", s) + // 生放送の場合はデータが既にあればupdateしない + if hls.isTimeshift || hls.dbKVExist("comment/thread") == 0 { + if d, ok := attrMap["thread"].(float64); ok { + hls.dbKVSet("comment/thread", fmt.Sprintf("%.f", d)) + } else if s, ok := attrMap["thread"].(string); ok { + hls.dbKVSet("comment/thread", s) + } + //fmt.Println("Write dbKVSet(command/thread)") } } @@ -399,6 +396,7 @@ const ( MAIN_INVALID_STREAM_QUALITY MAIN_TEMPORARILY_ERROR PLAYLIST_END + TIMESHIFT_STOP PLAYLIST_403 PLAYLIST_ERROR DELAY @@ -488,6 +486,20 @@ func (hls *NicoHls) markRestartMain(delay int) { } func (hls *NicoHls) checkReturnCode(code int) { // NEVER restart goroutines here except interrupt handler + var fPlaylistEnd = func() { + hls.finish = true + if hls.isTimeshift { + if hls.commentDone { + hls.stopPCGoroutines() + } else if !hls.getCommentStarted() { + hls.stopPCGoroutines() + } else { + fmt.Println("waiting comment") + } + } else { + hls.stopPCGoroutines() + } + } switch code { case NETWORK_ERROR, MAIN_TEMPORARILY_ERROR: delay := hls.getStartDelay() @@ -516,18 +528,11 @@ func (hls *NicoHls) checkReturnCode(code int) { case PLAYLIST_END: fmt.Println("playlist end.") - hls.finish = true - if hls.isTimeshift { - if hls.commentDone { - hls.stopPCGoroutines() - } else if !hls.getCommentStarted() { - hls.stopPCGoroutines() - } else { - fmt.Println("waiting comment") - } - } else { - hls.stopPCGoroutines() - } + fPlaylistEnd() + + case TIMESHIFT_STOP: + fmt.Println("timeshift stop.") + fPlaylistEnd() case MAIN_WS_ERROR: hls.stopPGoroutines() @@ -638,26 +643,6 @@ func (hls *NicoHls) waitAllGoroutines() { hls.waitMGoroutines() } -func (hls *NicoHls) getwaybackkey(threadId string) (waybackkey string, neterr, err error) { - - uri := fmt.Sprintf("https://live.nicovideo.jp/api/getwaybackkey?thread=%s", url.QueryEscape(threadId)) - resp, err, neterr := httpbase.Get(uri, map[string]string{"Cookie": "user_session=" + hls.NicoSession}) - if err != nil { - return - } - if neterr != nil { - return - } - defer resp.Body.Close() - - dat, neterr := ioutil.ReadAll(resp.Body) - if neterr != nil { - return - } - - waybackkey = strings.TrimPrefix(string(dat), "waybackkey=") - return -} func (hls *NicoHls) getTsCommentFromWhen() (res_from int, when float64) { return hls.dbGetFromWhen() } @@ -687,7 +672,7 @@ func (hls *NicoHls) startComment(messageServerUri, threadId, waybackkey string) conn, _, err := websocket.DefaultDialer.Dial( messageServerUri, map[string][]string{ - "Origin": []string{"https://live2.nicovideo.jp"}, + "Origin": []string{"https://live.nicovideo.jp"}, "Sec-WebSocket-Protocol": []string{"msg.nicovideo.jp#json"}, "User-Agent": []string{httpbase.GetUserAgent()}, }, @@ -814,7 +799,7 @@ func (hls *NicoHls) startComment(messageServerUri, threadId, waybackkey string) OBJ{"thread": OBJ{ "fork": 0, "nicoru": 0, - "res_from": -1000, + "res_from": -100, "scores": 1, "thread": threadId, "user_id": hls.myUserId, @@ -884,7 +869,7 @@ func getStringBase(uri string, header map[string]string) (s string, code int, t t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.Get(uri, header) + resp, err, neterr := httpbase.Get(uri, header, nil) if err != nil { return } @@ -916,7 +901,7 @@ func postStringHeader(uri string, header map[string]string, val url.Values) (s s t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.PostForm(uri, header, val) + resp, err, neterr := httpbase.PostForm(uri, header, nil, val) if err != nil { return } @@ -943,7 +928,7 @@ func getBytes(uri string) (code int, buff []byte, t int64, err, neterr error) { t = (time.Now().UnixNano() - start) / (1000 * 1000) }() - resp, err, neterr := httpbase.Get(uri, nil) + resp, err, neterr := httpbase.Get(uri, nil, nil) if err != nil { return } @@ -1039,7 +1024,7 @@ func (hls *NicoHls) saveMedia(seqno int, uri string) (is403, is404, is500 bool, return } -func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, is500 bool, neterr, err error) { +func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, isStop, is500 bool, neterr, err error) { u := argUri.String() m3u8, code, millisec, err, neterr := getString(u) if hls.nicoDebug { @@ -1227,15 +1212,17 @@ func (hls *NicoHls) getPlaylist(argUri *url.URL) (is403, isEnd, is500 bool, nete } if hls.isTimeshift { - if !hls.ultrafastTimeshift { td := seqlist[0].duration * float64(time.Second) hls.playlist.nextTime = time.Now().Add(time.Duration(td)) - } } // prints Current SeqNo if hls.isTimeshift { sec := int(hls.playlist.position) + if hls.timeshiftStop != 0 && sec >= hls.timeshiftStop { + isStop = true + return + } var pos string if sec >= 3600 { pos += fmt.Sprintf("%02d:%02d:%02d", sec/3600, (sec%3600)/60, sec%60) @@ -1485,7 +1472,7 @@ func (hls *NicoHls) startPlaylist(uri string) { //fmt.Println(uri) - is403, isEnd, is500, neterr, err := hls.getPlaylist(uri) + is403, isEnd, isStop, is500, neterr, err := hls.getPlaylist(uri) if neterr != nil { if !hls.interrupted() { log.Println("playlist:", e) @@ -1510,6 +1497,9 @@ func (hls *NicoHls) startPlaylist(uri string) { if isEnd { return PLAYLIST_END } + if isStop { + return TIMESHIFT_STOP + } case <-sig: return GOT_SIGNAL @@ -1519,11 +1509,6 @@ func (hls *NicoHls) startPlaylist(uri string) { }) } func (hls *NicoHls) startMain() { - if hls.wsapi == 1 { - hls.startMainV1() - return - } - // エラー時はMAIN_*を返すこと hls.startPGoroutine(func(sig <-chan struct{}) int { if hls.nicoDebug { @@ -1787,7 +1772,6 @@ func (hls *NicoHls) startMain() { return OK }) } - func (hls *NicoHls) startMainV1() { return // old startMain } @@ -1800,23 +1784,51 @@ func (hls *NicoHls) serve(hlsPort int) { router := gin.Default() router.GET("", func(c *gin.Context) { - seqno := hls.dbGetLastSeqNo() + c.Redirect(http.StatusMovedPermanently, "/m3u8/2/0/index.m3u8") + c.Abort() + }) + + router.GET("/m3u8/:delay/:shift/index.m3u8", func(c *gin.Context) { + targetDuration := "2" + extInf := "1.5" + if hls.isTimeshift { + targetDuration = "3" + extInf = "3.0" + } + shift, err := strconv.Atoi(c.Param("shift")) + if err != nil { + shift = 0 + } + if shift < 0 { + shift = 0 + } + delay, err := strconv.Atoi(c.Param("delay")) + if err != nil { + delay = 0 + } + if delay < 2 { + delay = 2 + } + if !hls.isTimeshift { + if delay < 4 { + delay = 4 + } + } + seqno := hls.dbGetLastSeqNo(0) - int64(shift) body := fmt.Sprintf( `#EXTM3U #EXT-X-VERSION:3 -#EXT-X-TARGETDURATION:1 +#EXT-X-TARGETDURATION:%s #EXT-X-MEDIA-SEQUENCE:%d -#EXTINF:1.0, -/ts/%d/test.ts - -#EXTINF:1.0, -/ts/%d/test.ts - -#EXTINF:1.0, +`, targetDuration, seqno) + for i := int64(delay); i >= 0; i-- { + body += fmt.Sprintf( + `#EXTINF:%s, /ts/%d/test.ts -`, seqno-2, seqno-2, seqno-1, seqno) +`, extInf, seqno-i) + } c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) return }) @@ -1924,6 +1936,8 @@ func postTsRsvBase(num int, vid, session string) (err error) { header := map[string]string{ "Cookie": "user_session=" + session, + "Referer": "https://live.nicovideo.jp/watch/lv" + vid, + "Origin": "https://live.nicovideo.jp", } dat0, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { @@ -1934,12 +1948,14 @@ func postTsRsvBase(num int, vid, session string) (err error) { } var token string - if ma := regexp.MustCompile( - `TimeshiftActions\.(doRegister|confirmToWatch|moveWatch)\(['"].*?['"]\s*(?:,\s*['"](.+?)['"])`). - FindStringSubmatch(dat0); len(ma) > 0 { - if len(ma) > 2 { - token = ma[2] - } + if ma := regexp.MustCompile(`(ulck_\d+)`).FindStringSubmatch(dat0); len(ma) > 0 { + token = ma[1] + } else if strings.Contains(dat0, "システムエラーが発生しました") { + err = fmt.Errorf("postTsRsv: system error try again in a few minutes") + return + } else if strings.Contains(dat0, "申し込み期限切れ") { + err = fmt.Errorf("postTsRsv: deadline expired") + return } else if strings.Contains(dat0, "視聴済み") { err = fmt.Errorf("postTsRsv: already watched") return @@ -1951,13 +1967,13 @@ func postTsRsvBase(num int, vid, session string) (err error) { // "X-Requested-With": "XMLHttpRequest", // "Origin": "https://live.nicovideo.jp", - // "Referer": fmt.Sprintf("https://live.nicovideo.jp/gate/%s", opt.NicoLiveId), + // "Referer": fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId), // "X-Prototype-Version": "1.6.0.3", var vals url.Values if num == 0 { vals = url.Values{ - "mode": []string{"overwrite"}, + "mode": []string{"auto_register"}, "vid": []string{vid}, "token": []string{token}, "rec_pos": []string{""}, @@ -1971,7 +1987,7 @@ func postTsRsvBase(num int, vid, session string) (err error) { "mode": []string{"use"}, "vid": []string{vid}, "token": []string{token}, - "_": []string{""}, + "": []string{""}, } } @@ -1986,19 +2002,68 @@ func postTsRsvBase(num int, vid, session string) (err error) { fmt.Printf("postTsRsv: status not ok: >>>%s<<<\n", dat1) err = fmt.Errorf("postTsRsv: status not ok") return + } else { + if num == 0 { + fmt.Println("postTsRsv0: status ok") + } else { + fmt.Println("postTsRsv1: status ok") + } + } + + return +} + +func postRsvUseTs(isRsv bool, opt options.Option) (err error) { + + var vid string + if ma := regexp.MustCompile(`lv(\d+)`).FindStringSubmatch(opt.NicoLiveId); len(ma) > 0 { + vid = ma[1] + } + uri := "https://live.nicovideo.jp/api/timeshift.ticket.use" + RsvMsg := "Use" + if isRsv { + uri = "https://live.nicovideo.jp/api/timeshift.reservations"; + RsvMsg = "Rsv" + } + header := map[string]string{ + "Cookie": "user_session=" + opt.NicoSession, + "Accept": "application/json, text/plain */*", + "Referer": fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId), + "Origin": "https://live.nicovideo.jp", + } + vals := url.Values{"vid": []string{vid},} + + dat, _, _, err, neterr := postStringHeader(uri, header, vals) + if err != nil || neterr != nil { + if err == nil { + err = neterr + } + return + } + if !strings.Contains(dat, "status\":200") { + if ma := regexp.MustCompile(`\"description\":\"([^\"]+)\"`).FindStringSubmatch(dat); len(ma) > 0 { + err = fmt.Errorf("postRsvUseTs: %s %s\n", RsvMsg, ma[1]) + } else { + fmt.Printf("postRsvUseTs: status not ok: >>>%s<<<\n", dat) + err = fmt.Errorf("postRsvUseTs: %s status not ok", RsvMsg) + return + } + } else { + fmt.Printf("postRsvUseTs: %s status ok\n", RsvMsg) } return } -func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, tsRsv1 bool, err error) { +func getProps(opt options.Option) (props interface{}, notLogin, rsvTs, useTs bool, err error) { header := map[string]string{} if opt.NicoSession != "" { header["Cookie"] = "user_session=" + opt.NicoSession } - uri := fmt.Sprintf("https://live2.nicovideo.jp/watch/%s", opt.NicoLiveId) + //ログインチェック + uri := "https://www.nicovideo.jp" dat, _, _, err, neterr := getStringHeader(uri, header) if err != nil || neterr != nil { if err == nil { @@ -2023,28 +2088,63 @@ func getProps(opt options.Option) (props interface{}, isFlash, notLogin, tsRsv0, notLogin = true } - // 新配信 + nicocas - if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { - str := html.UnescapeString(string(ma[1])) - if err = json.Unmarshal([]byte(str), &props); err != nil { - return + if notLogin && opt.NicoLoginOnly { + return + } + + // ログインアカウント種別取得 + if ma := regexp.MustCompile(`member_status['"]*\s*[=:]\s*['"](.*?)['"]`).FindStringSubmatch(dat); len(ma) > 0 { + fmt.Println("account:", ma[1]) + } + if notLogin { + fmt.Println("account: not_login") + } + + // 放送ページ取得 + uri = fmt.Sprintf("https://live.nicovideo.jp/watch/%s", opt.NicoLiveId) + dat, _, _, err, neterr = getStringHeader(uri, header) + if err != nil || neterr != nil { + if err == nil { + err = neterr } return - } else if strings.Contains(dat, "nicoliveplayer.swf") { - // 旧Flashプレイヤー - isFlash = true - } else if regexp.MustCompile(`この番組は.{1,50}に終了`).MatchString(dat) { - // タイムシフト予約ボタン - if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.register`).FindStringSubmatch(dat); len(ma) > 0 { - fmt.Printf("timeshift reservation required\n") - tsRsv0 = true - return + } + + // 放送種別(official/channel/community)取得 + var providertype string + // "providerType": "(.+)l", + if ma := regexp.MustCompile(`"providerType":"(\w+)"`).FindStringSubmatch(dat); len(ma) > 0 { + providertype = ma[1] + } + fmt.Println("providertype:", providertype) + + // 新配信 + if regexp.MustCompile(`ご指定のページが見つかりませんでした`).MatchString(dat) { + err = fmt.Errorf("getProps: page not found") + } else if regexp.MustCompile(`(放送者により削除されました|削除された可能性があります)`).MatchString(dat) { + err = fmt.Errorf("getProps: page not found") + } else if ma := regexp.MustCompile(`rejectedReasons":\[([^\]]+)\]`).FindStringSubmatch(dat); len(ma) > 0 { + ttt := strings.ReplaceAll(html.UnescapeString(ma[1]), "\",\"", " ") + if !notLogin && providertype == "official" && regexp.MustCompile(`notHaveTimeshiftTicket`).MatchString(ttt) { + //チケット予約処理 + fmt.Println("notHaveTimeshiftTicket: timeshift reservation required") + rsvTs = true + } else if !notLogin && providertype == "official" && regexp.MustCompile(`notUseTimeshiftTicket`).MatchString(ttt) { + //チケット使用処理 + fmt.Println("notUseTimeshiftTicket: timeshift reservation required") + useTs = true + } else { + err = fmt.Errorf("getProps: %s", strings.Trim(ttt, "\"")) } - if ma := regexp.MustCompile(`Nicolive\.WatchingReservation\.confirm`).FindStringSubmatch(dat); len(ma) > 0 { - fmt.Printf("timeshift reservation required\n") - tsRsv1 = true + } else if regexp.MustCompile(`webSocketUrl":"",`).MatchString(dat) { + err = fmt.Errorf("getProps: webSocketUrl not found") + } else if ma := regexp.MustCompile(`data-props="(.+?)"`).FindStringSubmatch(dat); len(ma) > 0 { + str := html.UnescapeString(string(ma[1])) + if err = json.Unmarshal([]byte(str), &props); err != nil { return } + } else { + err = fmt.Errorf("getProps: unknown error") } return @@ -2055,9 +2155,9 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, //http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 32 //var props interface{} - //var isFlash bool - //var tsRsv bool - props, isFlash, notLogin, tsRsv0, tsRsv1, err := getProps(opt) + //var rsvTs bool + //var useTs bool + props, notLogin, rsvTs, useTs, err := getProps(opt) if err != nil { //fmt.Println(err) return @@ -2074,37 +2174,30 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, } // TS予約必要 - if (tsRsv0 || tsRsv1) && opt.NicoForceResv { - if tsRsv0 { - err = postTsRsv0(opt) - } else { - err = postTsRsv1(opt) + if (rsvTs || useTs) && opt.NicoForceResv { + if rsvTs { + err = postRsvUseTs(true, opt) + if (err != nil) { + return + } } + err = postRsvUseTs(false, opt) if err == nil { reserved = true } return } - if isFlash { - fmt.Println("Flash page detected.") - return - } - if false { objs.PrintAsJson(props) os.Exit(9) } proplist := map[string][]string{ - // "broadcaster" // nicocas - "cas-userName": []string{"broadcaster", "nickname"}, // ユーザ名 - "cas-userPageUrl": []string{"broadcaster", "pageUrl"}, // "https://www.nicovideo.jp/user/\d+" // "community" "comId": []string{"community", "id"}, // "co\d+" // "program" - "beginTime": []string{"program", "beginTime"}, // integer - //"broadcastId": []string{"program", "broadcastId"}, // "\d+" + "beginTime": []string{"program", "beginTime"}, // integer "description": []string{"program", "description"}, // 放送説明 "endTime": []string{"program", "endTime"}, // integer "isFollowerOnly": []string{"program", "isFollowerOnly"}, // bool @@ -2139,79 +2232,36 @@ func NicoRecHls(opt options.Option) (done, playlistEnd, notLogin, reserved bool, v, ok := objs.Find(props, a...) if ok { kv[k] = v - - if opt.NicoDebug { - fmt.Println(k, v) - fmt.Println("----------") - } } } - var nicocas bool - if _, ok := kv["nicocas"]; ok { - nicocas = true - } - - if nicocas { - fmt.Println("nicocas not supported.") - return - - } else { - for _, k := range []string{ - "nicoliveProgramId", - "//webSocketUrl", - //"//myId", - } { - if _, ok := kv[k]; !ok { - fmt.Printf("%v not found\n", k) - return - } - } - - if opt.NicoFormat == "" { - opt.NicoFormat = "?PID?-?UNAME?-?TITLE?" - } - - hls, e := NewHls(opt, kv) - if e != nil { - err = e - fmt.Println(err) + for _, k := range []string{ + "//webSocketUrl", + //"//myId", + } { + if _, ok := kv[k]; !ok { + fmt.Printf("%v not found\n", k) return } - defer hls.Close() - - hls.Wait(opt.NicoTestTimeout, opt.NicoHlsPort) - - dbName = hls.dbName - playlistEnd = hls.finish - done = true } - /* - pageUrl, _ := objs.FindString(props, "broadcaster", "pageUrl") - - if regexp.MustCompile(`\Ahttps?://cas\.nicovideo\.jp/.*?/.*`).MatchString(pageUrl) { - // 実験放送 - userId, ok := objs.FindString(props, "broadcaster", "id") - if ! ok { - fmt.Printf("userId not found") - } - - nickname, ok := objs.FindString(props, "broadcaster", "nickname") - if ! ok { - fmt.Printf("nickname not found") - } + if opt.NicoFormat == "" { + opt.NicoFormat = "?PID?-?UNAME?-?TITLE?" + } - var isArchive bool - switch status { - case "ENDED": - isArchive = true - } + hls, e := NewHls(opt, kv) + if e != nil { + err = e + fmt.Println(err) + return + } + defer hls.Close() - } + hls.Wait(opt.NicoTestTimeout, opt.NicoHlsPort) - log4gui.Info(fmt.Sprintf("isLoggedIn: %v, user_id: %s, nickname: %s", isLoggedIn, user_id, nickname)) - */ + dbName = hls.dbName + playlistEnd = hls.finish + done = true return } diff --git a/src/niconico/nico_rtmp.go b/src/niconico/nico_rtmp.go deleted file mode 100644 index d61cb55..0000000 --- a/src/niconico/nico_rtmp.go +++ /dev/null @@ -1,656 +0,0 @@ -package niconico - -import ( - "encoding/xml" - "fmt" - "io/ioutil" - "log" - "net/url" - "regexp" - "strings" - "sync" - "time" - - "github.com/himananiito/livedl/amf" - "github.com/himananiito/livedl/files" - "github.com/himananiito/livedl/httpbase" - "github.com/himananiito/livedl/options" - "github.com/himananiito/livedl/rtmps" -) - -type Content struct { - Id string `xml:"id,attr"` - Text string `xml:",chardata"` -} -type Tickets struct { - Name string `xml:"name,attr"` - Text string `xml:",chardata"` -} -type Status struct { - Title string `xml:"stream>title"` - CommunityId string `xml:"stream>default_community"` - Id string `xml:"stream>id"` - Provider string `xml:"stream>provider_type"` - IsArchive bool `xml:"stream>archive"` - IsArchivePlayerServer bool `xml:"stream>is_archiveplayserver"` - Ques []string `xml:"stream>quesheet>que"` - Contents []Content `xml:"stream>contents_list>contents"` - IsPremium bool `xml:"user>is_premium"` - Url string `xml:"rtmp>url"` - Ticket string `xml:"rtmp>ticket"` - Tickets []Tickets `xml:"tickets>stream"` - ErrorCode string `xml:"error>code"` - streams []Stream - chStream chan struct{} - wg *sync.WaitGroup -} -type Stream struct { - originUrl string - streamName string - originTicket string -} - -func (status *Status) quesheet() { - stream := make(map[string][]Stream) - playType := make(map[string]string) - - // timeshift; tag - re_pub := regexp.MustCompile(`\A/publish\s+(\S+)\s+(?:(\S+?),)?(\S+?)(?:\?(\S+))?\z`) - re_play := regexp.MustCompile(`\A/play\s+(\S+)\s+(\S+)\z`) - - for _, q := range status.Ques { - // /publish lv* /content/*/lv*_*_1_*.f4v - if ma := re_pub.FindStringSubmatch(q); len(ma) >= 5 { - stream[ma[1]] = append(stream[ma[1]], Stream{ - originUrl: ma[2], - streamName: ma[3], - originTicket: ma[4], - }) - - // /play ... - } else if ma := re_play.FindStringSubmatch(q); len(ma) > 0 { - // /play case:sp:rtmp:lv*_s_lv*,mobile:rtmp:lv*_s_lv*_sub1,premium:rtmp:lv*_s_lv*_sub1,default:rtmp:lv*_s_lv* main - if strings.HasPrefix(ma[1], "case:") { - s0 := ma[1] - s0 = strings.TrimPrefix(s0, "case:") - cases := strings.Split(s0, ",") - // sp:rtmp:lv*_s_lv* - re := regexp.MustCompile(`\A(\S+?):rtmp:(\S+?)\z`) - for _, c := range cases { - if ma := re.FindStringSubmatch(c); len(ma) > 0 { - playType[ma[1]] = ma[2] - } - } - - // /play rtmp:lv* main - } else { - re := regexp.MustCompile(`\Artmp:(\S+?)\z`) - if ma := re.FindStringSubmatch(ma[1]); len(ma) > 0 { - playType["default"] = ma[1] - } - } - } - } - - pt, ok := playType["premium"] - if ok && status.IsPremium { - s, ok := stream[pt] - if ok { - status.streams = s - } - } else { - pt, ok := playType["default"] - if ok { - s, ok := stream[pt] - if ok { - status.streams = s - } - } - } -} -func (status *Status) initStreams() { - - if len(status.streams) > 0 { - return - } - - //if status.isOfficialLive() { - status.contentsOfficialLive() - //} else if status.isLive() { - status.contentsNonOfficialLive() - //} else { - status.quesheet() - //} - - return -} -func (status *Status) getFileName(index int) (name string) { - if len(status.streams) == 1 { - //name = fmt.Sprintf("%s.flv", status.Id) - name = fmt.Sprintf("%s-%s-%s.flv", status.Id, status.CommunityId, status.Title) - } else if len(status.streams) > 1 { - //name = fmt.Sprintf("%s-%d.flv", status.Id, 1 + index) - name = fmt.Sprintf("%s-%s-%s#%d.flv", status.Id, status.CommunityId, status.Title, 1+index) - } else { - log.Fatalf("No stream") - } - name = files.ReplaceForbidden(name) - return -} -func (status *Status) contentsNonOfficialLive() { - re := regexp.MustCompile(`\A(?:rtmp:)?(rtmp\w*://\S+?)(?:,(\S+?)(?:\?(\S+))?)?\z`) - - // Live (not timeshift); tag - for _, c := range status.Contents { - if ma := re.FindStringSubmatch(c.Text); len(ma) > 0 { - status.streams = append(status.streams, Stream{ - originUrl: ma[1], - streamName: ma[2], - originTicket: ma[3], - }) - } - } - -} -func (status *Status) contentsOfficialLive() { - - tickets := make(map[string]string) - for _, t := range status.Tickets { - tickets[t.Name] = t.Text - } - - for _, c := range status.Contents { - if strings.HasPrefix(c.Text, "case:") { - c.Text = strings.TrimPrefix(c.Text, "case:") - - for _, c := range strings.Split(c.Text, ",") { - c, e := url.PathUnescape(c) - if e != nil { - fmt.Printf("%v\n", e) - } - - re := regexp.MustCompile(`\A(\S+?):(?:limelight:|akamai:)?(\S+),(\S+)\z`) - if ma := re.FindStringSubmatch(c); len(ma) > 0 { - fmt.Printf("\n%#v\n", ma) - switch ma[1] { - default: - fmt.Printf("unknown contents case %#v\n", ma[1]) - case "mobile": - case "middle": - case "default": - status.Url = ma[2] - t, ok := tickets[ma[3]] - if !ok { - fmt.Printf("not found %s\n", ma[3]) - } - fmt.Printf("%s\n", t) - status.streams = append(status.streams, Stream{ - streamName: ma[3], - originTicket: t, - }) - } - } - } - } - } -} - -func (status *Status) relayStreamName(i, offset int) (s string) { - s = regexp.MustCompile(`[^/\\]+\z`).FindString(status.streams[i].streamName) - if offset >= 0 { - s += fmt.Sprintf("_%d", offset) - } - return -} - -func (status *Status) streamName(i, offset int) (name string, err error) { - if status.isOfficialLive() { - if i >= len(status.streams) { - err = fmt.Errorf("(status *Status) streamName(i int): Out of index: %d\n", i) - return - } - - name = status.streams[i].streamName - if status.streams[i].originTicket != "" { - name += "?" + status.streams[i].originTicket - } - return - - } else if status.isOfficialTs() { - name = status.streams[i].streamName - name = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(name, "") - if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(name) { - name = "mp4:" + name - } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(name) { - name = "raw:" + name - } - - } else { - name = status.relayStreamName(i, offset) - } - - return -} -func (status *Status) tcUrl() (url string, err error) { - if status.Url != "" { - url = status.Url - return - } else { - status.contentsOfficialLive() - } - - if status.Url != "" { - url = status.Url - return - } - - err = fmt.Errorf("tcUrl not found") - return -} -func (status *Status) isTs() bool { - return status.IsArchive -} -func (status *Status) isLive() bool { - return (!status.IsArchive) -} -func (status *Status) isOfficialLive() bool { - return (status.Provider == "official") && (!status.IsArchive) -} -func (status *Status) isOfficialTs() bool { - if status.IsArchive { - switch status.Provider { - case "official": - return true - case "channel": - return status.IsArchivePlayerServer - } - } - return false -} - -func (st Stream) relayStreamName(offset int) (s string) { - s = regexp.MustCompile(`[^/\\]+\z`).FindString(st.streamName) - if offset >= 0 { - s += fmt.Sprintf("_%d", offset) - } - return -} -func (st Stream) noticeStreamName(offset int) (s string) { - s = st.streamName - s = regexp.MustCompile(`(?i:\.flv)$`).ReplaceAllString(s, "") - if regexp.MustCompile(`(?i:\.(?:f4v|mp4))$`).MatchString(s) { - s = "mp4:" + s - } else if regexp.MustCompile(`(?i:\.raw)$`).MatchString(s) { - s = "raw:" + s - } - - if st.originTicket != "" { - s += "?" + st.originTicket - } - - return -} - -func (status *Status) recStream(index int, opt options.Option) (err error) { - defer func() { - <-status.chStream - status.wg.Done() - }() - - stream := status.streams[index] - - tcUrl, err := status.tcUrl() - if err != nil { - return - } - - rtmp, err := rtmps.NewRtmp( - // tcUrl - tcUrl, - // swfUrl - "http://live.nicovideo.jp/nicoliveplayer.swf?180116154229", - // pageUrl - "http://live.nicovideo.jp/watch/"+status.Id, - // option - status.Ticket, - ) - if err != nil { - return - } - defer rtmp.Close() - - fileName, err := files.GetFileNameNext(status.getFileName(index)) - if err != nil { - return - } - rtmp.SetFlvName(fileName) - - tryRecord := func() (incomplete bool, err error) { - - if err = rtmp.Connect(); err != nil { - return - } - - // default: 2500000 - //if err = rtmp.SetPeerBandwidth(100*1000*1000, 0); err != nil { - if err = rtmp.SetPeerBandwidth(2500000, 0); err != nil { - fmt.Printf("SetPeerBandwidth: %v\n", err) - return - } - - if err = rtmp.WindowAckSize(2500000); err != nil { - fmt.Printf("WindowAckSize: %v\n", err) - return - } - - if err = rtmp.CreateStream(); err != nil { - fmt.Printf("CreateStream %v\n", err) - return - } - - if err = rtmp.SetBufferLength(0, 2000); err != nil { - fmt.Printf("SetBufferLength: %v\n", err) - return - } - - var offset int - if status.IsArchive { - offset = 0 - } else { - offset = -2 - } - - if status.isOfficialTs() { - for i := 0; true; i++ { - if i > 30 { - err = fmt.Errorf("sendFileRequest: No response") - return - } - data, e := rtmp.Command( - "sendFileRequest", []interface{}{ - nil, - amf.SwitchToAmf3(), - []string{ - stream.streamName, - }, - }) - if e != nil { - err = e - return - } - - var resCnt int - switch data.(type) { - case map[string]interface{}: - resCnt = len(data.(map[string]interface{})) - case map[int]interface{}: - resCnt = len(data.(map[int]interface{})) - case []interface{}: - resCnt = len(data.([]interface{})) - case []string: - resCnt = len(data.([]string)) - } - if resCnt > 0 { - break - } - time.Sleep(10 * time.Second) - } - - } else if !status.isOfficialLive() { - // /publishの第二引数 - // streamName(param1:String) - // 「,」で区切る - // ._originUrl, streamName(playStreamName) - // streamName に、「?」がついてるなら originTickt となる - // streamName の.flvは削除する - // streamNameが/\.(f4v|mp4)$/iなら、頭にmp4:をつける - // /\.raw$/iなら、raw:をつける。 - // relayStreamName: streamNameの頭からスラッシュまでを削除したもの - - _, err = rtmp.Command( - "nlPlayNotice", []interface{}{ - nil, - // _connection.request.originUrl - stream.originUrl, - - // this._connection.request.playStreamRequest - // originticket あるなら - // playStreamName ? this._originTicket - // 無いなら playStreamName - stream.noticeStreamName(offset), - - // var _loc1_:String = this._relayStreamName; - // if(this._offset != -2) - // { - // _loc1_ = _loc1_ + ("_" + this.offset); - // } - // user nama: String 'lvxxxxxxxxx' - // user kako: lvxxxxxxxxx_xxxxxxxxxxxx_1_xxxxxx.f4v_0 - stream.relayStreamName(offset), - - // seek offset - // user nama: -2, user kako: 0 - offset, - }) - if err != nil { - fmt.Printf("nlPlayNotice %v\n", err) - return - } - } - - if err = rtmp.SetBufferLength(1, 3600*1000); err != nil { - fmt.Printf("SetBufferLength: %v\n", err) - return - } - - // No return - rtmp.SetFixAggrTimestamp(true) - - // user kako: lv*********_************_*_******.f4v_0 - // official or channel ts: mp4:/content/********/lv*********_************_*_******.f4v - //if err = rtmp.Play(status.origin.playStreamName(status.isTsOfficial(), offset)); err != nil { - streamName, err := status.streamName(index, offset) - if err != nil { - return - } - - if status.isOfficialTs() { - ts := rtmp.GetTimestamp() - if ts > 1000 { - err = rtmp.PlayTime(streamName, ts-1000) - } else { - err = rtmp.PlayTime(streamName, -5000) - } - - } else if status.isTs() { - rtmp.SetFlush(true) - err = rtmp.PlayTime(streamName, -5000) - - } else { - err = rtmp.Play(streamName) - } - if err != nil { - fmt.Printf("Play: %v\n", err) - return - } - - // Non-recordedなタイムシフトでseekしても、timestampが変わるだけで - // 最初からの再生となってしまうのでやらないこと - - // 公式のタイムシフトでSeekしてもタイムスタンプがおかしい - - if opt.NicoTestTimeout > 0 { - // test mode - _, incomplete, err = rtmp.WaitTest(opt.NicoTestTimeout) - } else { - // normal mode - _, incomplete, err = rtmp.Wait() - } - return - } // end func - - //ticketTime := time.Now().Unix() - //rtmp.SetNoSeek(false) - for i := 0; i < 10; i++ { - incomplete, e := tryRecord() - if e != nil { - err = e - fmt.Printf("%v\n", e) - return - } else if incomplete && status.isOfficialTs() { - fmt.Println("incomplete") - time.Sleep(3 * time.Second) - - // update ticket - if true { - //if time.Now().Unix() > ticketTime + 60 { - //ticketTime = time.Now().Unix() - if ticket, e := getTicket(opt); e != nil { - err = e - return - } else { - rtmp.SetConnectOpt(ticket) - } - //} - } - - continue - } - break - } - - fmt.Printf("done\n") - return -} - -func (status *Status) recAllStreams(opt options.Option) (err error) { - - status.initStreams() - - var MaxConn int - if opt.NicoRtmpMaxConn == 0 { - if status.isOfficialTs() { - MaxConn = 1 - } else { - MaxConn = 4 - } - } else if opt.NicoRtmpMaxConn < 0 { - MaxConn = 1 - } else { - MaxConn = opt.NicoRtmpMaxConn - } - - status.wg = &sync.WaitGroup{} - status.chStream = make(chan struct{}, MaxConn) - - ticketTime := time.Now().Unix() - - for index, _ := range status.streams { - if opt.NicoRtmpIndex != nil { - if tes, ok := opt.NicoRtmpIndex[index]; !ok || !tes { - continue - } - } - - // blocks here - status.chStream <- struct{}{} - status.wg.Add(1) - - go status.recStream(index, opt) - - now := time.Now().Unix() - if now > ticketTime+60 { - ticketTime = now - if ticket, e := getTicket(opt); e != nil { - err = e - return - } else { - status.Ticket = ticket - } - } - } - - status.wg.Wait() - - return -} - -func getTicket(opt options.Option) (ticket string, err error) { - status, notLogin, err := getStatus(opt) - if err != nil { - return - } - if status.Ticket != "" { - ticket = status.Ticket - } else { - if notLogin { - err = fmt.Errorf("notLogin") - } else { - err = fmt.Errorf("Ticket not found") - } - } - return -} -func getStatus(opt options.Option) (status *Status, notLogin bool, err error) { - var uri string - - // experimental - if opt.NicoStatusHTTPS { - uri = fmt.Sprintf("https://ow.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) - } else { - uri = fmt.Sprintf("http://watch.live.nicovideo.jp/api/getplayerstatus?v=%s", opt.NicoLiveId) - } - - header := make(map[string]string, 4) - if opt.NicoSession != "" { - header["Cookie"] = "user_session=" + opt.NicoSession - } - - // experimental - //if opt.NicoStatusHTTPS { - // req.Header.Set("User-Agent", "Niconico/1.0 (Unix; U; iPhone OS 10.3.3; ja-jp; nicoiphone; iPhone5,2) Version/6.65") - //} - - resp, err, neterr := httpbase.Get(uri, header) - if err != nil { - return - } - if neterr != nil { - err = neterr - return - } - defer resp.Body.Close() - - dat, _ := ioutil.ReadAll(resp.Body) - status = &Status{} - err = xml.Unmarshal(dat, status) - if err != nil { - //fmt.Println(string(dat)) - fmt.Printf("error: %v", err) - return - } - - switch status.ErrorCode { - case "": - case "notlogin": - notLogin = true - default: - err = fmt.Errorf("Error code: %s\n", status.ErrorCode) - return - } - - return -} - -func NicoRecRtmp(opt options.Option) (notLogin bool, err error) { - status, notLogin, err := getStatus(opt) - if err != nil { - return - } - if notLogin { - return - } - - status.recAllStreams(opt) - return -} diff --git a/src/options/options.go b/src/options/options.go index bc1183f..2fe0bfd 100644 --- a/src/options/options.go +++ b/src/options/options.go @@ -11,12 +11,16 @@ import ( "strconv" "strings" + "github.com/himananiito/livedl/buildno" "github.com/himananiito/livedl/cryptoconf" "github.com/himananiito/livedl/files" + "github.com/himananiito/livedl/httpbase" "golang.org/x/crypto/sha3" ) +const MinimumHttpTimeout = 5 + var DefaultTcasRetryTimeoutMinute = 5 // TcasRetryTimeoutMinute var DefaultTcasRetryInterval = 60 // TcasRetryInterval @@ -26,11 +30,8 @@ type Option struct { NicoStatusHTTPS bool NicoSession string NicoLoginAlias string - NicoRtmpMaxConn int - NicoRtmpOnly bool - NicoRtmpIndex map[int]bool - NicoHlsOnly bool NicoLoginOnly bool + NicoCookies string NicoTestTimeout int TcasId string TcasRetry bool @@ -42,8 +43,9 @@ type Option struct { ZipFile string DBFile string NicoHlsPort int - NicoLimitBw int + NicoLimitBw string NicoTsStart float64 + NicoTsStop int NicoFormat string NicoFastTs bool NicoUltraFastTs bool @@ -52,14 +54,22 @@ type Option struct { NicoDebug bool // デバッグ情報の記録 ConvExt string ExtractChunks bool + NicoConvForceConcat bool + NicoConvSeqnoStart int64 + NicoConvSeqnoEnd int64 NicoForceResv bool // 終了番組の上書きタイムシフト予約 YtNoStreamlink bool + YtCommentStart float64 YtNoYoutubeDl bool + YtEmoji bool NicoSkipHb bool // コメント出力時に/hbコマンドを出さない + NicoAdjustVpos bool // コメント出力時にvposを補正する HttpRootCA string HttpSkipVerify bool HttpProxy string NoChdir bool + HttpTimeout int + } func getCmd() (cmd string) { @@ -92,29 +102,32 @@ COMMAND: -tcas ツイキャスの録画 -yt YouTube Liveの録画 -d2m 録画済みのdb(.sqlite3)をmp4に変換する(-db-to-mp4) + -dbinfo 録画済みのdb(.sqlite3)の各種情報を表示する + e.g. $ livedl -dbinfo -- 'C:/home/hogehoge/livedl/rec/lvxxxxxxxx.sqlite3' + -d2h [実験的] 録画済みのdb(.sqlite3)を視聴するためのHLSサーバを立てる(-db-to-hls) + 開始シーケンス番号は（変換ではないが） -nico-conv-seqno-start で指定 + 使用例：$ livedl lvXXXXXXXXX.sqlite3 -d2h -nico-hls-port 12345 -nico-conv-seqno-start 2780 オプション/option: -h ヘルプを表示 -vh 全てのオプションを表示 -v バージョンを表示 - -no-chdir 起動する時chdirしない + -no-chdir 起動する時chdirしない(conf.dbは起動したディレクトリに作成されます) -- 後にオプションが無いことを指定 ニコニコ生放送録画用オプション: -nico-login , (+) ニコニコのIDとパスワードを指定する + 2段階認証(MFA)に対応しています -nico-session Cookie[user_session]を指定する -nico-login-only=on (+) 必ずログイン状態で録画する -nico-login-only=off (+) 非ログインでも録画可能とする(デフォルト) - -nico-hls-only 録画時にHLSのみを試す - -nico-hls-only=on (+) 上記を有効に設定 - -nico-hls-only=off (+) 上記を無効に設定(デフォルト) - -nico-rtmp-only 録画時にRTMPのみを試す - -nico-rtmp-only=on (+) 上記を有効に設定 - -nico-rtmp-only=off (+) 上記を無効に設定(デフォルト) - -nico-rtmp-max-conn RTMPの同時接続数を設定 - -nico-rtmp-index [,] RTMP録画を行うメディアファイルの番号を指定 + -nico-cookies firefox[:profile|cookiefile] + firefoxのcookieを使用する(デフォルトはdefault-release) + profileまたはcookiefileを直接指定も可能 + スペースが入る場合はquoteで囲む -nico-hls-port [実験的] ローカルなHLSサーバのポート番号 -nico-limit-bw (+) HLSのBANDWIDTHの上限値を指定する。0=制限なし + audio_high or audio_only = 音声のみ -nico-format "FORMAT" (+) 保存時のファイル名を指定する -nico-fast-ts 倍速タイムシフト録画を行う(新配信タイムシフト) -nico-fast-ts=on (+) 上記を有効に設定 @@ -128,8 +141,20 @@ COMMAND: -nico-force-reservation=off (+) 自動的にタイムシフト予約しない(デフォルト) -nico-skip-hb=on (+) コメント書き出し時に/hbコマンドを出さない -nico-skip-hb=off (+) コメント書き出し時に/hbコマンドも出す(デフォルト) - -nico-ts-start タイムシフトの録画を指定した再生時間(秒)から開始する - -nico-ts-start-min タイムシフトの録画を指定した再生時間(分)から開始する + -nico-adjust-vpos=on (+) コメント書き出し時にvposの値を補正する + vposの値が-1000より小さい場合はコメント出力しない + -nico-adjust-vpos=off (+) コメント書き出し時にvposの値をそのまま出力する(デフォルト) + -nico-ts-start タイムシフトの録画を指定した再生時間（秒）から開始する + -nico-ts-stop タイムシフトの録画を指定した再生時間（秒）で停止する + 上記2つは ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + -nico-ts-start-min タイムシフトの録画を指定した再生時間（分）から開始する + -nico-ts-stop-min タイムシフトの録画を指定した再生時間（分）で停止する + 上記2つは ＜時＞:＜分＞ の形式でも指定可能 + -nico-conv-seqno-start MP4への変換を指定したセグメント番号から開始する + -nico-conv-seqno-end MP4への変換を指定したセグメント番号で終了する + -nico-conv-force-concat MP4への変換で画質変更または抜けがあっても分割しないように設定 + -nico-conv-force-concat=on (+) 上記を有効に設定 + -nico-conv-force-concat=off (+) 上記を無効に設定(デフォルト) ツイキャス録画用オプション: -tcas-retry=on (+) 録画終了後に再試行を行う @@ -144,6 +169,11 @@ Youtube live録画用オプション: -yt-no-streamlink=off (+) Streamlinkを使用する(デフォルト) -yt-no-youtube-dl=on (+) youtube-dlを使用しない -yt-no-youtube-dl=off (+) youtube-dlを使用する(デフォルト) + -yt-comment-start YouTube Liveアーカイブでコメント取得開始時間（秒）を指定 + ＜分＞:＜秒＞ | ＜時＞:＜分＞:＜秒＞ の形式でも指定可能 + 0：続きからコメント取得 1：最初からコメント取得 + -yt-emoji=on (+) コメントにemojiを表示する(デフォルト) + -yt-emoji=off (+) コメントにemojiを表示しない 変換オプション: -extract-chunks=off (+) -d2mで動画ファイルに書き出す(デフォルト) @@ -154,13 +184,14 @@ Youtube live録画用オプション: HTTP関連 -http-skip-verify=on (+) TLS証明書の認証をスキップする (32bit版対策) -http-skip-verify=off (+) TLS証明書の認証をスキップしない (デフォルト) + -http-timeout (+) タイムアウト時間（秒）デフォルト: 5秒（最低値） (+)のついたオプションは、次回も同じ設定が使用されることを示す。 FILE: ニコニコ生放送/nicolive: - http://live2.nicovideo.jp/watch/lvXXXXXXXXX + https://live.nicovideo.jp/watch/lvXXXXXXXXX lvXXXXXXXXX ツイキャス/twitcasting: https://twitcasting.tv/XXXXX @@ -226,6 +257,7 @@ func SetNicoLogin(hash, user, pass string) (err error) { fmt.Printf("niconico account saved.\n") return } + func SetNicoSession(hash, session string) (err error) { db, err := dbAccountOpen() if err != nil { @@ -389,6 +421,56 @@ func dbOpen() (db *sql.DB, err error) { return } +func GetBrowserName(str string) (name string) { + name = "error" + if len(str) <= 0 { + return + } + if m := regexp.MustCompile(`^(firefox:?['\"]?.*['\"]?)`).FindStringSubmatch(str); len(m) > 0 { + name = m[1] + } + return +} + +func parseTime(arg string) (ret int, err error) { + var hour, min, sec int + + if m := regexp.MustCompile(`^(\d+):(\d+):(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + hour, err = strconv.Atoi(m[1]) + if err != nil { + return + } + min, err = strconv.Atoi(m[2]) + if err != nil { + return + } + sec, err = strconv.Atoi(m[3]) + if err != nil { + return + } + } else if m := regexp.MustCompile(`^(\d+):(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + min, err = strconv.Atoi(m[1]) + if err != nil { + return + } + sec, err = strconv.Atoi(m[2]) + if err != nil { + return + } + } else if m := regexp.MustCompile(`^(\d+)$`).FindStringSubmatch(arg); len(m) > 0 { + sec, err = strconv.Atoi(m[1]) + if err != nil { + return + } + } else { + err = fmt.Errorf("regexp not matched") + } + + ret = hour * 3600 + min * 60 + sec + + return +} + func ParseArgs() (opt Option) { //dbAccountOpen() db, err := dbOpen() @@ -403,8 +485,6 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "NicoFormat"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoLimitBw"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginOnly"), 0), - IFNULL((SELECT v FROM conf WHERE k == "NicoHlsOnly"), 0), - IFNULL((SELECT v FROM conf WHERE k == "NicoRtmpOnly"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoFastTs"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoLoginAlias"), ""), IFNULL((SELECT v FROM conf WHERE k == "NicoAutoConvert"), 0), @@ -414,17 +494,19 @@ func ParseArgs() (opt Option) { IFNULL((SELECT v FROM conf WHERE k == "TcasRetryInterval"), 0), IFNULL((SELECT v FROM conf WHERE k == "ConvExt"), ""), IFNULL((SELECT v FROM conf WHERE k == "ExtractChunks"), 0), + IFNULL((SELECT v FROM conf WHERE k == "NicoConvForceConcat"), 0), IFNULL((SELECT v FROM conf WHERE k == "NicoForceResv"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoStreamlink"), 0), IFNULL((SELECT v FROM conf WHERE k == "YtNoYoutubeDl"), 0), + IFNULL((SELECT v FROM conf WHERE k == "YtEmoji"), 1), IFNULL((SELECT v FROM conf WHERE k == "NicoSkipHb"), 0), - IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0); + IFNULL((SELECT v FROM conf WHERE k == "NicoAdjustVpos"), 0), + IFNULL((SELECT v FROM conf WHERE k == "HttpSkipVerify"), 0), + IFNULL((SELECT v FROM conf WHERE k == "HttpTimeout"), 5); `).Scan( &opt.NicoFormat, &opt.NicoLimitBw, &opt.NicoLoginOnly, - &opt.NicoHlsOnly, - &opt.NicoRtmpOnly, &opt.NicoFastTs, &opt.NicoLoginAlias, &opt.NicoAutoConvert, @@ -434,11 +516,15 @@ func ParseArgs() (opt Option) { &opt.TcasRetryInterval, &opt.ConvExt, &opt.ExtractChunks, + &opt.NicoConvForceConcat, &opt.NicoForceResv, &opt.YtNoStreamlink, &opt.YtNoYoutubeDl, + &opt.YtEmoji, &opt.NicoSkipHb, + &opt.NicoAdjustVpos, &opt.HttpSkipVerify, + &opt.HttpTimeout, ) if err != nil { log.Println(err) @@ -590,6 +676,14 @@ func ParseArgs() (opt Option) { opt.Command = "DB2MP4" return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?(?:d|db|sqlite3?)-?(?:2|to)-?(?:h|hls)\z`), func() error { + opt.Command = "DB2HLS" + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?(?:d|db|sqlite3?)-?(?:i|info)\z`), func() error { + opt.Command = "DBINFO" + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?login-?only(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoLoginOnly = true @@ -602,30 +696,6 @@ func ParseArgs() (opt Option) { } return nil }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?hls-?only(?:=(on|off))?\z`), func() error { - if strings.EqualFold(match[1], "on") { - opt.NicoHlsOnly = true - dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) - } else if strings.EqualFold(match[1], "off") { - opt.NicoHlsOnly = false - dbConfSet(db, "NicoHlsOnly", opt.NicoHlsOnly) - } else { - opt.NicoHlsOnly = true - } - return nil - }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?only(?:=(on|off))?\z`), func() error { - if strings.EqualFold(match[1], "on") { - opt.NicoRtmpOnly = true - dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) - } else if strings.EqualFold(match[1], "off") { - opt.NicoRtmpOnly = false - dbConfSet(db, "NicoRtmpOnly", opt.NicoRtmpOnly) - } else { - opt.NicoRtmpOnly = true - } - return nil - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?fast-?ts(?:=(on|off))?\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoFastTs = true @@ -673,27 +743,6 @@ func ParseArgs() (opt Option) { opt.NicoUltraFastTs = true return nil }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?index\z`), func() (err error) { - str, err := nextArg() - if err != nil { - return - } - ar := strings.Split(str, ",") - if len(ar) > 0 { - opt.NicoRtmpIndex = make(map[int]bool) - } - for _, s := range ar { - num, err := strconv.Atoi(s) - if err != nil { - return fmt.Errorf("--nico-rtmp-index: Not a number: %s\n", s) - } - if num <= 0 { - return fmt.Errorf("--nico-rtmp-index: Invalid: %d: must be greater than or equal to 1\n", num) - } - opt.NicoRtmpIndex[num-1] = true - } - return - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?status-?https\z`), func() error { // experimental opt.NicoStatusHTTPS = true @@ -719,11 +768,16 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + if m := regexp.MustCompile(`^(audio_only|audio_high)$`).FindStringSubmatch(s); len(m) > 0 { + opt.NicoLimitBw = m[0] + dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) + return nil + } + _, err = strconv.Atoi(s) if err != nil { return fmt.Errorf("--nico-limit-bw: Not a number: %s\n", s) } - opt.NicoLimitBw = num + opt.NicoLimitBw = s dbConfSet(db, "NicoLimitBw", opt.NicoLimitBw) return nil }}, @@ -732,7 +786,7 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + num, err := parseTime(s) if err != nil { return fmt.Errorf("--nico-ts-start: Not a number %s\n", s) } @@ -744,11 +798,35 @@ func ParseArgs() (opt Option) { if err != nil { return err } - num, err := strconv.Atoi(s) + num, err := parseTime(s + ":0") if err != nil { return fmt.Errorf("--nico-ts-start-min: Not a number %s\n", s) } - opt.NicoTsStart = float64(num * 60) + opt.NicoTsStart = float64(num) + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?stop\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s) + if err != nil { + return fmt.Errorf("--nico-ts-stop: Not a number %s\n", s) + } + opt.NicoTsStop = num + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?ts-?stop-?min\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s + ":0") + if err != nil { + return fmt.Errorf("--nico-ts-stop-min: Not a number %s\n", s) + } + opt.NicoTsStop = num return nil }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?(?:format|fmt)\z`), func() (err error) { @@ -786,12 +864,26 @@ func ParseArgs() (opt Option) { opt.NicoLoginAlias = fmt.Sprintf("%x", sha3.Sum256([]byte(loginId))) SetNicoLogin(opt.NicoLoginAlias, loginId, loginPass) dbConfSet(db, "NicoLoginAlias", opt.NicoLoginAlias) + opt.NicoLoginOnly = true } else { return fmt.Errorf("--nico-login: ,") } return }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?cookies?\z`), func() (err error) { + str, err := nextArg() + if err != nil { + return + } + str = GetBrowserName(str) + if str != "error" { + opt.NicoCookies = str + } else { + return fmt.Errorf("--nico-cookies: invalid browser name") + } + return + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?session\z`), func() (err error) { str, err := nextArg() if err != nil { @@ -817,19 +909,6 @@ func ParseArgs() (opt Option) { return }}, - Parser{regexp.MustCompile(`\A(?i)--?nico-?rtmp-?max-?conn\z`), func() (err error) { - str, err := nextArg() - if err != nil { - return - } - - num, err := strconv.Atoi(str) - if err != nil { - return fmt.Errorf("--nico-rtmp-max-conn %v: %v", str, err) - } - opt.NicoRtmpMaxConn = num - return - }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?debug\z`), func() error { opt.NicoDebug = true return nil @@ -849,6 +928,8 @@ func ParseArgs() (opt Option) { case "", "DB2MP4": opt.Command = "DB2MP4" opt.DBFile = match[0] + case "DB2HLS": + opt.DBFile = match[0] default: return fmt.Errorf("%s: Use -- option before \"%s\"", opt.Command, match[0]) } @@ -872,6 +953,48 @@ func ParseArgs() (opt Option) { dbConfSet(db, "ExtractChunks", opt.ExtractChunks) return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?force-?concat(?:=(on|off))?\z`), func() error { + if strings.EqualFold(match[1], "on") { + opt.NicoConvForceConcat = true + dbConfSet(db, "NicoConvForceConcat", opt.NicoConvForceConcat) + } else if strings.EqualFold(match[1], "off") { + opt.NicoConvForceConcat = false + dbConfSet(db, "NicoConvForceConcat", opt.NicoConvForceConcat) + } else { + opt.NicoConvForceConcat = true + } + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?seqno-?start\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--nico-conv-seqno-start: Not a number: %s\n", s) + } + if num < 0 { + return fmt.Errorf("--nico-conv-seqno-start: Invalid: %d: must be greater than or equal to 0\n", num) + } + opt.NicoConvSeqnoStart = int64(num) + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?conv-?seqno-?end\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--nico-conv-seqno-end: Not a number: %s\n", s) + } + if num < 0 { + return fmt.Errorf("--nico-conv-seqno-end: Invalid: %d: must be greater than or equal to 0\n", num) + } + opt.NicoConvSeqnoEnd = int64(num) + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?force-?(?:re?sv|reservation)(?:=(on|off))\z`), func() error { if strings.EqualFold(match[1], "on") { opt.NicoForceResv = true @@ -916,6 +1039,30 @@ func ParseArgs() (opt Option) { } return nil }}, + Parser{regexp.MustCompile(`\A(?i)--?yt-?emoji(?:=(on|off))?\z`), func() (err error) { + if strings.EqualFold(match[1], "on") { + opt.YtEmoji = true + dbConfSet(db, "YtEmoji", opt.YtEmoji) + } else if strings.EqualFold(match[1], "off") { + opt.YtEmoji = false + dbConfSet(db, "YtEmoji", opt.YtEmoji) + } else { + opt.YtEmoji = true + } + return + }}, + Parser{regexp.MustCompile(`\A(?i)--?yt-?comment-?start\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := parseTime(s) + if err != nil { + return fmt.Errorf("--yt-comment-start: Not a number %s\n", s) + } + opt.YtCommentStart = float64(num) + return nil + }}, Parser{regexp.MustCompile(`\A(?i)--?nico-?skip-?hb(?:=(on|off))?\z`), func() (err error) { if strings.EqualFold(match[1], "on") { opt.NicoSkipHb = true @@ -964,6 +1111,34 @@ func ParseArgs() (opt Option) { opt.NoChdir = true return }}, + Parser{regexp.MustCompile(`\A(?i)--?http-?timeout\z`), func() (err error) { + s, err := nextArg() + if err != nil { + return err + } + num, err := strconv.Atoi(s) + if err != nil { + return fmt.Errorf("--http-timeout: Not a number: %s\n", s) + } + if num < MinimumHttpTimeout { + return fmt.Errorf("--http-timeout: Invalid: %d: must be greater than or equal to %#v\n", num, MinimumHttpTimeout) + } + opt.HttpTimeout = num + dbConfSet(db, "HttpTimeout", opt.HttpTimeout) + return nil + }}, + Parser{regexp.MustCompile(`\A(?i)--?nico-?adjust-?vpos(?:=(on|off))?\z`), func() (err error) { + if strings.EqualFold(match[1], "on") { + opt.NicoAdjustVpos = true + dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) + } else if strings.EqualFold(match[1], "off") { + opt.NicoAdjustVpos = false + dbConfSet(db, "NicoAdjustVpos", opt.NicoAdjustVpos) + } else { + opt.NicoAdjustVpos = false + } + return + }}, } checkFILE := func(arg string) bool { @@ -1001,7 +1176,7 @@ func ParseArgs() (opt Option) { opt.ZipFile = arg return true } - case "DB2MP4": + case "DB2MP4", "DB2HLS", "DBINFO": if ma := regexp.MustCompile(`(?i)\.sqlite3`).FindStringSubmatch(arg); len(ma) > 0 { opt.DBFile = arg return true @@ -1049,6 +1224,11 @@ LB_ARG: opt.ConfFile = fmt.Sprintf("%s.conf", getCmd()) } + if opt.HttpTimeout == 0 { + opt.HttpTimeout = MinimumHttpTimeout + } + httpbase.SetTimeout(opt.HttpTimeout) + // [deprecated] // load session info if data, e := cryptoconf.Load(opt.ConfFile, opt.ConfPass); e != nil { @@ -1074,31 +1254,40 @@ LB_ARG: fmt.Printf("Conf(NicoLoginOnly): %#v\n", opt.NicoLoginOnly) fmt.Printf("Conf(NicoFormat): %#v\n", opt.NicoFormat) fmt.Printf("Conf(NicoLimitBw): %#v\n", opt.NicoLimitBw) - fmt.Printf("Conf(NicoHlsOnly): %#v\n", opt.NicoHlsOnly) - fmt.Printf("Conf(NicoRtmpOnly): %#v\n", opt.NicoRtmpOnly) fmt.Printf("Conf(NicoFastTs): %#v\n", opt.NicoFastTs) fmt.Printf("Conf(NicoAutoConvert): %#v\n", opt.NicoAutoConvert) if opt.NicoAutoConvert { fmt.Printf("Conf(NicoAutoDeleteDBMode): %#v\n", opt.NicoAutoDeleteDBMode) fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) + fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) } fmt.Printf("Conf(NicoForceResv): %#v\n", opt.NicoForceResv) fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) + fmt.Printf("Conf(NicoAdjustVpos): %#v\n", opt.NicoAdjustVpos) case "YOUTUBE": fmt.Printf("Conf(YtNoStreamlink): %#v\n", opt.YtNoStreamlink) fmt.Printf("Conf(YtNoYoutubeDl): %#v\n", opt.YtNoYoutubeDl) + fmt.Printf("Conf(YtEmoji): %#v\n", opt.YtEmoji) case "TWITCAS": fmt.Printf("Conf(TcasRetry): %#v\n", opt.TcasRetry) fmt.Printf("Conf(TcasRetryTimeoutMinute): %#v\n", opt.TcasRetryTimeoutMinute) fmt.Printf("Conf(TcasRetryInterval): %#v\n", opt.TcasRetryInterval) - case "DB2MP4": + case "DB2MP4", "DBINFO": fmt.Printf("Conf(ExtractChunks): %#v\n", opt.ExtractChunks) + fmt.Printf("Conf(NicoConvForceConcat): %#v\n", opt.NicoConvForceConcat) fmt.Printf("Conf(ConvExt): %#v\n", opt.ConvExt) + fmt.Printf("Conf(NicoSkipHb): %#v\n", opt.NicoSkipHb) + fmt.Printf("Conf(NicoAdjustVpos): %#v\n", opt.NicoAdjustVpos) + fmt.Printf("Conf(YtEmoji): %#v\n", opt.YtEmoji) + case "DB2HLS": + fmt.Printf("Conf(NicoHlsPort): %#v\n", opt.NicoHlsPort) + fmt.Printf("Conf(NicoConvSeqnoStart): %#v\n", opt.NicoConvSeqnoStart) } fmt.Printf("Conf(HttpSkipVerify): %#v\n", opt.HttpSkipVerify) + fmt.Printf("Conf(HttpTimeout): %#v\n", opt.HttpTimeout) if opt.NicoDebug { fmt.Printf("Conf(NicoDebug): %#v\n", opt.NicoDebug) @@ -1126,7 +1315,7 @@ LB_ARG: if opt.ZipFile == "" { Help() } - case "DB2MP4": + case "DB2MP4", "DB2HLS", "DBINFO": if opt.DBFile == "" { Help() } diff --git a/src/youtube/comment.go b/src/youtube/comment.go index ac07608..784f9e0 100644 --- a/src/youtube/comment.go +++ b/src/youtube/comment.go @@ -7,11 +7,12 @@ import ( "fmt" "log" "os" - "path/filepath" "strconv" - "strings" "sync" "time" + "html" + "io/ioutil" + "regexp" "github.com/himananiito/livedl/files" "github.com/himananiito/livedl/gorman" @@ -20,7 +21,9 @@ import ( _ "github.com/mattn/go-sqlite3" ) -func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, continuation, name string) (done bool) { +type OBJ = map[string]interface{} + +func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan struct{}, isReplay bool, commentStart float64, continuation, name string) (done bool) { dbName := files.ChangeExtention(name, "yt.sqlite3") db, err := dbOpen(ctx, dbName) @@ -33,11 +36,12 @@ func getComment(gm *gorman.GoroutineManager, ctx context.Context, sig <-chan str mtx := &sync.Mutex{} testContinuation, count, _ := dbGetContinuation(ctx, db, mtx) - if testContinuation != "" { + if commentStart < 0.5 && testContinuation != "" { continuation = testContinuation } var printTime int64 + var isFirst bool = true MAINLOOP: for { @@ -51,25 +55,51 @@ MAINLOOP: timeoutMs, _done, err, neterr := func() (timeoutMs int, _done bool, err, neterr error) { var uri string if isReplay { - uri = fmt.Sprintf("https://www.youtube.com/live_chat_replay?continuation=%s&pbj=1", continuation) + uri = fmt.Sprintf("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8") } else { - uri = fmt.Sprintf("https://www.youtube.com/live_chat/get_live_chat?continuation=%s&pbj=1", continuation) + uri = fmt.Sprintf("https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8") } - code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ - "Cookie": Cookie, + postData := OBJ{ + "context": OBJ{ + "client": OBJ{ + "clientName": "WEB", + "clientVersion": "2.20210128.02.00", + }, + }, + "continuation": continuation, + } + if !isFirst && isReplay && commentStart > 1.5 { + postData["currentPlayerState"] = OBJ{ + "playerOffsetMs": commentStart * 1000.0, + } + } + resp, err, neterr := httpbase.PostJson(uri, map[string]string { + "Cookie": Cookie, "User-Agent": UserAgent, - }) + }, nil, postData) if err != nil { return } if neterr != nil { return } - if code != 200 { - neterr = fmt.Errorf("Status code: %v\n", code) + buff, neterr := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if neterr != nil { return } + code := resp.StatusCode + if code != 200 { + if code == 404 { + fmt.Printf("Status code: %v (ignored)\n", code) + time.Sleep(1000 * time.Millisecond) + return + } else { + neterr = fmt.Errorf("Status code: %v\n", code) + return + } + } var data interface{} err = json.Unmarshal(buff, &data) @@ -78,13 +108,15 @@ MAINLOOP: return } - liveChatContinuation, ok := objs.Find(data, "response", "continuationContents", "liveChatContinuation") + liveChatContinuation, ok := objs.Find(data, "continuationContents", "liveChatContinuation") if !ok { err = fmt.Errorf("(response liveChatContinuation) not found") return } - if actions, ok := objs.FindArray(liveChatContinuation, "actions"); ok { + if isFirst && isReplay && commentStart > 1.5 { + isFirst = false + } else if actions, ok := objs.FindArray(liveChatContinuation, "actions"); ok { var videoOffsetTimeMsec string for _, a := range actions { @@ -121,7 +153,24 @@ MAINLOOP: if !ok { continue } - message, _ := objs.FindString(liveChatMessageRenderer, "message", "simpleText") + var message string + if runs, ok := objs.FindArray(liveChatMessageRenderer, "message", "runs"); ok { + for _, run := range runs { + if text, ok := objs.FindString(run, "text"); ok { + message += text + } else if emojis, ok := objs.FindArray(run, "emoji", "shortcuts"); ok { + if emoji, ok := emojis[0].(string); ok { + message += emoji + } + } + } + } + var others string + var amount string + amount, _ = objs.FindString(liveChatMessageRenderer, "purchaseAmountText", "simpleText") + if amount != "" { + others += ` amount="` + html.EscapeString(amount) + `"` + } timestampUsec, ok := objs.FindString(liveChatMessageRenderer, "timestampUsec") if !ok { continue @@ -129,7 +178,7 @@ MAINLOOP: if false { fmt.Printf("%v ", videoOffsetTimeMsec) - fmt.Printf("%v %v %v %v %v\n", timestampUsec, authorName, authorExternalChannelId, message, id) + fmt.Printf("%v %v %v %v %v %v [%v ]\n", timestampUsec, count, authorName, authorExternalChannelId, message, id, others) } dbInsert(ctx, gm, db, mtx, @@ -140,6 +189,7 @@ MAINLOOP: authorExternalChannelId, message, continuation, + others, count, ) count++ @@ -263,6 +313,7 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { channelId TEXT, message TEXT, continuation TEXT, + others TEXT, count INTEGER NOT NULL ) `) @@ -272,9 +323,9 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { _, err = db.ExecContext(ctx, ` CREATE UNIQUE INDEX IF NOT EXISTS comment0 ON comment(id); - CREATE UNIQUE INDEX IF NOT EXISTS comment1 ON comment(timestampUsec); - CREATE UNIQUE INDEX IF NOT EXISTS comment2 ON comment(videoOffsetTimeMsec); - CREATE UNIQUE INDEX IF NOT EXISTS comment3 ON comment(count); + CREATE INDEX IF NOT EXISTS comment1 ON comment(timestampUsec); + CREATE INDEX IF NOT EXISTS comment2 ON comment(videoOffsetTimeMsec); + CREATE INDEX IF NOT EXISTS comment3 ON comment(count); `) if err != nil { return @@ -284,7 +335,7 @@ func dbCreate(ctx context.Context, db *sql.DB) (err error) { } func dbInsert(ctx context.Context, gm *gorman.GoroutineManager, db *sql.DB, mtx *sync.Mutex, - id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation string, count int) { + id, timestampUsec, videoOffsetTimeMsec, authorName, authorExternalChannelId, message, continuation, others string, count int) { usec, err := strconv.ParseInt(timestampUsec, 10, 64) if err != nil { @@ -304,14 +355,14 @@ func dbInsert(ctx context.Context, gm *gorman.GoroutineManager, db *sql.DB, mtx } query := `INSERT OR IGNORE INTO comment - (id, timestampUsec, videoOffsetTimeMsec, authorName, channelId, message, continuation, count) VALUES (?,?,?,?,?,?,?,?)` + (id, timestampUsec, videoOffsetTimeMsec, authorName, channelId, message, continuation, others, count) VALUES (?,?,?,?,?,?,?,?,?)` gm.Go(func(<-chan struct{}) int { mtx.Lock() defer mtx.Unlock() if _, err := db.ExecContext(ctx, query, - id, usec, offset, authorName, authorExternalChannelId, message, continuation, count, + id, usec, offset, authorName, authorExternalChannelId, message, continuation, others, count, ); err != nil { if err.Error() != "context canceled" { fmt.Println(err) @@ -332,17 +383,57 @@ func dbGetContinuation(ctx context.Context, db *sql.DB, mtx *sync.Mutex) (res st return } +func DbGetCountComment(db *sql.DB) (res int64) { + db.QueryRow("SELECT COUNT(id) FROM comment").Scan(&res) + return +} + +func ShowDbInfo(fileName string) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") + if err != nil { + return + } + defer db.Close() + + fmt.Println("----- DATABASE info. -----") + fmt.Println("sqlite3 file :", fileName) + for _, tbl := range []string{"comment"} { + if !dbIsExistTable(db, tbl) { + fmt.Println("table", tbl, "not found") + } else { + fmt.Println("table", tbl, "exist") + } + } + + comm_data := DbGetCountComment(db) + fmt.Println("----- comment info. -----") + fmt.Println("data: ", comm_data) + + done = true + + return +} + var SelComment = `SELECT timestampUsec, IFNULL(videoOffsetTimeMsec, -1), authorName, channelId, - message + message, + others, + count FROM comment ORDER BY timestampUsec ` -func WriteComment(db *sql.DB, fileName string) { +func WriteComment(db *sql.DB, fileName string, emoji bool) { + + regexp1 := regexp.MustCompile(":[a-zA-Z0-9\\-\\_]*:") rows, err := db.Query(SelComment) if err != nil { @@ -352,15 +443,12 @@ func WriteComment(db *sql.DB, fileName string) { defer rows.Close() fileName = files.ChangeExtention(fileName, "xml") - - dir := filepath.Dir(fileName) - base := filepath.Base(fileName) - base, err = files.GetFileNameNext(base) + fileName, err = files.GetFileNameNext(fileName) + fmt.Println("xml file: ", fileName) if err != nil { fmt.Println(err) os.Exit(1) } - fileName = filepath.Join(dir, base) f, err := os.Create(fileName) if err != nil { log.Fatalln(err) @@ -377,6 +465,8 @@ func WriteComment(db *sql.DB, fileName string) { var authorName string var channelId string var message string + var others string + var count int64 err = rows.Scan( ×tampUsec, @@ -384,6 +474,8 @@ func WriteComment(db *sql.DB, fileName string) { &authorName, &channelId, &message, + &others, + &count, ) if err != nil { log.Println(err) @@ -402,19 +494,37 @@ func WriteComment(db *sql.DB, fileName string) { } line := fmt.Sprintf( - ``) } +func dbIsExistTable(db *sql.DB, table_name string) (ret bool) { + var res int + ret = false + if len(table_name) > 0 { + db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE TYPE='table' AND name=?", table_name).Scan(&res) + if res > 0 { + ret = true + } + } + return +} diff --git a/src/youtube/youtube.go b/src/youtube/youtube.go index d60f868..0f7b225 100644 --- a/src/youtube/youtube.go +++ b/src/youtube/youtube.go @@ -53,7 +53,7 @@ var split = func(data []byte, atEOF bool) (advance int, token []byte, err error) func getChatContinuation(buff []byte) (isReplay bool, continuation string, err error) { - if ma := regexp.MustCompile(`(?s)\Wwindow\["ytInitialData"\]\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`).FindSubmatch(buff); len(ma) > 1 { + if ma := regexp.MustCompile(`(?s)\WytInitialData\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`).FindSubmatch(buff); len(ma) > 1 { var data interface{} err = json.Unmarshal(ma[1], &data) if err != nil { @@ -112,27 +112,27 @@ func getChatContinuation(buff []byte) (isReplay bool, continuation string, err e func getInfo(buff []byte) (title, ucid, author string, err error) { var data interface{} - re := regexp.MustCompile(`(?s)\Wytplayer\.config\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) + re := regexp.MustCompile(`(?s)\WytInitialPlayerResponse\p{Zs}*=\p{Zs}*({.*?})\p{Zs}*;`) if ma := re.FindSubmatch(buff); len(ma) > 1 { str := html.UnescapeString(string(ma[1])) if err = json.Unmarshal([]byte(str), &data); err != nil { - err = fmt.Errorf("ytplayer parse error") + err = fmt.Errorf("ytInitialPlayerResponse parse error") return } } else { - err = fmt.Errorf("ytplayer.config not found") + err = fmt.Errorf("ytInitialPlayerResponse not found") return } //objs.PrintAsJson(data); return - title, ok := objs.FindString(data, "args", "title") + title, ok := objs.FindString(data, "videoDetails", "title") if !ok { err = fmt.Errorf("title not found") return } - ucid, _ = objs.FindString(data, "args", "ucid") - author, _ = objs.FindString(data, "args", "author") + ucid, _ = objs.FindString(data, "videoDetails", "channelId") + author, _ = objs.FindString(data, "videoDetails", "author") return } @@ -299,13 +299,13 @@ func execYoutube_dl(gm *gorman.GoroutineManager, uri, name string) (err error) { var COMMENT_DONE = 1000 -func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { +func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool, ytCommentStart float64) (err error) { uri := fmt.Sprintf("https://www.youtube.com/watch?v=%s", id) code, buff, err, neterr := httpbase.GetBytes(uri, map[string]string{ "Cookie": Cookie, "User-Agent": UserAgent, - }) + }, nil) if err != nil { return } @@ -390,7 +390,7 @@ func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { if continuation != "" { gmCom.Go(func(c <-chan struct{}) int { - getComment(gmCom, ctx, c, isReplay, continuation, origName) + getComment(gmCom, ctx, c, isReplay, ytCommentStart, continuation, origName) fmt.Printf("\ncomment done\n") return COMMENT_DONE }) @@ -424,7 +424,9 @@ func Record(id string, ytNoStreamlink, ytNoYoutube_dl bool) (err error) { } } else { - gmCom.Cancel() + if !ytNoStreamlink || !ytNoYoutube_dl { + gmCom.Cancel() + } gmCom.Wait() } } diff --git a/src/zip2mp4/zip2mp4.go b/src/zip2mp4/zip2mp4.go index 30d5f50..47c2722 100644 --- a/src/zip2mp4/zip2mp4.go +++ b/src/zip2mp4/zip2mp4.go @@ -21,6 +21,10 @@ import ( "github.com/himananiito/livedl/procs/ffmpeg" "github.com/himananiito/livedl/youtube" _ "github.com/mattn/go-sqlite3" + + "context" + "github.com/gin-gonic/gin" + "net/http" ) type ZipMp4 struct { @@ -437,16 +441,36 @@ func Convert(fileName string) (err error) { return } -func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { - db, err := sql.Open("sqlite3", fileName) +func ExtractChunks(fileName string, skipHb, adjustVpos bool, seqnoStart, seqnoEnd int64) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") if err != nil { return } defer db.Close() - niconico.WriteComment(db, fileName, skipHb) + seqstart := niconico.DbGetFirstSeqNo(db, 1) + seqend := niconico.DbGetLastSeqNo(db, 1) + var seqoffset int64 + + if seqnoStart > 0 && seqnoStart > seqstart { + seqoffset = seqnoStart - seqstart // リアルタイム放送の開始時間の計算用 + seqstart = seqnoStart + } + if seqnoEnd > 0 && seqnoEnd < seqend { + seqend = seqnoEnd + } + fmt.Println("seqstart: ", seqstart) + fmt.Println("seqoffset: ", seqoffset) + fmt.Println("seqend: ", seqend) + + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend, seqoffset) - rows, err := db.Query(niconico.SelMedia) + rows, err := db.Query(niconico.SelMediaF(seqstart, seqend)) if err != nil { return } @@ -492,14 +516,34 @@ func ExtractChunks(fileName string, skipHb bool) (done bool, err error) { return } -func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err error) { - db, err := sql.Open("sqlite3", fileName) +func ConvertDB(fileName, ext string, skipHb, adjustVpos, forceConcat bool, seqnoStart, seqnoEnd int64) (done bool, nMp4s int, skipped bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") if err != nil { return } defer db.Close() - niconico.WriteComment(db, fileName, skipHb) + seqstart := niconico.DbGetFirstSeqNo(db, 1) + seqend := niconico.DbGetLastSeqNo(db, 1) + var seqoffset int64 + + if seqnoStart > 0 && seqnoStart > seqstart { + seqoffset = seqnoStart - seqstart // リアルタイム放送の開始時間の計算用 + seqstart = seqnoStart + } + if seqnoEnd > 0 && seqnoEnd < seqend { + seqend = seqnoEnd + } + fmt.Println("seqstart: ", seqstart) + fmt.Println("seqoffset: ", seqoffset) + fmt.Println("seqend: ", seqend) + + niconico.WriteComment(db, fileName, skipHb, adjustVpos, seqstart, seqend, seqoffset) var zm *ZipMp4 defer func() { @@ -512,7 +556,7 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err zm = &ZipMp4{ZipName: fileName} zm.OpenFFMpeg(ext) - rows, err := db.Query(niconico.SelMedia) + rows, err := db.Query(niconico.SelMediaF(seqstart, seqend)) if err != nil { return } @@ -543,7 +587,10 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err // zm.CloseFFInput() // zm.Wait() //} - zm.OpenFFMpeg(ext) + if ! forceConcat { + zm.OpenFFMpeg(ext) + } + skipped = true } prevBw = bw prevIndex = seqno @@ -563,13 +610,166 @@ func ConvertDB(fileName, ext string, skipHb bool) (done bool, nMp4s int, err err return } -func YtComment(fileName string) (done bool, err error) { +func ReplayDB(fileName string, hlsPort int, seqnoStart int64) (err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } db, err := sql.Open("sqlite3", fileName) if err != nil { return } defer db.Close() - youtube.WriteComment(db, fileName) + var isTimeshift bool + if m := regexp.MustCompile(`\(TS\)\.sqlite3$`).FindStringSubmatch(fileName); len(m) > 0 { + isTimeshift = true + } + fmt.Println("isTimeshift:", isTimeshift) + + seqnoInit := seqnoStart + + timeStart := time.Now() + timeLast := time.Now() + + seqnoCurrent := seqnoStart + + if (true) { + gin.SetMode(gin.ReleaseMode) + gin.DefaultErrorWriter = ioutil.Discard + gin.DefaultWriter = ioutil.Discard + router := gin.Default() + + router.GET("", func(c *gin.Context) { + c.Redirect(http.StatusMovedPermanently, "/m3u8/2/0/index.m3u8") + c.Abort() + }) + + router.GET("/m3u8/:delay/:shift/index.m3u8", func(c *gin.Context) { + secPerSegment := 1.5 + targetDuration := "2" + targetDurationFloat := 2.0 + extInf := "1.5" + if isTimeshift { + secPerSegment = 5.0 + targetDuration = "3" + targetDurationFloat = 3.0 + extInf = "3.0" + } + shift, err := strconv.Atoi(c.Param("shift")) + if err != nil { + shift = 0 + } + if shift < 0 { + shift = 0 + } + delay, err := strconv.Atoi(c.Param("delay")) + if err != nil { + delay = 0 + } + if delay < 2 { + delay = 2 + } + if (! isTimeshift) { + if delay < 4 { + delay = 4 + } + } + seqnoRewind := int64(delay) + timeout := targetDurationFloat * float64(delay + 1) * 2 + 1 + timeNow := time.Now() + if float64(timeNow.Sub(timeLast) / time.Second) > timeout { + fmt.Printf("(%s) CONTINUE\n", timeNow.Format("15:04:05")) + seqnoStart = seqnoCurrent - seqnoRewind + if seqnoStart < seqnoInit { + seqnoStart = seqnoInit + } + timeStart = timeNow + seqnoCurrent = seqnoStart + } else { + seqnoCurrent = int64(float64(timeNow.Sub(timeStart) / time.Second) / secPerSegment) + seqnoStart + } + timeLast = timeNow + seqno := seqnoCurrent - int64(shift) + body := fmt.Sprintf( +`#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:%s +#EXT-X-MEDIA-SEQUENCE:%d + +`, targetDuration, seqno) + for i := int64(delay); i >= 0; i-- { + body += fmt.Sprintf( +`#EXTINF:%s, +/ts/%d/test.ts + +`, extInf, seqno - i) + } + if shift > 0 { + fmt.Printf("(%s) Current SeqNo: %d(-%d)\n", timeNow.Format("15:04:05"), seqnoCurrent, shift) + } else { + fmt.Printf("(%s) Current SeqNo: %d\n", timeNow.Format("15:04:05"), seqnoCurrent) + } + c.Data(http.StatusOK, "application/x-mpegURL", []byte(body)) + return + }) + + router.GET("/ts/:idx/test.ts", func(c *gin.Context) { + i, _ := strconv.Atoi(c.Param("idx")) + var b []byte + db.QueryRow("SELECT data FROM media WHERE seqno = ?", i).Scan(&b) + c.Data(http.StatusOK, "video/MP2T", b) + return + }) + + srv := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1:%d", hlsPort), + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + chLocal := make(chan struct{}) + idleConnsClosed := make(chan struct{}) + defer func(){ + close(chLocal) + }() + go func() { + select { + case <-chLocal: + } + if err := srv.Shutdown(context.Background()); err != nil { + log.Printf("srv.Shutdown: %v\n", err) + } + close(idleConnsClosed) + }() + + // クライアントはlocalhostでなく127.0.0.1で接続すること + // localhostは遅いため + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + log.Printf("srv.ListenAndServe: %v\n", err) + } + + <-idleConnsClosed + } + + return +} + +func YtComment(fileName string, ytemoji bool) (done bool, err error) { + _, err = os.Stat(fileName) + if err != nil { + fmt.Println("sqlite3 file not found:") + return + } + db, err := sql.Open("sqlite3", "file:"+fileName+"?mode=ro&immutable=1") + if err != nil { + return + } + defer db.Close() + + youtube.WriteComment(db, fileName, ytemoji) return }