Youtube-dl support

Cargo.toml

@@ -18,6 +18,7 @@ frontmatter = "^0.4"
 yaml-rust = "^0.4"
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = "^1.0"
+futures = "^0.3.14"
 
 [dependencies.readability]
 git = "https://github.com/trashhalo/readability.git"

manifest.json

@@ -1,9 +1,9 @@
 {
   "id": "extract-url",
   "name": "Extract url content",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Extract url converting content into markdown",
   "author": "Stephen Solka",
   "authorUrl": "https://github.com/trashhalo",
   "isDesktopOnly": true
-}
+}

package.json

@@ -17,6 +17,7 @@
     "rollup-plugin-base64": "^1.0.1"
   },
   "dependencies": {
+    "hasbin": "^1.2.3",
     "node-fetch": "2.6.1"
   }
-}
+}

src/transform/oembed.rs

@@ -1,3 +1,4 @@
+mod metadata;
 use crate::fetch;
 use html2md::parse_html;
 use serde::Deserialize;
@@ -23,6 +24,9 @@ pub enum OembedError {
 
     #[error("Error serializing oembed data. {0}")]
     Serde(#[from] serde_json::error::Error),
+
+    #[error("error getting youtube metadata")]
+    Metadata(#[from] metadata::MetadataError),
 }
 
 impl std::convert::From<wasm_bindgen::JsValue> for OembedError {
@@ -42,6 +46,7 @@ pub struct OembedData {
 }
 
 pub async fn oembed_content(_body: String, url: &Url) -> Result<String, OembedError> {
+    let m = metadata::metadata(url).await?;
     let mut href = Url::parse("https://noembed.com/embed")?;
     href.query_pairs_mut().append_pair("url", &url.to_string());
     let resp_value = JsFuture::from(fetch::with_url(&href.to_string())).await?;
@@ -52,7 +57,13 @@ pub async fn oembed_content(_body: String, url: &Url) -> Result<String, OembedEr
         None => Err(OembedError::NoHtml),
         Some(html) => {
             if html.contains("iframe") {
-                Ok(format!("# [{}]({})\n{}", data.title, url, html))
+                match m {
+                    None => Ok(format!("# [{}]({})\n{}", data.title, url, html)),
+                    Some(video) => Ok(format!(
+                        "# [{}]({})\n{}\n\n[{}]({})\n{}",
+                        data.title, url, html, video.channel, video.uploader_url, video.description
+                    )),
+                }
             } else {
                 Ok(format!(
                     "# [{}]({})\n{}",

src/transform/oembed/metadata.rs

@@ -0,0 +1,85 @@
+use futures::channel::oneshot;
+use js_sys::Error;
+use serde::Deserialize;
+use serde_json;
+use thiserror::Error;
+use url::{Host, Url};
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen(module = "hasbin")]
+extern "C" {
+    #[wasm_bindgen(js_name=sync)]
+    pub fn has_bin(app: &str) -> bool;
+}
+
+#[wasm_bindgen(module = "child_process")]
+extern "C" {
+    #[wasm_bindgen(js_name=exec)]
+    pub fn node_exec(cmd: &str, f: &Closure<dyn FnMut(Option<Error>, String, String)>) -> JsValue;
+}
+
+#[derive(Error, Debug)]
+pub enum ExecError {
+    #[error("error executing command. {0} {1}")]
+    Error(String, String),
+
+    #[error("oneshot caneled")]
+    Canceled(#[from] oneshot::Canceled),
+}
+
+async fn exec(cmd: &str) -> Result<Option<String>, ExecError> {
+    let (sender, receiver) = oneshot::channel::<Result<String, ExecError>>();
+    let cb = Closure::once(|err_val: Option<Error>, out: String, out_err: String| {
+        let res = match err_val {
+            None => sender.send(Ok(out)),
+            Some(err) => {
+                let msg: String = err.message().into();
+                sender.send(Err(ExecError::Error(msg, out_err)))
+            }
+        };
+        res.unwrap();
+    });
+    node_exec(cmd, &cb);
+    let body = receiver.await??;
+    Ok(Some(body))
+}
+
+#[derive(Debug, Deserialize)]
+pub struct VideoMetadata {
+    pub channel: String,
+    pub uploader_url: String,
+    pub description: String,
+}
+
+#[derive(Error, Debug)]
+pub enum MetadataError {
+    #[error("error running youtube-dl {0}")]
+    Exec(#[from] ExecError),
+
+    #[error("Error serializing youtub data. {0}")]
+    Serde(#[from] serde_json::error::Error),
+}
+
+pub async fn metadata(url: &Url) -> Result<Option<VideoMetadata>, MetadataError> {
+    if !is_youtube(url) {
+        return Ok(None);
+    } else if !has_bin("youtube-dl") {
+        return Ok(None);
+    }
+    match exec(&format!("youtube-dl -j {}", url)).await? {
+        None => Ok(None),
+        Some(s) => {
+            let m: VideoMetadata = serde_json::from_str(&s)?;
+            Ok(Some(m))
+        }
+    }
+}
+
+fn is_youtube(url: &Url) -> bool {
+    match url.host() {
+        Some(Host::Domain("youtu.be"))
+        | Some(Host::Domain("youtube.com"))
+        | Some(Host::Domain("www.youtube.com")) => true,
+        _ => false,
+    }
+}

yarn.lock

@@ -58,6 +58,11 @@
   dependencies:
     "@types/node" "*"
 
+async@~1.5:
+  version "1.5.2"
+  resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
+  integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=
+
 balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@@ -135,6 +140,13 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+hasbin@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/hasbin/-/hasbin-1.2.3.tgz#78c5926893c80215c2b568ae1fd3fcab7a2696b0"
+  integrity sha1-eMWSaJPIAhXCtWiuH9P8q3omlrA=
+  dependencies:
+    async "~1.5"
+
 inflight@^1.0.4:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"