Simplify fetch to display progress (#258)

kanaverse · Apr 23, 2024 · 3cd9463 · 3cd9463
1 parent 60e6b3e
commit 3cd9463
Show file tree

Hide file tree

Showing 2 changed files with 109 additions and 97 deletions.
diff --git a/src/workers/DownloadsDBHandler.js b/src/workers/DownloadsDBHandler.js
@@ -32,6 +32,112 @@ export function initialize() {
   return init;
 }
 
+async function fetchWithProgress(url, startFun, iterFun, endFun, params=null) {
+  let res;
+  if (params == null) {
+    res = await fetch(url);
+  } else {
+    res = await fetch(url, params);
+  }
+
+  if (!res.ok) {
+    throw new Error("oops, failed to download '" + url + "'");
+  }
+
+  const cl = res.headers.get("content-length"); // WARNING: this might be NULL!
+  const id = startFun(cl);
+
+  const reader = res.body.getReader();
+  const chunks = [];
+  let total = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) {
+      break;
+    }
+    chunks.push(value);
+    total += value.length;
+    iterFun(id, total);
+  }
+
+  let output = new Uint8Array(total);
+  let start = 0;
+  for (const x of chunks) {
+    output.set(x, start);
+    start += x.length;
+  }
+
+  endFun(id, total);
+  return output;
+}
+
+async function fetchWrapper(url, params = null) {
+  try {
+    const out = await fetchWithProgress(
+      url,
+      (cl) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "START",
+          url: String(url),
+          total_bytes: String(cl),
+          msg: "Total size is " + String(cl) + " bytes!",
+        });
+        return url;
+      },
+      (id, sofar) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "PROGRESS",
+          url: String(url),
+          downloaded_bytes: String(sofar),
+          msg: "Progress so far, got " + String(sofar) + " bytes!",
+        });
+      },
+      (id, total) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "COMPLETE",
+          url: String(url),
+          msg: "Finished, got " + String(total) + " bytes!",
+        });
+      },
+      params
+    );
+
+    return out;
+  } catch (error) {
+    // console.log("oops error", error)
+    postMessage({
+      type: `DOWNLOAD for url: ` + String(url),
+      download: "START",
+      url: String(url),
+      total_bytes: 100,
+    });
+
+    let req;
+    if (params == null) {
+      req = fetch(url);
+    } else {
+      req = fetch(url, params);
+    }
+
+    var res = await req;
+    if (!res.ok) {
+      throw new Error("failed to download '" + url + "' (" + res.status + ")");
+    }
+    var buffer = await res.arrayBuffer();
+
+    postMessage({
+      type: `DOWNLOAD for url: ` + String(url),
+      download: "COMPLETE",
+      url: String(url),
+    });
+    return new Uint8Array(buffer);
+  }
+}
+
 export async function get(url, params = null, force = false) {
   await init;
 
@@ -59,18 +165,7 @@ export async function get(url, params = null, force = false) {
     }
   }
 
-  var req;
-  if (params == null) {
-    req = fetch(url);
-  } else {
-    req = fetch(url, params);
-  }
-
-  var res = await req;
-  if (!res.ok) {
-    throw new Error("failed to download '" + url + "' (" + res.status + ")");
-  }
-  var buffer = await res.arrayBuffer();
+  var buffer = await fetchWrapper(url, params)
 
   // Technically, this isn't quite right, because we need to close the read
   // transaction before opening the write transaction; multiple queries to

diff --git a/src/workers/helpers.js b/src/workers/helpers.js
@@ -6,91 +6,8 @@ import * as downloads from "./DownloadsDBHandler.js";
 // Evade CORS problems and enable caching.
 const proxy = "https://cors-proxy.aaron-lun.workers.dev";
 async function proxyAndCache(url) {
-  const url_with_proxy = proxy + "/" + encodeURIComponent(url);
-
-  try {
-    const out = await fetchWithProgress(
-      url_with_proxy,
-      (cl) => {
-        postMessage({
-          type: `DOWNLOAD for url: ` + String(url),
-          download: "START",
-          url: String(url),
-          total_bytes: String(cl),
-          msg: "Total size is " + String(cl) + " bytes!",
-        });
-        return url_with_proxy;
-      },
-      (id, sofar) => {
-        postMessage({
-          type: `DOWNLOAD for url: ` + String(url),
-          download: "PROGRESS",
-          url: String(url),
-          downloaded_bytes: String(sofar),
-          msg: "Progress so far, got " + String(sofar) + " bytes!",
-        });
-      },
-      (id, total) => {
-        postMessage({
-          type: `DOWNLOAD for url: ` + String(url),
-          download: "COMPLETE",
-          url: String(url),
-          msg: "Finished, got " + String(total) + " bytes!",
-        });
-      }
-    );
-
-    return out;
-  } catch (error) {
-    // console.log("oops error", error)
-    postMessage({
-      type: `DOWNLOAD for url: ` + String(url),
-      download: "START",
-      url: String(url),
-      total_bytes: 100,
-    });
-    let buffer = await downloads.get(url_with_proxy);
-    postMessage({
-      type: `DOWNLOAD for url: ` + String(url),
-      download: "COMPLETE",
-      url: String(url),
-    });
-    return new Uint8Array(buffer);
-  }
-}
-
-async function fetchWithProgress(url, startFun, iterFun, endFun) {
-  const res = await fetch(url);
-  if (!res.ok) {
-    throw new Error("oops, failed to download '" + url + "'");
-  }
-
-  const cl = res.headers.get("content-length"); // WARNING: this might be NULL!
-  const id = startFun(cl);
-
-  const reader = res.body.getReader();
-  const chunks = [];
-  let total = 0;
-
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) {
-      break;
-    }
-    chunks.push(value);
-    total += value.length;
-    iterFun(id, total);
-  }
-
-  let output = new Uint8Array(total);
-  let start = 0;
-  for (const x of chunks) {
-    output.set(x, start);
-    start += x.length;
-  }
-
-  endFun(id, total);
-  return output;
+  let buffer = await downloads.get(proxy + "/" + encodeURIComponent(url));
+  return new Uint8Array(buffer);
 }
 
 bakana.CellLabellingState.setDownload(proxyAndCache);