From ab2c86356eea89ba58c4ccf32a9c7486ff8eac67 Mon Sep 17 00:00:00 2001
From: Young-Flash <dongyang@apache.org>
Date: Fri, 29 Nov 2024 10:10:24 +0800
Subject: [PATCH] internal: use gz to compress json data

---
 Cargo.lock         | 35 +++++++++++++++++++++++++++++++++++
 Cargo.toml         |  1 +
 src/main.rs        | 13 +++++++++++++
 webapp/src/App.tsx | 10 ++++++----
 4 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8136b83..54b7893 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@@ -181,6 +187,15 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "crc32fast"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "errno"
 version = "0.3.9"
@@ -197,6 +212,16 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
+[[package]]
+name = "flate2"
+version = "1.0.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -295,6 +320,15 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
+]
+
 [[package]]
 name = "moon_dashboard"
 version = "0.1.0"
@@ -303,6 +337,7 @@ dependencies = [
  "chrono",
  "clap",
  "colored",
+ "flate2",
  "form_urlencoded",
  "home",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 223c85a..38558ef 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,3 +15,4 @@ serde_json = "1.0.127"
 tempfile = "3.12.0"
 thiserror = "1.0.63"
 walkdir = "2.5.0"
+flate2 = "1.0.35"
diff --git a/src/main.rs b/src/main.rs
index 0934900..1a04282 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -8,6 +8,7 @@ use chrono::{FixedOffset, Local};
 
 use clap::Parser;
 use colored::Colorize;
+use flate2::{write::GzEncoder, Compression};
 use moon_dashboard::{
     cli,
     dashboard::{
@@ -443,6 +444,18 @@ fn main0() -> anyhow::Result<()> {
             let mut writer = std::io::BufWriter::new(fp);
             writeln!(writer, "{}", serde_json::to_string(&dashboard)?)?;
             writer.flush()?;
+
+            let fp = std::fs::OpenOptions::new()
+                .create(true)
+                .write(true)
+                .truncate(true) // make sure to truncate the file
+                .open("webapp/public/latest_data.jsonl.gz")?;
+
+            // create a gzip encoder
+            let encoder = GzEncoder::new(fp, Compression::default());
+            let mut writer = std::io::BufWriter::new(encoder);
+            writeln!(writer, "{}", serde_json::to_string(&dashboard)?)?;
+            writer.flush()?;
             Ok(())
         }
         Err(e) => Err(anyhow::anyhow!(e)),
diff --git a/webapp/src/App.tsx b/webapp/src/App.tsx
index 77274f2..bab0f3c 100644
--- a/webapp/src/App.tsx
+++ b/webapp/src/App.tsx
@@ -51,11 +51,13 @@ interface BuildState {
 }
 
 async function get_data(): Promise<MoonBuildDashboard> {
-  const response = await fetch('/data.jsonl');
+  const response = await fetch('/latest_data.jsonl.gz', {
+    headers: {
+      'Accept-Encoding': 'gzip'
+    }
+  });
   const text = await response.text();
-  const lines = text.split('\n').filter((line) => line.trim() !== '');
-  const parsedData = lines.map((line) => JSON.parse(line));
-  return parsedData[parsedData.length - 1];
+  return JSON.parse(text);
 }
 
 interface ModalProps {