From d1eeb91eefab54631f3c7a0f240a3cc11c07ac9b Mon Sep 17 00:00:00 2001
From: Jack Kelly <jack@openclimatefix.org>
Date: Thu, 12 Sep 2024 14:44:51 +0100
Subject: [PATCH] Remove duckdb #11

---
 Cargo.toml                           |  9 +++--
 crates/hypergrib_manifest/Cargo.toml |  1 -
 crates/hypergrib_manifest/src/lib.rs | 52 ----------------------------
 3 files changed, 4 insertions(+), 58 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 32ce105..5df292e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,16 +11,15 @@ readme = "README.md"
 authors=["Jack Kelly <jack@openclimatefix.org>"]
 
 [workspace.dependencies] # In alphabetical order
-anyhow = "1.0.86"
+anyhow = "1.0.88"
 bytes = "1.7.1"
-clap = "4.5.16"  # parse command line arguments etc.
+clap = "4.5.17"  # parse command line arguments etc.
 csv = "1.3.0"
-duckdb = "1.0.0"
 futures-util = "0.3.30"
-gribberish = "0.20.1"
+gribberish = "0.20.2"
 indicatif = "0.17.8"  # progress bars etc.
 object_store = "0.11.0"
-serde = "1.0.209"
+serde = "1.0.210"
 tokio = { version = "1.40.0", features = ["rt-multi-thread"]}
 url = "2.5.2"
 
diff --git a/crates/hypergrib_manifest/Cargo.toml b/crates/hypergrib_manifest/Cargo.toml
index d29aafd..864b64f 100644
--- a/crates/hypergrib_manifest/Cargo.toml
+++ b/crates/hypergrib_manifest/Cargo.toml
@@ -15,7 +15,6 @@ anyhow.workspace = true
 bytes.workspace = true
 clap = { workspace = true, features = ["derive"] }
 csv.workspace = true
-duckdb = { workspace = true, features = ["bundled"] }
 futures-util.workspace = true
 gribberish.workspace = true
 indicatif.workspace = true
diff --git a/crates/hypergrib_manifest/src/lib.rs b/crates/hypergrib_manifest/src/lib.rs
index 07e61df..73ea773 100644
--- a/crates/hypergrib_manifest/src/lib.rs
+++ b/crates/hypergrib_manifest/src/lib.rs
@@ -24,52 +24,6 @@ fn parse_idx(b: &[u8]) -> anyhow::Result<Vec<IdxRecord>> {
     Ok(records)
 }
 
-fn read_idx_into_duck_db(filename: String) -> anyhow::Result<()> {
-    let conn = duckdb::Connection::open_in_memory()?;
-
-    conn.execute_batch(
-        r"CREATE TABLE grib_message (
-            msg_id INTEGER,
-            byte_offset INTEGER,
-            init_time TIMESTAMP,
-            nwp_variable VARCHAR,
-            vertical_level VARCHAR,
-            step_raw VARCHAR,
-            step INTERVAL,
-            ens_member VARCHAR,
-            );
-        ",
-    )?;
-    // TODO: Change to enums: nwp_variable, vertical_level, ensemble_member
-    // TODO: Add filename of GRIB file (with full path)
-
-    conn.execute_batch(
-        r"COPY grib_message (msg_id, byte_offset, init_time, nwp_variable, vertical_level, step_raw, ens_member) FROM 
-            '/home/jack/dev/rust/hypergrib/gec00.t00z.pgrb2af000.idx'
-            (
-              DELIMITER ':',
-              FORMAT CSV,
-              HEADER false,
-              AUTO_DETECT false,
-              TIMESTAMPFORMAT 'd=%Y%m%d%H'
-            );
-          ",
-    )?;
-    // TODO: Pass in filename argument!
-    // TODO: Convert step_raw to step, maybe by first storing step_raw as VARCHAR, replacing 'anl'
-    // with '0', taking just the first two characters of the string, converting that to an int, and
-    // passing that to [`to_hours(integer)`](https://duckdb.org/docs/sql/functions/interval#to_hoursinteger)
-
-    // Print some test data:
-    let mut stmt = conn.prepare("SELECT * FROM grib_message WHERE msg_id < 5")?;
-    let mut rows = stmt.query([])?;
-    while let Some(row) = rows.next()? {
-        println!("{:?}", row.get::<_, u32>(1)?);
-    }
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -98,10 +52,4 @@ mod tests {
         );
         Ok(())
     }
-
-    #[test]
-    fn test_read_idx_into_duck_db() -> anyhow::Result<()> {
-        read_idx_into_duck_db(String::from("FILENAME NOT USED YET!"))?;
-        Ok(())
-    }
 }