Merge pull request #35 from thedataquarry/rust-ci

Add Rust CI

.github/workflows/testing.yml

@@ -0,0 +1,54 @@
+name: Testing
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+  POSTGRES_PASSWORD: "test_password"
+  DATABASE_URL: "postgresql://postgres:[email protected]:5432/etl"
+jobs:
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/[email protected]
+      - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
+        run: sh ./scripts/prep_tests.sh
+      - name: Run cargo clippy
+        run: cargo clippy --all-targets -- --deny warnings
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/[email protected]
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check
+  test:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+      - name: Cache dependencies
+        uses: Swatinem/[email protected]
+      - name: Prep data and sqlx # This prevents sqlx errors if the database isn't running
+        run: sh ./scripts/prep_tests.sh
+      - name: Run cargo test
+        run: cargo test

.gitignore

@@ -164,3 +164,5 @@ cython_debug/
 
 # Rust
 /target
+
+/scripts/data/*

pieces/intro/rust/src/main.rs

@@ -153,11 +153,18 @@ fn run8() {
     // Check for presence of value
     let value = "AMD Ryzen 3";
     let mut values = processors.values();
-    println!("Is \"AMD Ryzen 3\" in the hashmap of processors?: {}", values.any(|v| v == &value));
+    println!(
+        "Is \"AMD Ryzen 3\" in the hashmap of processors?: {}",
+        values.any(|v| v == &value)
+    );
     // Lookup by key
     let key = "13900KS";
     let lookup_by_key = processors.get(key);
-    println!("Key \"{}\" has the value \"{}\"", key, lookup_by_key.unwrap());
+    println!(
+        "Key \"{}\" has the value \"{}\"",
+        key,
+        lookup_by_key.unwrap()
+    );
     /*
     Is "AMD Ryzen 3" in the hashmap of processors?: true
     Key "13900KS" has the value "Intel Core i9"
@@ -178,7 +185,10 @@ fn run9() {
     processors.insert("AMD Ryzen 5");
     // Check for presence of value
     let value = "AMD Ryzen 3";
-    println!("Is \"AMD Ryzen 3\" in the hashset of processors?: {}", processors.contains(&value));
+    println!(
+        "Is \"AMD Ryzen 3\" in the hashset of processors?: {}",
+        processors.contains(&value)
+    );
     /*
     Is "AMD Ryzen 3" in the hashset of processors?: true
     */

pieces/postgres_etl/rust/src/main.rs

@@ -51,7 +51,6 @@ async fn main() -> Result<(), sqlx::Error> {
     dotenv().ok();
     // Obtain connection
     let pg_uri = dotenvy::var("DATABASE_URL").unwrap();
-    // let pool = Arc::new(PgPool::connect(&pg_uri).await.unwrap());
     let pool = PgPoolOptions::new()
         .min_connections(5)
         .max_connections(5)

scripts/docker-compose.yml

@@ -0,0 +1,13 @@
+version: "3.9"
+
+services:
+  database:
+    image: "postgres:latest"
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    volumes:
+      - ./pg/:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+      - ./data:/data

scripts/init.sql

@@ -0,0 +1,21 @@
+SELECT 'CREATE DATABASE etl'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'etl')\gexec
+
+\connect etl
+
+CREATE TABLE IF NOT EXISTS persons(
+    id integer PRIMARY KEY,
+    name text,
+    age smallint,
+    isMarried boolean,
+    city text,
+    state text,
+    country text
+)\gexec
+
+TRUNCATE TABLE persons\gexec
+
+COPY persons(id, name, age, isMarried, city, state, country)
+FROM '/data/persons.csv'
+DELIMITER ','
+CSV HEADER\gexec

scripts/prep_tests.sh

@@ -0,0 +1,5 @@
+#!bin/bash
+
+mkdir scripts/data
+cp pieces/postgres_etl/data/persons.csv scripts/data
+docker compose -f scripts/docker-compose.yml up -d