Organize migrations for multiple backends (#134)

* re-estructured migrations

* change workflow

* change workflow again

* fix readme link and fix readme explanation to setup `PostgreSQL`

* modify `diesel_cli` install

* I believe this does the same thing as postgreSQL initial setup

* implementing postgres migrations programatically

* add some logs in example

* sqlite migrations

.github/workflows/rust.yml

@@ -55,12 +55,12 @@ jobs:
           command: install
           args: diesel_cli --no-default-features --features "postgres"
 
-      - name: Setup db
-        working-directory: ./fang
+      - name: Setup Postgres db
+        working-directory: ./fang/postgres_migrations
         run: diesel setup
 
       - name: Change working dir
-        working-directory: ./..
+        working-directory: ./../..
         run: pwd
 
       - name: Run tests

.gitignore

@@ -3,3 +3,4 @@ Cargo.lock
 src/schema.rs
 docs/content/docs/CHANGELOG.md
 docs/content/docs/README.md
+fang.db

Makefile

@@ -1,15 +1,41 @@
-db:
+db_postgres:
 	docker run --rm -d --name postgres -p 5432:5432 \
   -e POSTGRES_DB=fang \
   -e POSTGRES_USER=postgres \
   -e POSTGRES_PASSWORD=postgres \
   postgres:latest
+
+# login is root fang
+db_mysql:
+	docker run --rm -d --name mysql -p 3306:3306 \
+  	-e MYSQL_DATABASE=fang \
+  	-e MYSQL_ROOT_PASSWORD=fang \
+	-e TZ=UTC \
+  	mysql:latest
+
+db_sqlite:
+	sqlite3 fang.db "VACUUM;"
+
 clippy:
-	cargo clippy --all-features
-diesel:
-	cd fang && DATABASE_URL=postgres://postgres:postgres@localhost/fang diesel migration run
-stop:
+	cargo clippy --verbose --all-targets --all-features -- -D warnings
+
+diesel_sqlite:
+	cd fang/sqlite_migrations && DATABASE_URL=sqlite://../../fang.db diesel migration run
+
+diesel_postgres:
+	cd fang/postgres_migrations && DATABASE_URL=postgres://postgres:postgres@localhost/fang diesel migration run
+
+diesel_mysql:
+	cd fang/mysql_migrations && DATABASE_URL=mysql://root:[email protected]/fang diesel migration run
+
+stop_mysql: 
+	docker kill mysql
+
+stop_postgres:
 	docker kill postgres
+
+stop_sqlite:
+	rm fang.db
 tests:
 	DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture
 

fang/Cargo.toml

@@ -15,13 +15,22 @@ rust-version = "1.62"
 doctest = false
 
 [features]
-default = ["blocking", "asynk", "derive-error"]
+default = ["blocking", "asynk", "derive-error", "postgres", "mysql" , "sqlite", "migrations_postgres", "migrations_sqlite" , "migrations_mysql"]
 blocking = ["dep:diesel", "dep:diesel-derive-enum", "dep:dotenvy"]
 asynk = ["dep:bb8-postgres", "dep:postgres-types", "dep:tokio", "dep:async-trait", "dep:async-recursion"]
 derive-error = ["dep:fang-derive-error"]
+postgres = ["diesel?/postgres" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"]
+sqlite = ["diesel?/sqlite" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"]
+mysql = ["diesel?/mysql" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"]
+migrations_postgres = ["migrations"]
+migrations_sqlite = ["migrations"]
+migrations_mysql = ["migrations"]
+migrations = ["dep:diesel_migrations"]
+
 
 [dev-dependencies]
 fang-derive-error = { version = "0.1.0"}
+diesel_migrations = { version = "2.1" , features = ["postgres", "sqlite" , "mysql"]}
 
 [dependencies]
 cron = "0.12"
@@ -40,8 +49,8 @@ fang-derive-error = { version = "0.1.0" , optional = true}
 
 [dependencies.diesel]
 version = "2.1"
-features = ["postgres", "serde_json", "chrono", "uuid", "r2d2"]
 optional = true
+default-features = false
 
 [dependencies.diesel-derive-enum]
 version = "2.1"
@@ -74,3 +83,8 @@ optional = true
 [dependencies.async-recursion]
 version = "1"
 optional = true
+
+[dependencies.diesel_migrations]
+version = "2.1.0"
+optional = true
+default-features = false

fang/README.md

@@ -53,7 +53,7 @@ fang = { version = "0.10.4" }
 
 *Supports rustc 1.62+*
 
-2. Create the `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/migrations/2022-08-20-151615_create_fang_tasks/up.sql).
+2. Create the `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/up.sql).
 
 ## Usage
 
@@ -394,18 +394,18 @@ Set sleep params with worker pools `TypeBuilder` in both modules.
 ### Running tests locally
 - Install diesel_cli.
 ```
-cargo install diesel_cli
+cargo install diesel_cli --no-default-features --features "postgres sqlite mysql"
 ```
 - Install docker on your machine.
 
 - Run a Postgres docker container. (See in Makefile.)
 ```
-make db
+make db_postgres
 ```
 
 - Run the migrations
 ```
-make diesel
+make diesel_postgres
 ```
 
 - Run tests

fang/fang_examples/asynk/simple_async_worker/Cargo.toml

@@ -6,8 +6,9 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-fang = { path = "../../../" , features = ["asynk"]}
+fang = { path = "../../../" , features = ["asynk", "postgres"]}
 env_logger = "0.9.0"
 log = "0.4.0"
 dotenvy = "0.15"
+diesel = {version = "2.1" , features = ["postgres"]}
 tokio = { version = "1", features = ["full"] }

fang/fang_examples/asynk/simple_async_worker/src/main.rs

@@ -1,7 +1,11 @@
+use connection::Connection;
+use diesel::connection;
+use diesel::PgConnection;
 use dotenvy::dotenv;
 use fang::asynk::async_queue::AsyncQueue;
 use fang::asynk::async_queue::AsyncQueueable;
 use fang::asynk::async_worker_pool::AsyncWorkerPool;
+use fang::run_migrations_postgres;
 use fang::AsyncRunnable;
 use fang::NoTls;
 use simple_async_worker::MyFailingTask;
@@ -15,6 +19,16 @@ async fn main() {
     env_logger::init();
     let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
 
+    let mut connection = PgConnection::establish(&database_url).unwrap();
+
+    log::info!("Running migrations");
+
+    run_migrations_postgres(&mut connection).unwrap();
+
+    log::info!("Migrations done :D");
+
+    drop(connection);
+
     log::info!("Starting...");
     let max_pool_size: u32 = 3;
     let mut queue = AsyncQueue::builder()

fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE fang_tasks;

fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql

@@ -0,0 +1,22 @@
+-- Your SQL goes here
+
+
+-- docker exec -ti mysql mysql -u root -pfang -P 3360 fang -e "$(catn fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql)"
+
+CREATE TABLE fang_tasks (
+    id VARCHAR(36) DEFAULT (uuid()) PRIMARY KEY,
+    metadata JSON NOT NULL,
+    error_message TEXT,
+    state ENUM('new', 'in_progress', 'failed', 'finished', 'retried') NOT NULL DEFAULT 'new',
+    task_type VARCHAR(255) NOT NULL DEFAULT 'common', -- TEXT type can not have default value, stupid MySQL policy
+    uniq_hash CHAR(64),
+    retries INTEGER NOT NULL DEFAULT 0,
+    scheduled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX fang_tasks_state_index ON fang_tasks(state);
+CREATE INDEX fang_tasks_type_index ON fang_tasks(task_type);
+CREATE INDEX fang_tasks_scheduled_at_index ON fang_tasks(scheduled_at);
+CREATE INDEX fang_tasks_uniq_hash ON fang_tasks(uniq_hash);

fang/postgres_migrations/diesel.toml

@@ -0,0 +1,2 @@
+[print_schema]
+file = "../src/blocking/postgres_schema.rs"

fang/sqlite_migrations/diesel.toml

@@ -0,0 +1,2 @@
+[print_schema]
+file = "../src/blocking/sqlite_schema.rs"

fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql

@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE fang_tasks;

fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql

@@ -0,0 +1,28 @@
+-- Your SQL goes here
+
+
+-- docker exec -ti mysql mysql -u root -pfang -P 3360 fang -e "$(catn fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql)"
+
+CREATE TABLE fang_tasks (
+    id TEXT CHECK (LENGTH(id) = 36) NOT NULL PRIMARY KEY, -- UUID generated inside the language
+    -- why uuid is a text ? https://stackoverflow.com/questions/17277735/using-uuids-in-sqlite
+    metadata TEXT NOT NULL, 
+    -- why metadata is text ? https://stackoverflow.com/questions/16603621/how-to-store-json-object-in-sqlite-database#16603687
+    error_message TEXT,
+    state TEXT CHECK ( state IN ('new', 'in_progress', 'failed', 'finished', 'retried') ) NOT NULL DEFAULT 'new',
+    -- why state is a text ? https://stackoverflow.com/questions/5299267/how-to-create-enum-type-in-sqlite#17203007
+    task_type TEXT NOT NULL DEFAULT 'common',
+    uniq_hash CHAR(64),
+    retries INTEGER NOT NULL DEFAULT 0,
+    -- The datetime() function returns the date and time as text in this formats: YYYY-MM-DD HH:MM:SS. 
+    -- https://www.sqlite.org/lang_datefunc.html
+    scheduled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    -- why timestamps are texts ? https://www.sqlite.org/datatype3.html
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX fang_tasks_state_index ON fang_tasks(state);
+CREATE INDEX fang_tasks_type_index ON fang_tasks(task_type);
+CREATE INDEX fang_tasks_scheduled_at_index ON fang_tasks(scheduled_at);
+CREATE INDEX fang_tasks_uniq_hash ON fang_tasks(uniq_hash);

fang/src/blocking.rs

@@ -1,12 +1,14 @@
 mod error;
+pub mod mysql_schema;
+pub mod postgres_schema;
 pub mod queue;
 pub mod runnable;
-pub mod schema;
+pub mod sqlite_schema;
 pub mod worker;
 pub mod worker_pool;
 
+pub use postgres_schema::*;
 pub use queue::*;
 pub use runnable::Runnable;
-pub use schema::*;
 pub use worker::*;
 pub use worker_pool::*;

fang/src/blocking/mysql_schema.rs

@@ -0,0 +1,29 @@
+// @generated automatically by Diesel CLI.
+
+pub mod sql_types {
+    #[derive(diesel::sql_types::SqlType)]
+    #[diesel(mysql_type(name = "Enum"))]
+    pub struct FangTasksStateEnum;
+}
+
+diesel::table! {
+    use diesel::sql_types::*;
+    use super::sql_types::FangTasksStateEnum;
+
+    fang_tasks (id) {
+        #[max_length = 36]
+        id -> Varchar,
+        metadata -> Json,
+        error_message -> Nullable<Text>,
+        #[max_length = 11]
+        state -> FangTasksStateEnum,
+        #[max_length = 255]
+        task_type -> Varchar,
+        #[max_length = 64]
+        uniq_hash -> Nullable<Char>,
+        retries -> Integer,
+        scheduled_at -> Timestamp,
+        created_at -> Timestamp,
+        updated_at -> Timestamp,
+    }
+}

fang/src/blocking/schema.rs → fang/src/blocking/postgres_schema.rs

@@ -16,6 +16,7 @@ diesel::table! {
         error_message -> Nullable<Text>,
         state -> FangTaskState,
         task_type -> Varchar,
+        #[max_length = 64]
         uniq_hash -> Nullable<Bpchar>,
         retries -> Int4,
         scheduled_at -> Timestamptz,

fang/src/blocking/queue.rs

@@ -1,8 +1,8 @@
 #[cfg(test)]
 mod queue_tests;
 
+use crate::postgres_schema::fang_tasks;
 use crate::runnable::Runnable;
-use crate::schema::fang_tasks;
 use crate::CronError;
 use crate::FangTaskState;
 use crate::Scheduled::*;

fang/src/blocking/sqlite_schema.rs

@@ -0,0 +1,16 @@
+// @generated automatically by Diesel CLI.
+
+diesel::table! {
+    fang_tasks (id) {
+        id -> Text,
+        metadata -> Text,
+        error_message -> Nullable<Text>,
+        state -> Text,
+        task_type -> Text,
+        uniq_hash -> Nullable<Text>,
+        retries -> Integer,
+        scheduled_at -> Timestamp,
+        created_at -> Timestamp,
+        updated_at -> Timestamp,
+    }
+}