From d91389be38840c412cf06f331ac17623c652460f Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:12:33 +0200 Subject: [PATCH] Organize migrations for multiple backends (#134) * re-estructured migrations * change workflow * change workflow again * fix readme link and fix readme explanation to setup `PostgreSQL` * modify `diesel_cli` install * I believe this does the same thing as postgreSQL initial setup * implementing postgres migrations programatically * add some logs in example * sqlite migrations --- .github/workflows/rust.yml | 6 +-- .gitignore | 1 + Makefile | 36 +++++++++++-- fang/Cargo.toml | 18 ++++++- fang/README.md | 8 +-- fang/diesel.toml | 2 - .../asynk/simple_async_worker/Cargo.toml | 3 +- .../asynk/simple_async_worker/src/main.rs | 14 +++++ .../down.sql | 2 + .../up.sql | 22 ++++++++ fang/postgres_migrations/diesel.toml | 2 + .../down.sql | 0 .../up.sql | 0 .../down.sql | 0 .../up.sql | 0 fang/sqlite_migrations/diesel.toml | 2 + .../down.sql | 2 + .../up.sql | 28 ++++++++++ fang/src/blocking.rs | 6 ++- fang/src/blocking/mysql_schema.rs | 29 ++++++++++ .../{schema.rs => postgres_schema.rs} | 1 + fang/src/blocking/queue.rs | 2 +- fang/src/blocking/sqlite_schema.rs | 16 ++++++ fang/src/lib.rs | 54 ++++++++++++++++++- 24 files changed, 233 insertions(+), 21 deletions(-) delete mode 100644 fang/diesel.toml create mode 100644 fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql create mode 100644 fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql create mode 100644 fang/postgres_migrations/diesel.toml rename fang/{ => postgres_migrations}/migrations/00000000000000_diesel_initial_setup/down.sql (100%) rename fang/{ => postgres_migrations}/migrations/00000000000000_diesel_initial_setup/up.sql (100%) rename fang/{ => postgres_migrations}/migrations/2022-08-20-151615_create_fang_tasks/down.sql (100%) rename fang/{ => postgres_migrations}/migrations/2022-08-20-151615_create_fang_tasks/up.sql (100%) create mode 100644 fang/sqlite_migrations/diesel.toml create mode 100644 fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql create mode 100644 fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql create mode 100644 fang/src/blocking/mysql_schema.rs rename fang/src/blocking/{schema.rs => postgres_schema.rs} (95%) create mode 100644 fang/src/blocking/sqlite_schema.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 380fde32..6e6d43a1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -55,12 +55,12 @@ jobs: command: install args: diesel_cli --no-default-features --features "postgres" - - name: Setup db - working-directory: ./fang + - name: Setup Postgres db + working-directory: ./fang/postgres_migrations run: diesel setup - name: Change working dir - working-directory: ./.. + working-directory: ./../.. run: pwd - name: Run tests diff --git a/.gitignore b/.gitignore index 9db20c89..57b9c821 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ Cargo.lock src/schema.rs docs/content/docs/CHANGELOG.md docs/content/docs/README.md +fang.db diff --git a/Makefile b/Makefile index ec8c2e8c..08a0723d 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,41 @@ -db: +db_postgres: docker run --rm -d --name postgres -p 5432:5432 \ -e POSTGRES_DB=fang \ -e POSTGRES_USER=postgres \ -e POSTGRES_PASSWORD=postgres \ postgres:latest + +# login is root fang +db_mysql: + docker run --rm -d --name mysql -p 3306:3306 \ + -e MYSQL_DATABASE=fang \ + -e MYSQL_ROOT_PASSWORD=fang \ + -e TZ=UTC \ + mysql:latest + +db_sqlite: + sqlite3 fang.db "VACUUM;" + clippy: - cargo clippy --all-features -diesel: - cd fang && DATABASE_URL=postgres://postgres:postgres@localhost/fang diesel migration run -stop: + cargo clippy --verbose --all-targets --all-features -- -D warnings + +diesel_sqlite: + cd fang/sqlite_migrations && DATABASE_URL=sqlite://../../fang.db diesel migration run + +diesel_postgres: + cd fang/postgres_migrations && DATABASE_URL=postgres://postgres:postgres@localhost/fang diesel migration run + +diesel_mysql: + cd fang/mysql_migrations && DATABASE_URL=mysql://root:fang@127.0.0.1/fang diesel migration run + +stop_mysql: + docker kill mysql + +stop_postgres: docker kill postgres + +stop_sqlite: + rm fang.db tests: DATABASE_URL=postgres://postgres:postgres@localhost/fang cargo test --all-features -- --color always --nocapture diff --git a/fang/Cargo.toml b/fang/Cargo.toml index 8ca8a5c6..dfa88d28 100644 --- a/fang/Cargo.toml +++ b/fang/Cargo.toml @@ -15,13 +15,22 @@ rust-version = "1.62" doctest = false [features] -default = ["blocking", "asynk", "derive-error"] +default = ["blocking", "asynk", "derive-error", "postgres", "mysql" , "sqlite", "migrations_postgres", "migrations_sqlite" , "migrations_mysql"] blocking = ["dep:diesel", "dep:diesel-derive-enum", "dep:dotenvy"] asynk = ["dep:bb8-postgres", "dep:postgres-types", "dep:tokio", "dep:async-trait", "dep:async-recursion"] derive-error = ["dep:fang-derive-error"] +postgres = ["diesel?/postgres" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"] +sqlite = ["diesel?/sqlite" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"] +mysql = ["diesel?/mysql" , "diesel?/serde_json", "diesel?/chrono" , "diesel?/uuid" , "diesel?/r2d2"] +migrations_postgres = ["migrations"] +migrations_sqlite = ["migrations"] +migrations_mysql = ["migrations"] +migrations = ["dep:diesel_migrations"] + [dev-dependencies] fang-derive-error = { version = "0.1.0"} +diesel_migrations = { version = "2.1" , features = ["postgres", "sqlite" , "mysql"]} [dependencies] cron = "0.12" @@ -40,8 +49,8 @@ fang-derive-error = { version = "0.1.0" , optional = true} [dependencies.diesel] version = "2.1" -features = ["postgres", "serde_json", "chrono", "uuid", "r2d2"] optional = true +default-features = false [dependencies.diesel-derive-enum] version = "2.1" @@ -74,3 +83,8 @@ optional = true [dependencies.async-recursion] version = "1" optional = true + +[dependencies.diesel_migrations] +version = "2.1.0" +optional = true +default-features = false \ No newline at end of file diff --git a/fang/README.md b/fang/README.md index 30110204..df52d776 100644 --- a/fang/README.md +++ b/fang/README.md @@ -53,7 +53,7 @@ fang = { version = "0.10.4" } *Supports rustc 1.62+* -2. Create the `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/migrations/2022-08-20-151615_create_fang_tasks/up.sql). +2. Create the `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/up.sql). ## Usage @@ -394,18 +394,18 @@ Set sleep params with worker pools `TypeBuilder` in both modules. ### Running tests locally - Install diesel_cli. ``` -cargo install diesel_cli +cargo install diesel_cli --no-default-features --features "postgres sqlite mysql" ``` - Install docker on your machine. - Run a Postgres docker container. (See in Makefile.) ``` -make db +make db_postgres ``` - Run the migrations ``` -make diesel +make diesel_postgres ``` - Run tests diff --git a/fang/diesel.toml b/fang/diesel.toml deleted file mode 100644 index 73a18e39..00000000 --- a/fang/diesel.toml +++ /dev/null @@ -1,2 +0,0 @@ -[print_schema] -file = "src/blocking/schema.rs" \ No newline at end of file diff --git a/fang/fang_examples/asynk/simple_async_worker/Cargo.toml b/fang/fang_examples/asynk/simple_async_worker/Cargo.toml index c12b9658..5e9d2446 100644 --- a/fang/fang_examples/asynk/simple_async_worker/Cargo.toml +++ b/fang/fang_examples/asynk/simple_async_worker/Cargo.toml @@ -6,8 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -fang = { path = "../../../" , features = ["asynk"]} +fang = { path = "../../../" , features = ["asynk", "postgres"]} env_logger = "0.9.0" log = "0.4.0" dotenvy = "0.15" +diesel = {version = "2.1" , features = ["postgres"]} tokio = { version = "1", features = ["full"] } diff --git a/fang/fang_examples/asynk/simple_async_worker/src/main.rs b/fang/fang_examples/asynk/simple_async_worker/src/main.rs index c4376ca7..5a148a5d 100644 --- a/fang/fang_examples/asynk/simple_async_worker/src/main.rs +++ b/fang/fang_examples/asynk/simple_async_worker/src/main.rs @@ -1,7 +1,11 @@ +use connection::Connection; +use diesel::connection; +use diesel::PgConnection; use dotenvy::dotenv; use fang::asynk::async_queue::AsyncQueue; use fang::asynk::async_queue::AsyncQueueable; use fang::asynk::async_worker_pool::AsyncWorkerPool; +use fang::run_migrations_postgres; use fang::AsyncRunnable; use fang::NoTls; use simple_async_worker::MyFailingTask; @@ -15,6 +19,16 @@ async fn main() { env_logger::init(); let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let mut connection = PgConnection::establish(&database_url).unwrap(); + + log::info!("Running migrations"); + + run_migrations_postgres(&mut connection).unwrap(); + + log::info!("Migrations done :D"); + + drop(connection); + log::info!("Starting..."); let max_pool_size: u32 = 3; let mut queue = AsyncQueue::builder() diff --git a/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql b/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql new file mode 100644 index 00000000..08038a8f --- /dev/null +++ b/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE fang_tasks; \ No newline at end of file diff --git a/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql b/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql new file mode 100644 index 00000000..b882b72b --- /dev/null +++ b/fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql @@ -0,0 +1,22 @@ +-- Your SQL goes here + + +-- docker exec -ti mysql mysql -u root -pfang -P 3360 fang -e "$(catn fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql)" + +CREATE TABLE fang_tasks ( + id VARCHAR(36) DEFAULT (uuid()) PRIMARY KEY, + metadata JSON NOT NULL, + error_message TEXT, + state ENUM('new', 'in_progress', 'failed', 'finished', 'retried') NOT NULL DEFAULT 'new', + task_type VARCHAR(255) NOT NULL DEFAULT 'common', -- TEXT type can not have default value, stupid MySQL policy + uniq_hash CHAR(64), + retries INTEGER NOT NULL DEFAULT 0, + scheduled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX fang_tasks_state_index ON fang_tasks(state); +CREATE INDEX fang_tasks_type_index ON fang_tasks(task_type); +CREATE INDEX fang_tasks_scheduled_at_index ON fang_tasks(scheduled_at); +CREATE INDEX fang_tasks_uniq_hash ON fang_tasks(uniq_hash); \ No newline at end of file diff --git a/fang/postgres_migrations/diesel.toml b/fang/postgres_migrations/diesel.toml new file mode 100644 index 00000000..09a233a6 --- /dev/null +++ b/fang/postgres_migrations/diesel.toml @@ -0,0 +1,2 @@ +[print_schema] +file = "../src/blocking/postgres_schema.rs" \ No newline at end of file diff --git a/fang/migrations/00000000000000_diesel_initial_setup/down.sql b/fang/postgres_migrations/migrations/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from fang/migrations/00000000000000_diesel_initial_setup/down.sql rename to fang/postgres_migrations/migrations/00000000000000_diesel_initial_setup/down.sql diff --git a/fang/migrations/00000000000000_diesel_initial_setup/up.sql b/fang/postgres_migrations/migrations/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from fang/migrations/00000000000000_diesel_initial_setup/up.sql rename to fang/postgres_migrations/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/fang/migrations/2022-08-20-151615_create_fang_tasks/down.sql b/fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/down.sql similarity index 100% rename from fang/migrations/2022-08-20-151615_create_fang_tasks/down.sql rename to fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/down.sql diff --git a/fang/migrations/2022-08-20-151615_create_fang_tasks/up.sql b/fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/up.sql similarity index 100% rename from fang/migrations/2022-08-20-151615_create_fang_tasks/up.sql rename to fang/postgres_migrations/migrations/2022-08-20-151615_create_fang_tasks/up.sql diff --git a/fang/sqlite_migrations/diesel.toml b/fang/sqlite_migrations/diesel.toml new file mode 100644 index 00000000..916b67f1 --- /dev/null +++ b/fang/sqlite_migrations/diesel.toml @@ -0,0 +1,2 @@ +[print_schema] +file = "../src/blocking/sqlite_schema.rs" \ No newline at end of file diff --git a/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql b/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql new file mode 100644 index 00000000..08038a8f --- /dev/null +++ b/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE fang_tasks; \ No newline at end of file diff --git a/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql b/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql new file mode 100644 index 00000000..afc60e3e --- /dev/null +++ b/fang/sqlite_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql @@ -0,0 +1,28 @@ +-- Your SQL goes here + + +-- docker exec -ti mysql mysql -u root -pfang -P 3360 fang -e "$(catn fang/mysql_migrations/migrations/2023-08-17-102017_create_fang_tasks/up.sql)" + +CREATE TABLE fang_tasks ( + id TEXT CHECK (LENGTH(id) = 36) NOT NULL PRIMARY KEY, -- UUID generated inside the language + -- why uuid is a text ? https://stackoverflow.com/questions/17277735/using-uuids-in-sqlite + metadata TEXT NOT NULL, + -- why metadata is text ? https://stackoverflow.com/questions/16603621/how-to-store-json-object-in-sqlite-database#16603687 + error_message TEXT, + state TEXT CHECK ( state IN ('new', 'in_progress', 'failed', 'finished', 'retried') ) NOT NULL DEFAULT 'new', + -- why state is a text ? https://stackoverflow.com/questions/5299267/how-to-create-enum-type-in-sqlite#17203007 + task_type TEXT NOT NULL DEFAULT 'common', + uniq_hash CHAR(64), + retries INTEGER NOT NULL DEFAULT 0, + -- The datetime() function returns the date and time as text in this formats: YYYY-MM-DD HH:MM:SS. + -- https://www.sqlite.org/lang_datefunc.html + scheduled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + -- why timestamps are texts ? https://www.sqlite.org/datatype3.html + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX fang_tasks_state_index ON fang_tasks(state); +CREATE INDEX fang_tasks_type_index ON fang_tasks(task_type); +CREATE INDEX fang_tasks_scheduled_at_index ON fang_tasks(scheduled_at); +CREATE INDEX fang_tasks_uniq_hash ON fang_tasks(uniq_hash); \ No newline at end of file diff --git a/fang/src/blocking.rs b/fang/src/blocking.rs index db0dfbfe..16e36385 100644 --- a/fang/src/blocking.rs +++ b/fang/src/blocking.rs @@ -1,12 +1,14 @@ mod error; +pub mod mysql_schema; +pub mod postgres_schema; pub mod queue; pub mod runnable; -pub mod schema; +pub mod sqlite_schema; pub mod worker; pub mod worker_pool; +pub use postgres_schema::*; pub use queue::*; pub use runnable::Runnable; -pub use schema::*; pub use worker::*; pub use worker_pool::*; diff --git a/fang/src/blocking/mysql_schema.rs b/fang/src/blocking/mysql_schema.rs new file mode 100644 index 00000000..4b98594f --- /dev/null +++ b/fang/src/blocking/mysql_schema.rs @@ -0,0 +1,29 @@ +// @generated automatically by Diesel CLI. + +pub mod sql_types { + #[derive(diesel::sql_types::SqlType)] + #[diesel(mysql_type(name = "Enum"))] + pub struct FangTasksStateEnum; +} + +diesel::table! { + use diesel::sql_types::*; + use super::sql_types::FangTasksStateEnum; + + fang_tasks (id) { + #[max_length = 36] + id -> Varchar, + metadata -> Json, + error_message -> Nullable, + #[max_length = 11] + state -> FangTasksStateEnum, + #[max_length = 255] + task_type -> Varchar, + #[max_length = 64] + uniq_hash -> Nullable, + retries -> Integer, + scheduled_at -> Timestamp, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} diff --git a/fang/src/blocking/schema.rs b/fang/src/blocking/postgres_schema.rs similarity index 95% rename from fang/src/blocking/schema.rs rename to fang/src/blocking/postgres_schema.rs index 4f9dff21..15b051c7 100644 --- a/fang/src/blocking/schema.rs +++ b/fang/src/blocking/postgres_schema.rs @@ -16,6 +16,7 @@ diesel::table! { error_message -> Nullable, state -> FangTaskState, task_type -> Varchar, + #[max_length = 64] uniq_hash -> Nullable, retries -> Int4, scheduled_at -> Timestamptz, diff --git a/fang/src/blocking/queue.rs b/fang/src/blocking/queue.rs index 11097735..3716833f 100644 --- a/fang/src/blocking/queue.rs +++ b/fang/src/blocking/queue.rs @@ -1,8 +1,8 @@ #[cfg(test)] mod queue_tests; +use crate::postgres_schema::fang_tasks; use crate::runnable::Runnable; -use crate::schema::fang_tasks; use crate::CronError; use crate::FangTaskState; use crate::Scheduled::*; diff --git a/fang/src/blocking/sqlite_schema.rs b/fang/src/blocking/sqlite_schema.rs new file mode 100644 index 00000000..1062df45 --- /dev/null +++ b/fang/src/blocking/sqlite_schema.rs @@ -0,0 +1,16 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + fang_tasks (id) { + id -> Text, + metadata -> Text, + error_message -> Nullable, + state -> Text, + task_type -> Text, + uniq_hash -> Nullable, + retries -> Integer, + scheduled_at -> Timestamp, + created_at -> Timestamp, + updated_at -> Timestamp, + } +} diff --git a/fang/src/lib.rs b/fang/src/lib.rs index f5047d91..e6abb131 100644 --- a/fang/src/lib.rs +++ b/fang/src/lib.rs @@ -108,7 +108,7 @@ pub struct FangError { #[cfg_attr(feature = "asynk", postgres(name = "fang_task_state"))] #[cfg_attr( feature = "blocking", - ExistingTypePath = "crate::schema::sql_types::FangTaskState" + ExistingTypePath = "crate::postgres_schema::sql_types::FangTaskState" )] pub enum FangTaskState { /// The task is ready to be executed @@ -205,3 +205,55 @@ pub use async_trait::async_trait; #[cfg(feature = "derive-error")] pub use fang_derive_error::ToFangError; + +#[cfg(feature = "migrations")] +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; + +#[cfg(feature = "migrations")] +use std::error::Error as SomeError; + +#[cfg(feature = "migrations_postgres")] +use diesel::pg::Pg; + +#[cfg(feature = "migrations_postgres")] +pub const MIGRATIONS_POSTGRES: EmbeddedMigrations = + embed_migrations!("postgres_migrations/migrations"); + +#[cfg(feature = "migrations_postgres")] +pub fn run_migrations_postgres( + connection: &mut impl MigrationHarness, +) -> Result<(), Box> { + connection.run_pending_migrations(MIGRATIONS_POSTGRES)?; + + Ok(()) +} + +#[cfg(feature = "migrations_mysql")] +use diesel::mysql::Mysql; + +#[cfg(feature = "migrations_mysql")] +pub const MIGRATIONS_MYSQL: EmbeddedMigrations = embed_migrations!("mysql_migrations/migrations"); + +#[cfg(feature = "migrations_mysql")] +pub fn run_migrations_mysql( + connection: &mut impl MigrationHarness, +) -> Result<(), Box> { + connection.run_pending_migrations(MIGRATIONS_MYSQL)?; + + Ok(()) +} + +#[cfg(feature = "migrations_sqlite")] +use diesel::sqlite::Sqlite; + +#[cfg(feature = "migrations_sqlite")] +pub const MIGRATIONS_SQLITE: EmbeddedMigrations = embed_migrations!("sqlite_migrations/migrations"); + +#[cfg(feature = "migrations_sqlite")] +pub fn run_migrations_sqlite( + connection: &mut impl MigrationHarness, +) -> Result<(), Box> { + connection.run_pending_migrations(MIGRATIONS_SQLITE)?; + + Ok(()) +}