diff --git a/.env.prod.example b/.env.prod.example index dfdf6868..6855421d 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -1,5 +1,3 @@ -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= ENV= MONGODB_URL= MS_CLIENT_ID= diff --git a/Cargo.lock b/Cargo.lock index f73758bc..90c14386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -235,6 +235,10 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "automation" +version = "0.0.0" + [[package]] name = "axum" version = "0.7.9" @@ -524,14 +528,14 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "changelog-generator" -version = "0.1.0" +version = "0.0.0" dependencies = [ "anyhow", "async-openai", "chrono", "clap", "dotenv", - "env_logger 0.11.8", + "env_logger", "log", "octocrab", "serde", @@ -700,15 +704,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if 1.0.0", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -990,27 +985,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -1076,16 +1050,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "env_filter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] - [[package]] name = "env_logger" version = "0.10.2" @@ -1099,19 +1063,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "env_logger" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -1628,21 +1579,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.20.9", - "rustls-native-certs 0.6.3", - "tokio", - "tokio-rustls 0.23.4", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1974,30 +1910,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "jiff" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.101", -] - [[package]] name = "js-sys" version = "0.3.77" @@ -2017,7 +1929,7 @@ dependencies = [ "base64 0.22.1", "js-sys", "pem", - "ring 0.17.14", + "ring", "serde", "serde_json", "simple_asn1", @@ -2035,16 +1947,6 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.9.1", - "libc", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2139,12 +2041,12 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "mcgill-courses-scraper" -version = "0.1.0" +version = "0.0.0" dependencies = [ "anyhow", "chrono", "clap", - "env_logger 0.11.8", + "env_logger", "include_dir", "log", "mockito", @@ -2155,24 +2057,12 @@ dependencies = [ "regex", "reqwest 0.12.15", "scraper", - "serde", "serde_json", "thirtyfour", "tokio", "totp-rs", ] -[[package]] -name = "md-5" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "md-5" version = "0.10.6" @@ -2263,7 +2153,6 @@ dependencies = [ "combine", "derivative", "serde", - "serde_json", "typeshare", ] @@ -2287,7 +2176,7 @@ dependencies = [ "hex", "hmac 0.12.1", "lazy_static", - "md-5 0.10.6", + "md-5", "pbkdf2", "percent-encoding", "rand 0.8.5", @@ -2327,7 +2216,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", + "spin", "version_check", ] @@ -2629,15 +2518,6 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "potential_utf" version = "0.1.2" @@ -2732,7 +2612,7 @@ dependencies = [ "getrandom 0.3.3", "lru-slab", "rand 0.9.1", - "ring 0.17.14", + "ring", "rustc-hash", "rustls 0.23.27", "rustls-pki-types", @@ -2912,17 +2792,6 @@ dependencies = [ "bitflags 2.9.1", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "regex" version = "1.11.1" @@ -3072,21 +2941,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.14" @@ -3097,92 +2951,10 @@ dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.16", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] -[[package]] -name = "rusoto_core" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" -dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes", - "crc32fast", - "futures", - "http 0.2.12", - "hyper 0.14.32", - "hyper-rustls 0.23.2", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version 0.4.1", - "serde", - "serde_json", - "tokio", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper 0.14.32", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - -[[package]] -name = "rusoto_s3" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d" -dependencies = [ - "async-trait", - "bytes", - "futures", - "rusoto_core", - "xml-rs", -] - -[[package]] -name = "rusoto_signature" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" -dependencies = [ - "base64 0.13.1", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http 0.2.12", - "hyper 0.14.32", - "log", - "md-5 0.9.1", - "percent-encoding", - "pin-project-lite", - "rusoto_credential", - "rustc_version 0.4.1", - "serde", - "sha2 0.9.9", - "tokio", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3236,18 +3008,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "rustls" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" -dependencies = [ - "log", - "ring 0.16.20", - "sct", - "webpki", -] - [[package]] name = "rustls" version = "0.21.12" @@ -3255,7 +3015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.14", + "ring", "rustls-webpki 0.101.7", "sct", ] @@ -3267,7 +3027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki 0.102.8", "subtle", @@ -3281,25 +3041,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki 0.103.3", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.7.3" @@ -3359,8 +3107,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3369,9 +3117,9 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -3380,9 +3128,9 @@ version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -3397,15 +3145,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.27" @@ -3442,8 +3181,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -3640,13 +3379,11 @@ dependencies = [ "axum-extra", "axum-server", "base64 0.21.7", - "bytes", "chrono", "clap", "db", "dotenv", - "env_logger 0.10.2", - "futures", + "env_logger", "http 1.3.1", "hyper 1.6.0", "log", @@ -3654,13 +3391,9 @@ dependencies = [ "model", "oauth2", "pretty_assertions", - "rand 0.9.1", "reqwest 0.11.27", - "rusoto_core", - "rusoto_s3", "serde", "serde_json", - "sha2 0.10.9", "tempfile", "tokio", "tower 0.4.13", @@ -3669,7 +3402,6 @@ dependencies = [ "tracing", "typeshare", "url", - "walkdir", ] [[package]] @@ -3822,12 +3554,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -4209,17 +3935,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.9", - "tokio", - "webpki", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -4548,12 +4263,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -4608,16 +4317,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -4746,16 +4445,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" -dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", -] - [[package]] name = "webpki-roots" version = "0.25.4" @@ -5142,12 +4831,6 @@ dependencies = [ "tap", ] -[[package]] -name = "xml-rs" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda" - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 2ed50c65..860256a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,11 +5,20 @@ edition = "2021" publish = false [workspace] -members = ["crates/*", "tools/changelog-generator", "tools/scraper"] +members = [ + "automation", + "crates/*", + "tools/changelog-generator", + "tools/scraper" +] [workspace.dependencies] anyhow = "1.0.98" chrono = "0.4.41" +clap = { version = "4.5.38", features = ["derive"] } +dotenv = "0.15.0" +env_logger = "0.10.2" +log = "0.4.27" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" tokio = { version = "1.45.0", features = ["rt-multi-thread", "macros"] } @@ -23,24 +32,18 @@ axum = { version = "0.7.9", features = ["json"] } axum-extra = { version = "0.9.6", features = ["cookie", "typed-header"] } axum-server = "0.6.0" base64 = "0.21.7" -bytes = "1.10.1" chrono = { workspace = true } -clap = { version = "4.5.38", features = ["derive"] } +clap = { workspace = true } db = { path = "crates/db" } -dotenv = "0.15.0" -env_logger = "0.10.2" -futures = "0.3.31" +dotenv = { workspace = true } +env_logger = { workspace = true } http = "1.3.1" -log = "0.4.27" +log = { workspace = true } model = { path = "crates/model" } oauth2 = "4.4.2" -rand = "0.9.1" reqwest = { version = "0.11.27", default-features = false, features = [ "blocking", "json", "rustls-tls"] } -rusoto_core = { version = "0.48.0", default-features = false, features = [ "rustls", ] } -rusoto_s3 = { version = "0.48.0", default-features = false, features = [ "rustls", ] } serde = { workspace = true } serde_json = { workspace = true } -sha2 = "0.10.9" tokio = { workspace = true } tower = { version = "0.4.13", features = ["tracing", "limit", "buffer"] } tower-http = { version = "0.5.2", features = ["cors", "fs", "trace"] } @@ -48,7 +51,6 @@ tower_governor = "0.2.0" tracing = "0.1.41" typeshare = { workspace = true } url = "2.5.4" -walkdir = "2.5.0" [dev-dependencies] hyper = { version = "1.6.0", features = ["server"] } diff --git a/Dockerfile b/Dockerfile index 4c58e69a..81e0a18f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,4 +27,4 @@ COPY --from=server /usr/src/app/seed seed COPY --from=server /usr/src/app/.well-known .well-known COPY --from=server /usr/src/app/target/release/server /usr/local/bin -CMD server --source seed --initialize --latest-courses --skip-reviews --asset-dir assets --db-name mcgill-courses +CMD server --db-name mcgill-courses serve --asset-dir assets diff --git a/automation/Cargo.toml b/automation/Cargo.toml new file mode 100644 index 00000000..417c9511 --- /dev/null +++ b/automation/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "automation" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] diff --git a/automation/checkout b/automation/checkout new file mode 100755 index 00000000..d757664e --- /dev/null +++ b/automation/checkout @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +BRANCH=$1 +DOMAIN=$2 + +if [[ ! -d bot ]]; then + git clone https://github.com/moyai-orz/comp520-bot bot +fi + +cd bot + +git fetch origin +git checkout -B "$BRANCH" +git reset --hard origin/"$BRANCH" + +./automation/setup "$DOMAIN" diff --git a/automation/src/main.rs b/automation/src/main.rs new file mode 100644 index 00000000..80a1832b --- /dev/null +++ b/automation/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/crates/db/src/initializer.rs b/crates/db/src/initializer.rs index e062f444..00e41e59 100644 --- a/crates/db/src/initializer.rs +++ b/crates/db/src/initializer.rs @@ -20,7 +20,7 @@ impl Initializer { Ok(()) } - fn collect(&self) -> Result> { + fn collect_seeds(&self) -> Result> { Ok(if self.options.source.is_file() { vec![Seed::from_content( self.options.source.clone(), @@ -116,25 +116,11 @@ impl Initializer { async fn seed(&self) -> Result { info!("Seeding the database..."); - let mut seeds = self.collect()?; - - if self.options.latest_courses { - let courses = seeds - .clone() - .into_iter() - .filter(|seed| matches!(seed, Seed::Courses(_))) - .collect::>(); - - seeds.retain(|seed| !matches!(seed, Seed::Courses(_))); - - if let Some(last) = courses.last() { - seeds.insert(0, last.clone()); - } - } + let seeds = self.collect_seeds()?; for seed in seeds { match seed { - Seed::Courses((path, courses)) if !self.options.skip_courses => { + Seed::Courses((path, courses)) => { info!("Seeding courses from {}...", path.display()); let runner = |db: Db, item: Course| async move { @@ -149,7 +135,7 @@ impl Initializer { self.populate(courses, runner).await?; } - Seed::Reviews((path, reviews)) if !self.options.skip_reviews => { + Seed::Reviews((path, reviews)) => { info!("Seeding reviews from {}...", path.display()); let runner = |db: Db, item: Review| async move { @@ -165,7 +151,6 @@ impl Initializer { path.display() ); } - _ => continue, } } diff --git a/crates/model/Cargo.toml b/crates/model/Cargo.toml index cb087143..af13e264 100644 --- a/crates/model/Cargo.toml +++ b/crates/model/Cargo.toml @@ -9,5 +9,4 @@ bson = { version = "2.14.0", features = ["chrono-0_4"] } combine = { path = "../combine" } derivative = "2.2.0" serde = { workspace = true } -serde_json = { workspace = true } typeshare = { workspace = true } diff --git a/crates/model/src/initialize_options.rs b/crates/model/src/initialize_options.rs index 276e16b2..b8e81291 100644 --- a/crates/model/src/initialize_options.rs +++ b/crates/model/src/initialize_options.rs @@ -2,9 +2,6 @@ use super::*; #[derive(Debug, Default)] pub struct InitializeOptions { - pub latest_courses: bool, pub multithreaded: bool, - pub skip_courses: bool, - pub skip_reviews: bool, pub source: PathBuf, } diff --git a/justfile b/justfile index abfcc855..3a386b40 100644 --- a/justfile +++ b/justfile @@ -37,7 +37,7 @@ dev: services typeshare --timestamp-format 'HH:mm:ss' \ --color \ -- \ - 'just watch run -- --db-name=mcgill-courses' \ + 'just watch run -- --db-name mcgill-courses serve' \ 'pnpm run dev' dev-deps: @@ -67,14 +67,14 @@ generate-changelog *args: {{args}} initialize *args: restart-services - cargo run -- --source=seed --initialize --db-name=mcgill-courses {{args}} + cargo run -- --db-name mcgill-courses seed --source seed {{args}} lint *args: pnpm run lint {{args}} load: - cargo run --manifest-path tools/scraper/Cargo.toml -- --source=seed \ - --batch-size=5 \ + cargo run --manifest-path tools/scraper/Cargo.toml -- --source seed \ + --batch-size 5 \ --scrape-vsb \ --user-agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36" \ --course-delay 1000 @@ -100,7 +100,7 @@ run-container: build-container mcgill.courses:latest serve: - cargo run -- --db-name=mcgill-courses + cargo run -- --db-name mcgill-courses serve services: docker compose up --no-recreate -d diff --git a/src/arguments.rs b/src/arguments.rs new file mode 100644 index 00000000..e1f8d73a --- /dev/null +++ b/src/arguments.rs @@ -0,0 +1,15 @@ +use super::*; + +#[derive(Debug, Parser)] +pub(crate) struct Arguments { + #[clap(flatten)] + options: Options, + #[clap(subcommand)] + subcommand: Subcommand, +} + +impl Arguments { + pub(crate) async fn run(self) -> Result { + self.subcommand.run(self.options).await + } +} diff --git a/src/hash.rs b/src/hash.rs deleted file mode 100644 index 9668a265..00000000 --- a/src/hash.rs +++ /dev/null @@ -1,119 +0,0 @@ -use super::*; - -pub(crate) trait Hash { - fn hash(&self) -> Result>; -} - -impl Hash for PathBuf { - fn hash(&self) -> Result> { - let mut hasher = Sha256::new(); - - let read = |path: &PathBuf| -> Result> { - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer)?; - Ok(buffer) - }; - - match (self.is_dir(), self.is_file()) { - (true, false) => { - for entry in WalkDir::new(self) - .into_iter() - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().is_file()) - { - hasher.update(read(&entry.path().to_path_buf())?); - } - } - (false, true) => { - hasher.update(read(self)?); - } - _ => { - return Err(Error(anyhow!( - "{} is neither a file nor a directory", - self.display() - ))); - } - } - - Ok(hasher.finalize().to_vec()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::tempdir; - - #[test] - fn hash_file() { - let directory = tempdir().unwrap(); - - let path = directory.path().join("test.txt"); - - let mut file = File::create(&path).unwrap(); - writeln!(file, "Hello, world!").unwrap(); - - let hash_a = path.hash().unwrap(); - assert!(!hash_a.is_empty()); - - let hash_b = path.hash().unwrap(); - assert!(!hash_b.is_empty()); - - assert_eq!(hash_a, hash_b); - - writeln!(file, "yeet").unwrap(); - - let hash_c = path.hash().unwrap(); - - assert!(!hash_c.is_empty()); - assert_ne!(hash_c, hash_a); - - directory.close().unwrap(); - } - - #[test] - fn hash_directory() { - let directory = tempdir().unwrap(); - - let path = directory.path().join("test.txt"); - - let mut file = File::create(path).unwrap(); - writeln!(file, "Hello, world!").unwrap(); - - let hash_a = directory.path().to_path_buf().hash().unwrap(); - assert!(!hash_a.is_empty()); - - let hash_b = directory.path().to_path_buf().hash().unwrap(); - assert!(!hash_b.is_empty()); - - assert_eq!(hash_a, hash_b); - - writeln!(file, "yeet").unwrap(); - - let hash_c = directory.path().to_path_buf().hash().unwrap(); - - assert!(!hash_c.is_empty()); - assert_ne!(hash_c, hash_a); - - directory.close().unwrap(); - } - - #[test] - fn hash_neither_file_nor_directory() { - let directory = tempdir().unwrap(); - - let result = directory.path().join("invalid").hash(); - - assert_eq!( - result.unwrap_err().to_string(), - format!( - "{} is neither a file nor a directory", - directory.path().join("invalid").display() - ) - ); - - directory.close().unwrap(); - } -} diff --git a/src/main.rs b/src/main.rs index b2b9af84..b8b6f33c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,14 @@ use { crate::{ + arguments::Arguments, assets::Assets, auth::{AuthRedirect, COOKIE_NAME}, error::Error, - hash::Hash, - object::Object, + options::Options, + seeder::Seeder, server::Server, state::State, + subcommand::Subcommand, user::User, }, anyhow::anyhow, @@ -29,7 +31,6 @@ use { db::Db, dotenv::dotenv, env_logger::Env, - futures::TryStreamExt, http::{ header, header::SET_COOKIE, request::Parts, HeaderMap, Request, StatusCode, }, @@ -42,19 +43,13 @@ use { basic::BasicClient, AuthType, AuthUrl, ClientId, ClientSecret, CsrfToken, RedirectUrl, Scope, TokenUrl, }, - rusoto_core::Region, - rusoto_s3::S3Client, - rusoto_s3::{GetObjectRequest, PutObjectOutput, PutObjectRequest, S3}, serde::{Deserialize, Serialize}, serde_json::json, - sha2::{Digest, Sha256}, std::{ backtrace::BacktraceStatus, env, fmt::{self, Display, Formatter}, fs, - fs::File, - io::Read, net::SocketAddr, path::PathBuf, process, @@ -73,27 +68,27 @@ use { tracing::Span, typeshare::typeshare, url::Url, - walkdir::WalkDir, }; +mod arguments; mod assets; mod auth; mod courses; mod error; -mod hash; mod instructors; mod interactions; mod notifications; -mod object; mod options; mod reviews; mod search; +mod seeder; mod server; mod state; +mod subcommand; mod subscriptions; mod user; -type Result = std::result::Result; +type Result = std::result::Result; #[tokio::main] async fn main() { @@ -102,7 +97,7 @@ async fn main() { dotenv().ok(); - if let Err(error) = Server::parse().run().await { + if let Err(error) = Arguments::parse().run().await { eprintln!("error: {error}"); for (i, error) in error.0.chain().skip(1).enumerate() { diff --git a/src/object.rs b/src/object.rs deleted file mode 100644 index 952e8cad..00000000 --- a/src/object.rs +++ /dev/null @@ -1,51 +0,0 @@ -use super::*; - -#[async_trait] -pub(crate) trait Object { - async fn get(&self, bucket: &str, key: &str) -> Result>>; - async fn put( - &self, - bucket: &str, - key: &str, - value: Vec, - ) -> Result; -} - -#[async_trait] -impl Object for S3Client { - async fn get(&self, bucket: &str, key: &str) -> Result>> { - let response = self - .get_object(GetObjectRequest { - bucket: bucket.into(), - key: key.into(), - ..Default::default() - }) - .await; - - Ok(match response { - Ok(response) => match response.body { - Some(stream) => Some(stream.map_ok(|b| b.to_vec()).try_concat().await?), - None => None, - }, - Err(_) => None, - }) - } - - async fn put( - &self, - bucket: &str, - key: &str, - value: Vec, - ) -> Result { - Ok( - self - .put_object(PutObjectRequest { - bucket: bucket.into(), - key: key.into(), - body: Some(value.into()), - ..Default::default() - }) - .await?, - ) - } -} diff --git a/src/options.rs b/src/options.rs index 7a11f5b9..50856bc1 100644 --- a/src/options.rs +++ b/src/options.rs @@ -1,7 +1,7 @@ use super::*; -#[derive(Parser)] +#[derive(Debug, Parser)] pub(crate) struct Options { - #[clap(long, default_value = "courses.json")] - pub(crate) source: PathBuf, + #[clap(long, default_value = "admin", help = "Database name")] + pub(crate) db_name: String, } diff --git a/src/seeder.rs b/src/seeder.rs new file mode 100644 index 00000000..0701d3b6 --- /dev/null +++ b/src/seeder.rs @@ -0,0 +1,28 @@ +use super::*; + +#[derive(Debug, Parser)] +pub(crate) struct Seeder { + #[clap(long, default_value = "false", help = "Enable multithreaded seeding")] + multithreaded: bool, + #[clap(long, default_value = "courses.json", help = "Path to seed from")] + source: PathBuf, +} + +impl Into for Seeder { + fn into(self) -> InitializeOptions { + InitializeOptions { + multithreaded: self.multithreaded, + source: self.source, + } + } +} + +impl Seeder { + pub(crate) async fn run(self, options: Options) -> Result { + let db = Arc::new(Db::connect(&options.db_name).await?); + + db.initialize(self.into()).await?; + + Ok(()) + } +} diff --git a/src/server.rs b/src/server.rs index 9a47e794..cb4b96ea 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,25 +1,11 @@ use super::*; -#[derive(Parser)] +#[derive(Debug, Parser)] pub(crate) struct Server { - #[clap(long, default_value = "courses.json")] - source: PathBuf, #[clap(long, help = "Directory to serve assets from")] asset_dir: Option, #[clap(long, default_value = "8000", help = "Port to listen on")] port: u16, - #[clap(long, default_value = "admin", help = "Database name")] - db_name: String, - #[clap(long, default_value = "false", help = "Seed latest courses only")] - latest_courses: bool, - #[clap(long, default_value = "false", help = "Enable multithreaded seeding")] - multithreaded: bool, - #[clap(long, default_value = "false", help = "Initialize the database")] - initialize: bool, - #[clap(long, default_value = "false", help = "Skip course seeding")] - skip_courses: bool, - #[clap(long, default_value = "false", help = "Skip review seeding")] - skip_reviews: bool, } #[derive(Debug)] @@ -31,51 +17,12 @@ struct AppConfig<'a> { } impl Server { - pub(crate) async fn run(self) -> Result { + pub(crate) async fn run(self, options: Options) -> Result { let addr = SocketAddr::from(([0, 0, 0, 0], self.port)); info!("Listening on port: {}", addr.port()); - let db = Arc::new(Db::connect(&self.db_name).await?); - - if self.initialize { - let source_hash = self.source.hash()?; - - let client = match env::var("ENV") { - Ok(env) if env == "production" => Some(S3Client::new(Region::UsEast1)), - _ => None, - }; - - let prev_hash = match client { - Some(ref client) => client.get("mcgill.courses", "source-hash").await?, - None => None, - }; - - if Some(&source_hash) != prev_hash.as_ref() { - let clone = db.clone(); - - if let Some(client) = client { - client - .put("mcgill.courses", "source-hash", source_hash) - .await?; - } - - tokio::spawn(async move { - if let Err(error) = clone - .initialize(InitializeOptions { - latest_courses: self.latest_courses, - multithreaded: self.multithreaded, - skip_courses: self.skip_courses, - skip_reviews: self.skip_reviews, - source: self.source, - }) - .await - { - error!("error: {error}"); - } - }); - } - } + let db = Arc::new(Db::connect(&options.db_name).await?); let assets = self.asset_dir.as_ref().map(|asset_dir| Assets { dir: ServeDir::new(asset_dir.clone()), diff --git a/src/subcommand.rs b/src/subcommand.rs new file mode 100644 index 00000000..03ade5d1 --- /dev/null +++ b/src/subcommand.rs @@ -0,0 +1,18 @@ +use super::*; + +#[derive(Debug, Parser)] +pub(crate) enum Subcommand { + #[clap(about = "Seed the database")] + Seed(Seeder), + #[clap(about = "Run the server")] + Serve(Server), +} + +impl Subcommand { + pub(crate) async fn run(self, options: Options) -> Result { + match self { + Self::Seed(seeder) => seeder.run(options).await, + Self::Serve(server) => server.run(options).await, + } + } +} diff --git a/tools/changelog-generator/Cargo.toml b/tools/changelog-generator/Cargo.toml index b4afc0a0..83c25e29 100644 --- a/tools/changelog-generator/Cargo.toml +++ b/tools/changelog-generator/Cargo.toml @@ -1,17 +1,18 @@ [package] name = "changelog-generator" -version = "0.1.0" +version = "0.0.0" edition = "2021" +publish = false [dependencies] -anyhow = "1.0.98" +anyhow = { workspace = true } async-openai = "0.21.0" -chrono = "0.4.41" -clap = { version = "4.5.38", features = ["derive"] } -dotenv = "0.15.0" -env_logger = "0.11.8" -log = "0.4.27" +chrono = { workspace = true } +clap = { workspace = true } +dotenv = { workspace = true } +env_logger = { workspace = true } +log = { workspace = true } octocrab = "0.38.0" -serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" -tokio = { version = "1.45.0", features = ["rt-multi-thread", "macros"] } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } diff --git a/tools/scraper/Cargo.toml b/tools/scraper/Cargo.toml index d119254b..d4046c70 100644 --- a/tools/scraper/Cargo.toml +++ b/tools/scraper/Cargo.toml @@ -1,24 +1,24 @@ [package] name = "mcgill-courses-scraper" -version = "0.1.0" +version = "0.0.0" edition = "2024" +publish = false [dependencies] -anyhow = "1.0.98" -chrono = "0.4.41" -clap = { version = "4.5.38", features = ["derive"] } -env_logger = "0.11.8" -log = "0.4.27" +anyhow = { workspace = true } +chrono = { workspace = true } +clap = { workspace = true } +env_logger = { workspace = true } +log = { workspace = true } model = { path = "../../crates/model" } rand = "0.9.1" rayon = "1.10.0" regex = "1.11.1" reqwest = { version = "0.12.15", default-features = false, features = [ "blocking", "json", "rustls-tls" ] } scraper = "0.23.1" -serde = "1.0.219" -serde_json = "1.0.140" +serde_json = { workspace = true } thirtyfour = "0.35.0" -tokio = "1.45.1" +tokio = { workspace = true } totp-rs = "5.7.0" [dev-dependencies]