From 27da9fc1eceae6ef2f197d52862d90d66953a1c2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2020 04:47:07 +0000 Subject: [PATCH 01/69] Bump rocket from 0.4.2 to 0.4.3 Bumps [rocket](https://github.com/SergioBenitez/Rocket) from 0.4.2 to 0.4.3. - [Release notes](https://github.com/SergioBenitez/Rocket/releases) - [Changelog](https://github.com/SergioBenitez/Rocket/blob/master/CHANGELOG.md) - [Commits](https://github.com/SergioBenitez/Rocket/compare/v0.4.2...v0.4.3) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 40 ++++++++++++++++------------------------ Cargo.toml | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2834d43..f05a409 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,6 +448,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.2.1" @@ -710,12 +716,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.2" @@ -959,7 +959,7 @@ dependencies = [ "cloudabi", "libc", "redox_syscall", - "smallvec 1.2.0", + "smallvec", "winapi 0.3.8", ] @@ -1234,9 +1234,9 @@ dependencies = [ [[package]] name = "rocket" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42c1e9deb3ef4fa430d307bfccd4231434b707ca1328fae339c43ad1201cc6f7" +checksum = "0aa9ab5fad4a9b366ae884f5b5461e166fe8f94d23a561a91dfaab046c2f4fde" dependencies = [ "atty", "base64 0.10.1", @@ -1255,11 +1255,12 @@ dependencies = [ [[package]] name = "rocket_codegen" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79aa1366f9b2eccddc05971e17c5de7bb75a5431eb12c2b5c66545fd348647f4" +checksum = "c56c540f73ae2291e8c1e4f9757f92fec7015c09fd90d7bb70349386a111602c" dependencies = [ "devise", + "glob", "indexmap", "quote 0.6.13", "rocket_http", @@ -1282,16 +1283,16 @@ dependencies = [ [[package]] name = "rocket_http" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1391457ee4e80b40d4b57fa5765c0f2836b20d73bcbee4e3f35d93cf3b80817" +checksum = "968a07cbfa9f8c8a0239aba359b80e1281e85da9415795c68f3ddeb5e1d636d6" dependencies = [ "cookie", "hyper 0.10.16", "indexmap", "pear", "percent-encoding 1.0.1", - "smallvec 0.6.13", + "smallvec", "state", "time", "unicode-xid 0.1.0", @@ -1444,15 +1445,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -[[package]] -name = "smallvec" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.2.0" @@ -1651,7 +1643,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.2.0", + "smallvec", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 80e28c4..74d0637 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ dotenv = "0.15" openssl = "0.10" r2d2 = "0.8" reqwest = { version = "0.10", features = ["blocking"] } -rocket = "0.4.2" +rocket = "0.4.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" zip = "0.5" From dfe9b90d2e09f3de0a811e26a615147ccc8c6bf8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2020 04:47:08 +0000 Subject: [PATCH 02/69] Bump reqwest from 0.10.2 to 0.10.4 Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.10.2 to 0.10.4. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.10.2...v0.10.4) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2834d43..b7c32b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,9 +450,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1" +checksum = "9d5c295d1c0c68e4e42003d75f908f5e16a1edd1cbe0b0d02e4dc2006a384f47" dependencies = [ "bytes", "fnv", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1c527bbc634be72aa7ba31e4e4def9bbb020f5416916279b7c705cd838893e" +checksum = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5" dependencies = [ "bytes", "futures-channel", @@ -562,7 +562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" dependencies = [ "bytes", - "hyper 0.13.2", + "hyper 0.13.3", "native-tls", "tokio", "tokio-tls", @@ -1187,9 +1187,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.2" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae3fc32eacd4a5200c6b34bd6c057b07fb64f5a1e55bb67d624cc1393354621" +checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" dependencies = [ "base64 0.11.0", "bytes", @@ -1198,7 +1198,7 @@ dependencies = [ "futures-util", "http", "http-body", - "hyper 0.13.2", + "hyper 0.13.3", "hyper-tls", "js-sys", "lazy_static", From 5fe28fe0a4f0bf3843687660831df2e830f64a16 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2020 04:44:45 +0000 Subject: [PATCH 03/69] Bump chrono from 0.4.10 to 0.4.11 Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.10 to 0.4.11. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/master/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.10...v0.4.11) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2834d43..b7267fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31850b4a4d6bae316f7a09e691c944c28299298837edc0a03f755618c23cbc01" +checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" dependencies = [ "num-integer", "num-traits", From 8a7a68a6f7c843834e9ad93dcd3a0246560cc98e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2020 04:45:11 +0000 Subject: [PATCH 04/69] Bump clap from `e4a7f50` to `2370665` Bumps [clap](https://github.com/clap-rs/clap) from `e4a7f50` to `2370665`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/e4a7f5012855e9dd906caaa7028393cf1926da8e...23706657fca033845bc5b8c0b74fb06fb96e780c) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2834d43..5dba2da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e4a7f5012855e9dd906caaa7028393cf1926da8e" +source = "git+https://github.com/clap-rs/clap.git#23706657fca033845bc5b8c0b74fb06fb96e780c" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e4a7f5012855e9dd906caaa7028393cf1926da8e" +source = "git+https://github.com/clap-rs/clap.git#23706657fca033845bc5b8c0b74fb06fb96e780c" dependencies = [ "heck", "proc-macro-error", From 03a4c6486e85ceeae551ab198f611f79dea681e9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2020 17:40:42 +0000 Subject: [PATCH 05/69] Bump zip from 0.5.4 to 0.5.5 Bumps [zip](https://github.com/mvdnes/zip-rs) from 0.5.4 to 0.5.5. - [Release notes](https://github.com/mvdnes/zip-rs/releases) - [Commits](https://github.com/mvdnes/zip-rs/commits) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7acc144..92a1098 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1931,9 +1931,9 @@ checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" [[package]] name = "zip" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e41ff37ba788e2169b19fa70253b70cb53d9f2db9fb9aea9bcfc5047e02c3bae" +checksum = "6df134e83b8f0f8153a094c7b0fd79dfebe437f1d76e7715afa18ed95ebe2fd7" dependencies = [ "bzip2", "crc32fast", From 8ae8d4acc9280a2cf23f59dec442bbae4ee14db4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2020 04:44:31 +0000 Subject: [PATCH 06/69] Bump rocket from 0.4.3 to 0.4.4 Bumps [rocket](https://github.com/SergioBenitez/Rocket) from 0.4.3 to 0.4.4. - [Release notes](https://github.com/SergioBenitez/Rocket/releases) - [Changelog](https://github.com/SergioBenitez/Rocket/blob/v0.4.4/CHANGELOG.md) - [Commits](https://github.com/SergioBenitez/Rocket/compare/v0.4.3...v0.4.4) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f59e56..27afdf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1234,9 +1234,9 @@ dependencies = [ [[package]] name = "rocket" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa9ab5fad4a9b366ae884f5b5461e166fe8f94d23a561a91dfaab046c2f4fde" +checksum = "e20afbad214b001cabbe31dd270b48b3be980a7153ee2ed8392e241f856d651b" dependencies = [ "atty", "base64 0.10.1", @@ -1255,9 +1255,9 @@ dependencies = [ [[package]] name = "rocket_codegen" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56c540f73ae2291e8c1e4f9757f92fec7015c09fd90d7bb70349386a111602c" +checksum = "2108b35e2c3a35759d3f16cc3002ece05523191d884d3ad6523693fd43324dde" dependencies = [ "devise", "glob", @@ -1283,9 +1283,9 @@ dependencies = [ [[package]] name = "rocket_http" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968a07cbfa9f8c8a0239aba359b80e1281e85da9415795c68f3ddeb5e1d636d6" +checksum = "6ce8ca76247376ea21cf271af0f95e3f2014596e3e4c7cc04e44ee6242a40ff2" dependencies = [ "cookie", "hyper 0.10.16", diff --git a/Cargo.toml b/Cargo.toml index 74d0637..c74e930 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ dotenv = "0.15" openssl = "0.10" r2d2 = "0.8" reqwest = { version = "0.10", features = ["blocking"] } -rocket = "0.4.3" +rocket = "0.4.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" zip = "0.5" From 89071a902195575cd6a55e3304909cb456943521 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2020 04:44:54 +0000 Subject: [PATCH 07/69] Bump clap from `2370665` to `bc738e1` Bumps [clap](https://github.com/clap-rs/clap) from `2370665` to `bc738e1`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/23706657fca033845bc5b8c0b74fb06fb96e780c...bc738e12c223666ac84fe29c74c21b4e276c5e59) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f59e56..e62fbf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#23706657fca033845bc5b8c0b74fb06fb96e780c" +source = "git+https://github.com/clap-rs/clap.git#bc738e12c223666ac84fe29c74c21b4e276c5e59" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#23706657fca033845bc5b8c0b74fb06fb96e780c" +source = "git+https://github.com/clap-rs/clap.git#bc738e12c223666ac84fe29c74c21b4e276c5e59" dependencies = [ "heck", "proc-macro-error", From 8c82a312f6ab59a12db862ee9dc8cd81b3402ec2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2020 04:45:15 +0000 Subject: [PATCH 08/69] Bump rocket_contrib from 0.4.2 to 0.4.3 Bumps [rocket_contrib](https://github.com/SergioBenitez/Rocket) from 0.4.2 to 0.4.3. - [Release notes](https://github.com/SergioBenitez/Rocket/releases) - [Changelog](https://github.com/SergioBenitez/Rocket/blob/master/CHANGELOG.md) - [Commits](https://github.com/SergioBenitez/Rocket/compare/v0.4.2...v0.4.3) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f59e56..23cfd2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,9 +1270,9 @@ dependencies = [ [[package]] name = "rocket_contrib" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0fa5c1392135adc0f96a02ba150ac4c765e27c58dbfd32aa40678e948f6e56f" +checksum = "66926894ef94396204c53885f54a1b0e94a88ce6c9478c2fd403261fc0efb523" dependencies = [ "log 0.4.8", "notify", diff --git a/Cargo.toml b/Cargo.toml index 74d0637..f362b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,6 @@ serde_json = "1.0" zip = "0.5" [dependencies.rocket_contrib] -version = "0.4.2" +version = "0.4.3" default-features = false features = ["json"] From a11e9504acc819d86902a52509bd1b382638aedf Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 10 Mar 2020 11:07:59 +0100 Subject: [PATCH 09/69] Upgrade rocket_contrib --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e00e07..11c4151 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,9 +1270,9 @@ dependencies = [ [[package]] name = "rocket_contrib" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66926894ef94396204c53885f54a1b0e94a88ce6c9478c2fd403261fc0efb523" +checksum = "a10e7471279bc2d4a21b6fddd9589016bb119e6fbb547b216dd54ef237f28341" dependencies = [ "log 0.4.8", "notify", diff --git a/Cargo.toml b/Cargo.toml index 1272be3..27d4767 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,6 @@ serde_json = "1.0" zip = "0.5" [dependencies.rocket_contrib] -version = "0.4.3" +version = "0.4.4" default-features = false features = ["json"] From e77229961246660fff5698f86de70441a6d8ce8a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2020 04:43:50 +0000 Subject: [PATCH 10/69] Bump clap from `bc738e1` to `9c8f9e2` Bumps [clap](https://github.com/clap-rs/clap) from `bc738e1` to `9c8f9e2`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/bc738e12c223666ac84fe29c74c21b4e276c5e59...9c8f9e20b989ff290a8942d5cfc2c6b8140f466f) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11c4151..1ee81ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#bc738e12c223666ac84fe29c74c21b4e276c5e59" +source = "git+https://github.com/clap-rs/clap.git#9c8f9e20b989ff290a8942d5cfc2c6b8140f466f" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#bc738e12c223666ac84fe29c74c21b4e276c5e59" +source = "git+https://github.com/clap-rs/clap.git#9c8f9e20b989ff290a8942d5cfc2c6b8140f466f" dependencies = [ "heck", "proc-macro-error", From c99986c2c006e005e11caf29111aae8cdfb6e430 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2020 04:45:32 +0000 Subject: [PATCH 11/69] Bump serde from 1.0.104 to 1.0.105 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.104 to 1.0.105. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.104...v1.0.105) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ee81ef..ca8366c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1378,18 +1378,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" dependencies = [ "proc-macro2 1.0.8", "quote 1.0.2", From ea25b7d2732bbbc6fc6196fa727374c88d6c9f74 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2020 04:49:02 +0000 Subject: [PATCH 12/69] Bump diesel from 1.4.3 to 1.4.4 Bumps [diesel](https://github.com/diesel-rs/diesel) from 1.4.3 to 1.4.4. - [Release notes](https://github.com/diesel-rs/diesel/releases) - [Changelog](https://github.com/diesel-rs/diesel/blob/v1.4.4/CHANGELOG.md) - [Commits](https://github.com/diesel-rs/diesel/compare/v1.4.3...v1.4.4) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ee81ef..e28c816 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "diesel" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7cc03b910de9935007861dce440881f69102aaaedfd4bc5a6f40340ca5840c" +checksum = "33d7ca63eb2efea87a7f56a283acc49e2ce4b2bd54adf7465dc1d81fef13d8fc" dependencies = [ "bitflags", "byteorder", From 6af2d0d1723a6db855e4047574ff9ef67e98aa21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2020 04:45:19 +0000 Subject: [PATCH 13/69] Bump serde_json from 1.0.48 to 1.0.50 Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.48 to 1.0.50. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.48...v1.0.50) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ee81ef..81bac8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,9 +1398,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +checksum = "78a7a12c167809363ec3bd7329fc0a3369056996de43c4b37ef3cd54a6ce4867" dependencies = [ "itoa", "ryu", From 132e5e973fa0c06c3ef86d425a7d31e85ec8a654 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2020 04:45:44 +0000 Subject: [PATCH 14/69] Bump clap from `9c8f9e2` to `37889c6` Bumps [clap](https://github.com/clap-rs/clap) from `9c8f9e2` to `37889c6`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/9c8f9e20b989ff290a8942d5cfc2c6b8140f466f...37889c661134e8286102f7d2ab3267965d010403) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ee81ef..269a80e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#9c8f9e20b989ff290a8942d5cfc2c6b8140f466f" +source = "git+https://github.com/clap-rs/clap.git#37889c661134e8286102f7d2ab3267965d010403" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#9c8f9e20b989ff290a8942d5cfc2c6b8140f466f" +source = "git+https://github.com/clap-rs/clap.git#37889c661134e8286102f7d2ab3267965d010403" dependencies = [ "heck", "proc-macro-error", From 0716225058fcd74d1453a499ed138986804584d2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 2 Apr 2020 04:48:25 +0000 Subject: [PATCH 15/69] Bump clap from `37889c6` to `b63c4ea` Bumps [clap](https://github.com/clap-rs/clap) from `37889c6` to `b63c4ea`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/37889c661134e8286102f7d2ab3267965d010403...b63c4ea03f8bd5c23cc05e957f22f47daafcce61) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2666ec..e94057b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#37889c661134e8286102f7d2ab3267965d010403" +source = "git+https://github.com/clap-rs/clap.git#b63c4ea03f8bd5c23cc05e957f22f47daafcce61" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#37889c661134e8286102f7d2ab3267965d010403" +source = "git+https://github.com/clap-rs/clap.git#b63c4ea03f8bd5c23cc05e957f22f47daafcce61" dependencies = [ "heck", "proc-macro-error", From 6c2b3ab4d2051a2773b5ca0b3a7903ea1a292258 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 04:42:39 +0000 Subject: [PATCH 16/69] Bump clap from `b63c4ea` to `4b5d772` Bumps [clap](https://github.com/clap-rs/clap) from `b63c4ea` to `4b5d772`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/b63c4ea03f8bd5c23cc05e957f22f47daafcce61...4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e94057b..3e29b82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#b63c4ea03f8bd5c23cc05e957f22f47daafcce61" +source = "git+https://github.com/clap-rs/clap.git#4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#b63c4ea03f8bd5c23cc05e957f22f47daafcce61" +source = "git+https://github.com/clap-rs/clap.git#4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8" dependencies = [ "heck", "proc-macro-error", From fa1465878ccd4cf43c1ad68eedc834af25559f6c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 04:45:44 +0000 Subject: [PATCH 17/69] Bump serde from 1.0.105 to 1.0.106 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.105 to 1.0.106. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.105...v1.0.106) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e29b82..c071bc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1378,18 +1378,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" dependencies = [ "proc-macro2 1.0.8", "quote 1.0.2", From 6eec18a4eaa9f21bf49846e4fc9068e67958661b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 04:46:06 +0000 Subject: [PATCH 18/69] Bump serde_json from 1.0.50 to 1.0.51 Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.50 to 1.0.51. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.50...v1.0.51) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e29b82..497061a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1398,9 +1398,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78a7a12c167809363ec3bd7329fc0a3369056996de43c4b37ef3cd54a6ce4867" +checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" dependencies = [ "itoa", "ryu", From f0313068c0c6ca07546e6d142988bf9cc3707981 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 04:46:30 +0000 Subject: [PATCH 19/69] Bump clap from `4b5d772` to `20daf00` Bumps [clap](https://github.com/clap-rs/clap) from `4b5d772` to `20daf00`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8...20daf00d519d12ec92923a7bde42cc9211823e69) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e29b82..38858e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8" +source = "git+https://github.com/clap-rs/clap.git#20daf00d519d12ec92923a7bde42cc9211823e69" dependencies = [ "ansi_term", "atty", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#4b5d7722bfbd7c6c5f78d0fe1d0aaae46c98e1a8" +source = "git+https://github.com/clap-rs/clap.git#20daf00d519d12ec92923a7bde42cc9211823e69" dependencies = [ "heck", "proc-macro-error", From b0193f62f8259efe506102d552be784d295892bc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2020 04:44:02 +0000 Subject: [PATCH 20/69] Bump openssl from 0.10.28 to 0.10.29 Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.28 to 0.10.29. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.28...openssl-v0.10.29) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e119e8..200cff4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -908,9 +908,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.28" +version = "0.10.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973293749822d7dd6370d6da1e523b0d1db19f06c459134c658b2a4261378b52" +checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" dependencies = [ "bitflags", "cfg-if", @@ -928,9 +928,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.54" +version = "0.9.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1024c0a59774200a555087a6da3f253a9095a5f344e353b212ac4c8b8e450986" +checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8" dependencies = [ "autocfg", "cc", From d4faf09df6fd672c0af7d16fd5f96340d33e23f2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2020 04:43:20 +0000 Subject: [PATCH 21/69] Bump clap from `20daf00` to `e1d83a7` Bumps [clap](https://github.com/clap-rs/clap) from `20daf00` to `e1d83a7`. - [Release notes](https://github.com/clap-rs/clap/releases) - [Commits](https://github.com/clap-rs/clap/compare/20daf00d519d12ec92923a7bde42cc9211823e69...e1d83a78804a271b053d4d21f69b67f7eb01ad01) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 200cff4..d466582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,15 +6,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi 0.3.8", -] - [[package]] name = "anyhow" version = "1.0.26" @@ -143,15 +134,15 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#20daf00d519d12ec92923a7bde42cc9211823e69" +source = "git+https://github.com/clap-rs/clap.git#e1d83a78804a271b053d4d21f69b67f7eb01ad01" dependencies = [ - "ansi_term", "atty", "bitflags", "clap_derive", "indexmap", "lazy_static", "strsim", + "termcolor", "textwrap", "unicode-width", "vec_map", @@ -160,7 +151,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#20daf00d519d12ec92923a7bde42cc9211823e69" +source = "git+https://github.com/clap-rs/clap.git#e1d83a78804a271b053d4d21f69b67f7eb01ad01" dependencies = [ "heck", "proc-macro-error", @@ -1516,6 +1507,15 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + [[package]] name = "textwrap" version = "0.11.0" From 138c2df932918c18ef5b072e1a5cc781b839337c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 11 May 2020 04:41:24 +0000 Subject: [PATCH 22/69] Bump serde_json from 1.0.51 to 1.0.53 Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.51 to 1.0.53. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.51...v1.0.53) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d466582..603f8f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1389,9 +1389,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.51" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" +checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" dependencies = [ "itoa", "ryu", From df2501ca84a383226971fadb32e7d64ad605d6b0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 11 May 2020 04:41:41 +0000 Subject: [PATCH 23/69] Bump serde from 1.0.106 to 1.0.110 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.106 to 1.0.110. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.106...v1.0.110) Signed-off-by: dependabot-preview[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d466582..467c67e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1369,18 +1369,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.106" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.106" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" dependencies = [ "proc-macro2 1.0.8", "quote 1.0.2", From cccf31d0923ce992f4a1a50b9b58a8799ff7a595 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Mon, 18 May 2020 12:15:31 +0200 Subject: [PATCH 24/69] Add recommended VSCode extensions --- .vscode/extensions.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..0260fc1 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + + // List of extensions which should be recommended for users of this workspace. + "recommendations": ["matklad.rust-analyzer"], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] +} From fa08fe88929ac433f66d055777cbe4ba1607dc5e Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 19 May 2020 12:55:09 +0200 Subject: [PATCH 25/69] Improve vscode config --- .vscode/settings.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..02a859c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "editor.codeActionsOnSave": { + "source.fixAll": true, + "source.organizeImports": true + } +} From 53fc443023d9e96da211138d85fafcd14f19f0b5 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Thu, 4 Jun 2020 17:34:15 +0200 Subject: [PATCH 26/69] Add INSEE connector and a new action (#128) Co-authored-by: Julien Blatecky --- .env.sample | 1 + Cargo.lock | 1 + Cargo.toml | 2 +- README.md | 1 + src/commands/serve/runner/common.rs | 4 +- src/commands/serve/runner/error.rs | 3 + src/commands/serve/runner/mod.rs | 22 +++--- src/commands/update.rs | 7 +- src/connectors/insee.rs | 68 +++++++++++++++++++ src/connectors/local.rs | 8 ++- src/connectors/mod.rs | 21 ++++++ src/models/update_metadata/common.rs | 1 + .../action/{download.rs => download_stock.rs} | 0 .../action/{insert.rs => insert_stock.rs} | 0 src/update/action/mod.rs | 16 +++-- src/update/action/sync_insee.rs | 37 ++++++++++ .../action/{unzip.rs => unzip_stock.rs} | 0 src/update/mod.rs | 4 ++ 18 files changed, 175 insertions(+), 21 deletions(-) create mode 100644 src/connectors/insee.rs rename src/update/action/{download.rs => download_stock.rs} (100%) rename src/update/action/{insert.rs => insert_stock.rs} (100%) create mode 100644 src/update/action/sync_insee.rs rename src/update/action/{unzip.rs => unzip_stock.rs} (100%) diff --git a/.env.sample b/.env.sample index 7a1a670..d21bd92 100644 --- a/.env.sample +++ b/.env.sample @@ -7,3 +7,4 @@ DATABASE_URL=postgresql://sirene:sirenepw@127.0.0.1:5432/sirene TEMP_FOLDER=./data/temp FILE_FOLDER=./data/files DB_FOLDER=./data/files +INSEE_CREDENTIALS=Base64(consumer-key:consumer-secret) \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 247a9a2..e68508d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1200,6 +1200,7 @@ dependencies = [ "percent-encoding 2.1.0", "pin-project-lite", "serde", + "serde_json", "serde_urlencoded", "time", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 27d4767..9a4a5cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" openssl = "0.10" r2d2 = "0.8" -reqwest = { version = "0.10", features = ["blocking"] } +reqwest = { version = "0.10", features = ["blocking", "json"] } rocket = "0.4.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/README.md b/README.md index cd28695..1240cf3 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ SUBCOMMANDS: insert-data Load CSV file in database in loader-table from DB_FOLDER swap-data Swap loader-table to production unzip-file Unzip file from TEMP_FOLDER, and move it to the FILE_FOLDER + sync-insee Synchronise daily data from INSEE since the last modification ``` ### HTTP API diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index c0eace1..b79905d 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -1,12 +1,12 @@ use super::super::super::common::FolderOptions; -use crate::connectors::Connectors; +use crate::connectors::ConnectorsBuilders; use crate::models::etablissement::common::Etablissement; use crate::models::unite_legale::common::UniteLegale; use crate::models::update_metadata::common::{SyntheticGroupType, UpdateSummary}; use serde::{Deserialize, Serialize}; pub struct Context { - pub connectors: Connectors, + pub builders: ConnectorsBuilders, pub api_key: Option, pub folder_options: FolderOptions, } diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index bc63169..a15bfa8 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -1,3 +1,4 @@ +use crate::connectors::Error as ConnectorError; use crate::models::{etablissement, unite_legale}; use crate::update::error::Error as InternalUpdateError; use custom_error::custom_error; @@ -10,6 +11,7 @@ custom_error! { pub Error InvalidData = "Invalid data", MissingApiKeyError = "[Admin] Missing API key in configuration", ApiKeyError = "[Admin] Wrong API key", + UpdateConnectorError {source: ConnectorError} = "[Update] Error while creating connector: {source}", UpdateError {source: InternalUpdateError} = "[Update] {source}", UniteLegaleError {source: unite_legale::error::Error} = "[UniteLegale] {source}", EtablissementError {source: etablissement::error::Error} = "[Etablissement] {source}", @@ -29,6 +31,7 @@ impl<'r> Responder<'r> for Error { Error::InvalidData => Status::BadRequest, Error::MissingApiKeyError => return Err(Status::Unauthorized), Error::ApiKeyError => return Err(Status::Unauthorized), + Error::UpdateConnectorError { source: _ } => Status::InternalServerError, Error::UpdateError { source: _ } => Status::InternalServerError, Error::UniteLegaleError { source } => match source { unite_legale::error::Error::UniteLegaleNotFound => Status::NotFound, diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index ba34358..645cae7 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -34,6 +34,8 @@ fn update( return Err(Error::ApiKeyError); } + let connectors = state.builders.create_with_insee()?; + let summary = update_data( options.group_type, DataConfig { @@ -43,7 +45,7 @@ fn update( file_folder: state.folder_options.file.clone(), db_folder: state.folder_options.db.clone(), }, - &state.connectors, + &connectors, )?; Ok(Json(UpdateResponse { summary })) @@ -58,10 +60,12 @@ fn unites_legales( return Err(Error::InvalidData); } - let unite_legale = models::unite_legale::get(&state.connectors, &siren)?; - let etablissements = models::etablissement::get_with_siren(&state.connectors, &siren)?; + let connectors = state.builders.create(); + + let unite_legale = models::unite_legale::get(&connectors, &siren)?; + let etablissements = models::etablissement::get_with_siren(&connectors, &siren)?; let etablissement_siege = - models::etablissement::get_siege_with_siren(&state.connectors, &unite_legale.siren)?; + models::etablissement::get_siege_with_siren(&connectors, &unite_legale.siren)?; Ok(Json(UniteLegaleResponse { unite_legale: UniteLegaleInnerResponse { @@ -81,10 +85,12 @@ fn etablissements( return Err(Error::InvalidData); } - let etablissement = models::etablissement::get(&state.connectors, &siret)?; - let unite_legale = models::unite_legale::get(&state.connectors, &etablissement.siren)?; + let connectors = state.builders.create(); + + let etablissement = models::etablissement::get(&connectors, &siret)?; + let unite_legale = models::unite_legale::get(&connectors, &etablissement.siren)?; let etablissement_siege = - models::etablissement::get_siege_with_siren(&state.connectors, &etablissement.siren)?; + models::etablissement::get_siege_with_siren(&connectors, &etablissement.siren)?; Ok(Json(EtablissementResponse { etablissement: EtablissementInnerResponse { @@ -107,7 +113,7 @@ pub fn run( .mount("/v3", routes![index, unites_legales, etablissements]) .mount("/admin", routes![update]) .manage(Context { - connectors: builders.create(), + builders, api_key, folder_options, }) diff --git a/src/commands/update.rs b/src/commands/update.rs index 71c6bca..20407b3 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -41,10 +41,14 @@ enum UpdateSubCommand { /// Clean files from FILE_FOLDER #[clap(name = "clean-file")] CleanFile, + + /// Synchronise daily data from INSEE since the last modification + #[clap(name = "sync-insee")] + SyncInsee, } pub fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { - let connectors = builders.create(); + let connectors = builders.create_with_insee().unwrap(); let synthetic_group_type: SyntheticGroupType = flags.group_type.into(); // Prepare config @@ -64,6 +68,7 @@ pub fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: Connecto UpdateSubCommand::InsertData => Step::InsertData, UpdateSubCommand::SwapData => Step::SwapData, UpdateSubCommand::CleanFile => Step::CleanFile, + UpdateSubCommand::SyncInsee => Step::SyncInsee, }; update_step(step, synthetic_group_type, config, &connectors) diff --git a/src/connectors/insee.rs b/src/connectors/insee.rs new file mode 100644 index 0000000..a382c71 --- /dev/null +++ b/src/connectors/insee.rs @@ -0,0 +1,68 @@ +use custom_error::custom_error; +use reqwest::header::AUTHORIZATION; +use serde::{Serialize, Deserialize}; +use std::env; + +custom_error! { pub Error + TokenNetworkError { source: reqwest::Error } = "Unable to retrieve INSEE token (network error: {source})", + TokenApiError = "Unable to retrieve INSEE token", + TokenMalformedError {source: serde_json::Error} = "Unable to read INSEE token ({source})", +} + +pub struct Connector { + pub token: String, +} + +pub struct ConnectorBuilder { + pub credentials: String, +} + +#[derive(Serialize)] +struct InseeTokenParameters { + pub grant_type: String, + pub validity_period: u32 +} + +#[derive(Deserialize)] +struct InseeTokenResponse { + pub access_token: String, + pub scope: String, + pub token_type: String, + pub expires_in: i64, +} + +impl ConnectorBuilder { + pub fn new() -> Option { + let credentials = env::var("INSEE_CREDENTIALS").ok(); + + if let Some(credentials) = credentials { + Some(ConnectorBuilder { credentials }) + } else { + None + } + } + + pub fn create(&self) -> Result { + self.generate_token().map(|token| Connector { token }) + } + + fn generate_token(&self) -> Result { + let client = reqwest::blocking::Client::new(); + let response: InseeTokenResponse = client + .post("https://api.insee.fr/token") + .header(AUTHORIZATION, format!("Basic {}", self.credentials)) + .form(&InseeTokenParameters { + grant_type: String::from("client_credentials"), + validity_period: 86400 + }) + .send()? + .json()?; + + Ok(response.access_token) + } +} + +impl Connector { + pub fn get_daily_unites_legales(&self) {} + pub fn get_daily_etablissements(&self) {} +} diff --git a/src/connectors/local.rs b/src/connectors/local.rs index 2ac8b80..c7773ed 100644 --- a/src/connectors/local.rs +++ b/src/connectors/local.rs @@ -25,8 +25,12 @@ impl ConnectorBuilder { .expect(&format!("Error connecting to {}", database_url)), }; - let connection = builder.pool.get().expect("Unable to connect for migrations"); - embedded_migrations::run_with_output(&connection, &mut std::io::stdout()).expect("Unable to run migrations"); + let connection = builder + .pool + .get() + .expect("Unable to connect for migrations"); + embedded_migrations::run_with_output(&connection, &mut std::io::stdout()) + .expect("Unable to run migrations"); builder } diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index 33320ef..8c1193f 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -1,23 +1,44 @@ +use custom_error::custom_error; + +pub mod insee; pub mod local; +custom_error! { pub Error + InseeError { source: insee::Error } = "{source}", +} + pub struct ConnectorsBuilders { local: local::ConnectorBuilder, + insee: Option, } pub struct Connectors { pub local: local::Connector, + pub insee: Option, } impl ConnectorsBuilders { pub fn new() -> Self { ConnectorsBuilders { local: local::ConnectorBuilder::new(), + insee: insee::ConnectorBuilder::new(), } } pub fn create(&self) -> Connectors { Connectors { local: self.local.create(), + insee: None, } } + + pub fn create_with_insee(&self) -> Result { + Ok(Connectors { + local: self.local.create(), + insee: match self.insee.as_ref() { + Some(insee_builder) => Some(insee_builder.create()?), + None => None, + }, + }) + } } diff --git a/src/models/update_metadata/common.rs b/src/models/update_metadata/common.rs index 540d348..b079ca1 100644 --- a/src/models/update_metadata/common.rs +++ b/src/models/update_metadata/common.rs @@ -67,6 +67,7 @@ pub enum Step { InsertData, SwapData, CleanFile, + SyncInsee, } #[derive(Deserialize, Serialize, Clone, Debug)] diff --git a/src/update/action/download.rs b/src/update/action/download_stock.rs similarity index 100% rename from src/update/action/download.rs rename to src/update/action/download_stock.rs diff --git a/src/update/action/insert.rs b/src/update/action/insert_stock.rs similarity index 100% rename from src/update/action/insert.rs rename to src/update/action/insert_stock.rs diff --git a/src/update/action/mod.rs b/src/update/action/mod.rs index a582d83..1727252 100644 --- a/src/update/action/mod.rs +++ b/src/update/action/mod.rs @@ -8,10 +8,11 @@ use common::Action; pub mod clean; pub mod common; -pub mod download; -pub mod insert; +pub mod download_stock; +pub mod insert_stock; pub mod swap; -pub mod unzip; +pub mod sync_insee; +pub mod unzip_stock; pub fn execute_step( step: Step, @@ -28,7 +29,7 @@ pub fn execute_step( } Ok(UpdateStepSummary { - step: Step::DownloadFile, + step, updated: groups_summary.iter().find(|&g| g.updated).is_some(), started_timestamp, finished_timestamp: Utc::now(), @@ -38,16 +39,16 @@ pub fn execute_step( fn build_action(config: &Config, step: Step) -> Box { match step { - Step::DownloadFile => Box::new(download::DownloadAction { + Step::DownloadFile => Box::new(download_stock::DownloadAction { temp_folder: config.temp_folder.clone(), force: config.force, }), - Step::UnzipFile => Box::new(unzip::UnzipAction { + Step::UnzipFile => Box::new(unzip_stock::UnzipAction { temp_folder: config.temp_folder.clone(), file_folder: config.file_folder.clone(), force: config.force, }), - Step::InsertData => Box::new(insert::InsertAction { + Step::InsertData => Box::new(insert_stock::InsertAction { db_folder: config.file_folder.clone(), force: config.force, }), @@ -58,5 +59,6 @@ fn build_action(config: &Config, step: Step) -> Box { temp_folder: config.temp_folder.clone(), file_folder: config.file_folder.clone(), }), + Step::SyncInsee => Box::new(sync_insee::SyncInseeAction {}), } } diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs new file mode 100644 index 0000000..1de06c8 --- /dev/null +++ b/src/update/action/sync_insee.rs @@ -0,0 +1,37 @@ +use super::super::error::Error; +use super::common::Action; +use crate::connectors::Connectors; +use crate::models::group_metadata::common::GroupType; +use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use chrono::Utc; + +pub struct SyncInseeAction {} + +impl Action for SyncInseeAction { + fn step(&self) -> Step { + Step::SyncInsee + } + + fn execute( + &self, + group_type: GroupType, + connectors: &Connectors, + ) -> Result { + println!("[SyncInsee] Starting {:#?}", group_type); + let started_timestamp = Utc::now(); + + if let Some(insee) = &connectors.insee { + // Use Insee connector only if present + println!("Insee access token: {}", insee.token); + } + + println!("[SyncInsee] Finished for {:#?}", group_type); + Ok(UpdateGroupSummary { + group_type, + updated: true, + status_label: String::from("synced"), + started_timestamp, + finished_timestamp: Utc::now(), + }) + } +} diff --git a/src/update/action/unzip.rs b/src/update/action/unzip_stock.rs similarity index 100% rename from src/update/action/unzip.rs rename to src/update/action/unzip_stock.rs diff --git a/src/update/mod.rs b/src/update/mod.rs index 1c2539a..9d9f82d 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -86,6 +86,7 @@ fn build_workflow(config: &Config) -> Vec { if !config.data_only { workflow.push(Step::DownloadFile); + // If INSEE && newly downloaded file, get update date from INSEE and update workflow.push(Step::UnzipFile); } @@ -96,5 +97,8 @@ fn build_workflow(config: &Config) -> Vec { workflow.push(Step::CleanFile); } + // If INSEE, download and insert daily modifications + workflow.push(Step::SyncInsee); + workflow } From 93ddd4f575901a6f449c93583972884731b12ed6 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Wed, 10 Jun 2020 11:28:10 +0200 Subject: [PATCH 27/69] Get daily unites legales from insee (#129) Co-authored-by: Julien Blatecky --- .../down.sql | 9 + .../up.sql | 7 + src/connectors/insee/error.rs | 11 + src/connectors/insee/implementation.rs | 49 +++++ src/connectors/{insee.rs => insee/mod.rs} | 28 +-- src/connectors/insee/types.rs | 197 ++++++++++++++++++ src/connectors/mod.rs | 2 +- src/models/common.rs | 13 ++ src/models/etablissement/mod.rs | 25 +++ src/models/group_metadata/common.rs | 1 + src/models/schema.rs | 1 + src/models/unite_legale/mod.rs | 25 +++ src/update/action/sync_insee.rs | 4 + src/update/error.rs | 2 + 14 files changed, 356 insertions(+), 18 deletions(-) create mode 100644 migrations/2020-06-05-141443_add_insee_synced_date/down.sql create mode 100644 migrations/2020-06-05-141443_add_insee_synced_date/up.sql create mode 100644 src/connectors/insee/error.rs create mode 100644 src/connectors/insee/implementation.rs rename src/connectors/{insee.rs => insee/mod.rs} (64%) create mode 100644 src/connectors/insee/types.rs diff --git a/migrations/2020-06-05-141443_add_insee_synced_date/down.sql b/migrations/2020-06-05-141443_add_insee_synced_date/down.sql new file mode 100644 index 0000000..ddcef17 --- /dev/null +++ b/migrations/2020-06-05-141443_add_insee_synced_date/down.sql @@ -0,0 +1,9 @@ +ALTER TABLE "public"."group_metadata" DROP COLUMN "last_insee_synced_timestamp"; + +DROP INDEX "public"."unite_legale_staging_date_dernier_traitement_idx"; + +DROP INDEX "public"."etablissement_staging_date_dernier_traitement_idx"; + +DROP INDEX "public"."unite_legale_date_dernier_traitement_idx"; + +DROP INDEX "public"."etablissement_date_dernier_traitement_idx"; diff --git a/migrations/2020-06-05-141443_add_insee_synced_date/up.sql b/migrations/2020-06-05-141443_add_insee_synced_date/up.sql new file mode 100644 index 0000000..e867fc2 --- /dev/null +++ b/migrations/2020-06-05-141443_add_insee_synced_date/up.sql @@ -0,0 +1,7 @@ +ALTER TABLE "public"."group_metadata" ADD COLUMN "last_insee_synced_timestamp" timestamptz DEFAULT NULL; + +CREATE INDEX "unite_legale_staging_date_dernier_traitement_idx" ON "public"."unite_legale_staging" USING BTREE ("date_dernier_traitement"); +CREATE INDEX "unite_legale_date_dernier_traitement_idx" ON "public"."unite_legale" USING BTREE ("date_dernier_traitement"); + +CREATE INDEX "etablissement_staging_date_dernier_traitement_idx" ON "public"."etablissement_staging" USING BTREE ("date_dernier_traitement"); +CREATE INDEX "etablissement_date_dernier_traitement_idx" ON "public"."etablissement" USING BTREE ("date_dernier_traitement"); diff --git a/src/connectors/insee/error.rs b/src/connectors/insee/error.rs new file mode 100644 index 0000000..4d9c36e --- /dev/null +++ b/src/connectors/insee/error.rs @@ -0,0 +1,11 @@ +use custom_error::custom_error; + +custom_error! { pub InseeTokenError + NetworkError { source: reqwest::Error } = "Unable to retrieve INSEE token (network error: {source})", + ApiError = "Unable to retrieve INSEE token", + MalformedError {source: serde_json::Error} = "Unable to read INSEE token ({source})", +} + +custom_error! { pub InseeUpdateError + NetworkError {source: reqwest::Error} = "Unable to retrieve INSEE data (network error: {source})", +} diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs new file mode 100644 index 0000000..dd530b6 --- /dev/null +++ b/src/connectors/insee/implementation.rs @@ -0,0 +1,49 @@ +use super::error::InseeUpdateError; +use super::types::InseeUniteLegaleResponse; +use super::Connector; +use crate::models::unite_legale::common::UniteLegale; +use chrono::{DateTime, Utc}; +use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; + +const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; + +impl Connector { + pub fn get_daily_unites_legales( + &self, + _start_timestamp: DateTime, + ) -> Result, InseeUpdateError> { + let mut unites_legales: Vec = vec![]; + + let client = reqwest::blocking::Client::new(); + let mut has_data = true; + let mut current_cursor = String::from("*"); + + while has_data { + println!("Current cursor: {}", current_cursor); + + let url = format!( + "{}/siren?q=dateDernierTraitementUniteLegale:[2020-06-09 TO *]&nombre=1000&curseur={}", + BASE_URL, current_cursor + ); + + let response: InseeUniteLegaleResponse = client + .get(&url) + .header(AUTHORIZATION, format!("Bearer {}", self.token)) + .header(ACCEPT, HeaderValue::from_static("application/json")) + .send()? + .error_for_status()? + .json()?; + + unites_legales.extend(response.unites_legales.iter().filter_map(|u| u.into())); + + has_data = response.header.curseur != response.header.curseur_suivant; + current_cursor = response.header.curseur_suivant; + } + + Ok(unites_legales) + } + + pub fn get_daily_etablissements(&self) -> Result { + Ok(String::from("Etablissement")) + } +} diff --git a/src/connectors/insee.rs b/src/connectors/insee/mod.rs similarity index 64% rename from src/connectors/insee.rs rename to src/connectors/insee/mod.rs index a382c71..aadd6ee 100644 --- a/src/connectors/insee.rs +++ b/src/connectors/insee/mod.rs @@ -1,14 +1,13 @@ -use custom_error::custom_error; +mod implementation; +mod types; + +pub mod error; + +use error::InseeTokenError; use reqwest::header::AUTHORIZATION; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use std::env; -custom_error! { pub Error - TokenNetworkError { source: reqwest::Error } = "Unable to retrieve INSEE token (network error: {source})", - TokenApiError = "Unable to retrieve INSEE token", - TokenMalformedError {source: serde_json::Error} = "Unable to read INSEE token ({source})", -} - pub struct Connector { pub token: String, } @@ -20,7 +19,7 @@ pub struct ConnectorBuilder { #[derive(Serialize)] struct InseeTokenParameters { pub grant_type: String, - pub validity_period: u32 + pub validity_period: u32, } #[derive(Deserialize)] @@ -42,18 +41,18 @@ impl ConnectorBuilder { } } - pub fn create(&self) -> Result { + pub fn create(&self) -> Result { self.generate_token().map(|token| Connector { token }) } - fn generate_token(&self) -> Result { + fn generate_token(&self) -> Result { let client = reqwest::blocking::Client::new(); let response: InseeTokenResponse = client .post("https://api.insee.fr/token") .header(AUTHORIZATION, format!("Basic {}", self.credentials)) .form(&InseeTokenParameters { grant_type: String::from("client_credentials"), - validity_period: 86400 + validity_period: 86400, }) .send()? .json()?; @@ -61,8 +60,3 @@ impl ConnectorBuilder { Ok(response.access_token) } } - -impl Connector { - pub fn get_daily_unites_legales(&self) {} - pub fn get_daily_etablissements(&self) {} -} diff --git a/src/connectors/insee/types.rs b/src/connectors/insee/types.rs new file mode 100644 index 0000000..7c623a0 --- /dev/null +++ b/src/connectors/insee/types.rs @@ -0,0 +1,197 @@ +use crate::models::unite_legale::common::UniteLegale; +use chrono::{NaiveDate, NaiveDateTime}; +use serde::Deserialize; +use std::fmt::Display; +use std::str::FromStr; + +fn default_as_false() -> bool { + false +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegaleResponse { + pub header: Header, + pub unites_legales: Vec, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Header { + total: u32, + debut: u32, + nombre: u32, + pub curseur: String, + pub curseur_suivant: String, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegaleInner { + pub siren: String, + pub statut_diffusion_unite_legale: String, + + #[serde(default = "default_as_false")] + pub unite_purgee_unite_legale: bool, + + pub date_creation_unite_legale: Option, + pub sigle_unite_legale: Option, + pub sexe_unite_legale: Option, + pub prenom1_unite_legale: Option, + pub prenom2_unite_legale: Option, + pub prenom3_unite_legale: Option, + pub prenom4_unite_legale: Option, + pub prenom_usuel_unite_legale: Option, + pub pseudonyme_unite_legale: Option, + pub identifiant_association_unite_legale: Option, + pub tranche_effectifs_unite_legale: Option, + + #[serde(deserialize_with = "from_str_optional")] + pub annee_effectifs_unite_legale: Option, + + pub date_dernier_traitement_unite_legale: Option, + pub nombre_periodes_unite_legale: Option, + pub categorie_entreprise: Option, + + #[serde(deserialize_with = "from_str_optional")] + pub annee_categorie_entreprise: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegale { + #[serde(flatten)] + pub content: InseeUniteLegaleInner, + pub periodes_unite_legale: Vec, +} + +#[derive(Debug)] +pub struct InseeUniteLegaleWithPeriode { + pub content: InseeUniteLegaleInner, + pub periode: PeriodeInseeUniteLegale, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PeriodeInseeUniteLegale { + pub date_fin: Option, + pub date_debut: Option, + + #[serde(deserialize_with = "deserialize_etat_administratif")] + pub etat_administratif_unite_legale: String, + + pub changement_etat_administratif_unite_legale: bool, + pub nom_unite_legale: Option, + pub changement_nom_unite_legale: bool, + pub nom_usage_unite_legale: Option, + pub changement_nom_usage_unite_legale: bool, + pub denomination_unite_legale: Option, + pub changement_denomination_unite_legale: bool, + pub denomination_usuelle1_unite_legale: Option, + pub denomination_usuelle2_unite_legale: Option, + pub denomination_usuelle3_unite_legale: Option, + pub changement_denomination_usuelle_unite_legale: bool, + pub categorie_juridique_unite_legale: Option, + pub changement_categorie_juridique_unite_legale: bool, + pub activite_principale_unite_legale: Option, + pub nomenclature_activite_principale_unite_legale: Option, + pub changement_activite_principale_unite_legale: bool, + pub nic_siege_unite_legale: Option, + pub changement_nic_siege_unite_legale: bool, + pub economie_sociale_solidaire_unite_legale: Option, + pub changement_economie_sociale_solidaire_unite_legale: bool, + pub caractere_employeur_unite_legale: Option, + pub changement_caractere_employeur_unite_legale: bool, +} + +impl From<&InseeUniteLegale> for Option { + fn from(u: &InseeUniteLegale) -> Self { + match u + .periodes_unite_legale + .iter() + .find(|p| p.date_fin.is_none()) + { + Some(periode) => { + // Convert + Some( + InseeUniteLegaleWithPeriode { + content: u.content.clone(), + periode: periode.clone(), + } + .into(), + ) + } + None => None, + } + } +} + +impl From for UniteLegale { + fn from(u: InseeUniteLegaleWithPeriode) -> Self { + UniteLegale { + siren: u.content.siren, + statut_diffusion: u.content.statut_diffusion_unite_legale, + unite_purgee: Some(u.content.unite_purgee_unite_legale.to_string()), + date_creation: u.content.date_creation_unite_legale, + sigle: u.content.sigle_unite_legale, + sexe: u.content.sexe_unite_legale, + prenom_1: u.content.prenom1_unite_legale, + prenom_2: u.content.prenom2_unite_legale, + prenom_3: u.content.prenom3_unite_legale, + prenom_4: u.content.prenom4_unite_legale, + prenom_usuel: u.content.prenom_usuel_unite_legale, + pseudonyme: u.content.pseudonyme_unite_legale, + identifiant_association: u.content.identifiant_association_unite_legale, + tranche_effectifs: u.content.tranche_effectifs_unite_legale, + annee_effectifs: u.content.annee_effectifs_unite_legale, + date_dernier_traitement: u.content.date_dernier_traitement_unite_legale, + nombre_periodes: u.content.nombre_periodes_unite_legale, + categorie_entreprise: u.content.categorie_entreprise, + annee_categorie_entreprise: u.content.annee_categorie_entreprise, + date_debut: u.periode.date_debut, + etat_administratif: u.periode.etat_administratif_unite_legale, + nom: u.periode.nom_unite_legale, + nom_usage: u.periode.nom_usage_unite_legale, + denomination: u.periode.denomination_unite_legale, + denomination_usuelle_1: u.periode.denomination_usuelle1_unite_legale, + denomination_usuelle_2: u.periode.denomination_usuelle2_unite_legale, + denomination_usuelle_3: u.periode.denomination_usuelle3_unite_legale, + categorie_juridique: u.periode.categorie_juridique_unite_legale, + activite_principale: u.periode.activite_principale_unite_legale, + nomenclature_activite_principale: u + .periode + .nomenclature_activite_principale_unite_legale, + nic_siege: u.periode.nic_siege_unite_legale, + economie_sociale_solidaire: u.periode.economie_sociale_solidaire_unite_legale, + caractere_employeur: u.periode.caractere_employeur_unite_legale, + } + } +} + +fn from_str_optional<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: FromStr, + T::Err: Display, + D: serde::Deserializer<'de>, +{ + let deser_res: Result = serde::Deserialize::deserialize(deserializer); + match deser_res { + Ok(serde_json::Value::String(s)) => T::from_str(&s) + .map_err(serde::de::Error::custom) + .map(Option::from), + Ok(_v) => { + // Commenting this print to prevent console spamming + // println!("string expected but found something else: {}", v); + Ok(None) + } + Err(_) => Ok(None), + } +} + +fn deserialize_etat_administratif<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let opt = Option::deserialize(deserializer)?; + Ok(opt.unwrap_or(String::from("C"))) +} diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index 8c1193f..583417e 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -4,7 +4,7 @@ pub mod insee; pub mod local; custom_error! { pub Error - InseeError { source: insee::Error } = "{source}", + InseeError { source: insee::error::InseeTokenError } = "{source}", } pub struct ConnectorsBuilders { diff --git a/src/models/common.rs b/src/models/common.rs index 9abbf42..8eb2e23 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -1,4 +1,6 @@ +use crate::connectors::insee::error::InseeUpdateError; use crate::connectors::Connectors; +use chrono::{DateTime, Utc}; use custom_error::custom_error; pub trait UpdatableModel { @@ -6,9 +8,20 @@ pub trait UpdatableModel { fn count_staging(&self, connectors: &Connectors) -> Result; fn insert_in_staging(&self, connectors: &Connectors, file_path: String) -> Result; fn swap(&self, connectors: &Connectors) -> Result<(), Error>; + fn get_last_insee_synced_timestamp( + &self, + connectors: &Connectors, + ) -> Result>, Error>; + fn update_daily_data( + &self, + connectors: &Connectors, + start_timestamp: DateTime, + ) -> Result<(), Error>; } custom_error! { pub Error LocalConnectionFailed{source: r2d2::Error} = "Unable to connect to local database ({source}).", DatabaseError{source: diesel::result::Error} = "Unable to run some operations on updatable model ({source}).", + UpdateError {source: InseeUpdateError} = "{source}", + MissingInseeConnector = "Missing required Insee connector", } diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index efa7671..6fcb75d 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -5,6 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; use crate::connectors::Connectors; +use chrono::{DateTime, Utc}; use common::Etablissement; use diesel::prelude::*; use diesel::sql_query; @@ -109,4 +110,28 @@ impl UpdatableModel for EtablissementModel { Ok(()) }) } + + // SELECT date_dernier_traitement FROM etablissement WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; + fn get_last_insee_synced_timestamp( + &self, + _connectors: &Connectors, + ) -> Result>, UpdatableError> { + Ok(None) + } + + fn update_daily_data( + &self, + connectors: &Connectors, + _start_timestamp: DateTime, + ) -> Result<(), UpdatableError> { + let insee = connectors + .insee + .as_ref() + .ok_or(UpdatableError::MissingInseeConnector)?; + + let result = insee.get_daily_etablissements()?; + + println!("{:#?}", result); + Ok(()) + } } diff --git a/src/models/group_metadata/common.rs b/src/models/group_metadata/common.rs index 59ce012..c2156a3 100644 --- a/src/models/group_metadata/common.rs +++ b/src/models/group_metadata/common.rs @@ -23,6 +23,7 @@ pub struct Metadata { pub url: String, pub created_at: DateTime, pub updated_at: DateTime, + pub last_insee_synced_timestamp: Option>, } #[derive(AsChangeset)] diff --git a/src/models/schema.rs b/src/models/schema.rs index 8077f85..dff139d 100644 --- a/src/models/schema.rs +++ b/src/models/schema.rs @@ -117,6 +117,7 @@ table! { url -> Text, created_at -> Timestamptz, updated_at -> Timestamptz, + last_insee_synced_timestamp -> Nullable, } } diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index f2e80ab..dd1bd87 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -5,6 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; use crate::connectors::Connectors; +use chrono::{DateTime, Utc}; use common::UniteLegale; use diesel::prelude::*; use diesel::sql_query; @@ -87,4 +88,28 @@ impl UpdatableModel for UniteLegaleModel { Ok(()) }) } + + // SELECT date_dernier_traitement FROM unite_legale WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; + fn get_last_insee_synced_timestamp( + &self, + _connectors: &Connectors, + ) -> Result>, UpdatableError> { + Ok(None) + } + + fn update_daily_data( + &self, + connectors: &Connectors, + start_timestamp: DateTime, + ) -> Result<(), UpdatableError> { + let insee = connectors + .insee + .as_ref() + .ok_or(UpdatableError::MissingInseeConnector)?; + + let result = insee.get_daily_unites_legales(start_timestamp)?; + + println!("[SyncInsee] Done updating {} UnitesLegales", result.len()); + Ok(()) + } } diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 1de06c8..e320745 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -23,6 +23,10 @@ impl Action for SyncInseeAction { if let Some(insee) = &connectors.insee { // Use Insee connector only if present println!("Insee access token: {}", insee.token); + + let model = group_type.get_updatable_model(); + + model.update_daily_data(connectors, started_timestamp)?; } println!("[SyncInsee] Finished for {:#?}", group_type); diff --git a/src/update/error.rs b/src/update/error.rs index 8928364..d6e37a3 100644 --- a/src/update/error.rs +++ b/src/update/error.rs @@ -1,3 +1,4 @@ +use crate::connectors::insee::error::InseeUpdateError; use crate::models; use crate::models::group_metadata::common::GroupType; use crate::models::{group_metadata, update_metadata}; @@ -26,6 +27,7 @@ custom_error! { pub Error InvalidCSVPath = "Invalid CSV path, not UTF8 compatible.", InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error}).", SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override.", + SyncInseeError {source: InseeUpdateError} = "[SyncInsee] {source}", } impl Error { From 48c4a3a2b0fa2ea05d5fc1394ca3610675393f17 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Wed, 10 Jun 2020 15:34:59 +0200 Subject: [PATCH 28/69] Use last treatment date to retrieve daily updates (#133) --- src/connectors/insee/implementation.rs | 23 +++++++++++++++++------ src/models/common.rs | 6 +++--- src/models/etablissement/mod.rs | 6 +++--- src/models/unite_legale/mod.rs | 16 +++++++++++----- src/update/action/sync_insee.rs | 4 +++- 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index dd530b6..8bdb583 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -2,29 +2,40 @@ use super::error::InseeUpdateError; use super::types::InseeUniteLegaleResponse; use super::Connector; use crate::models::unite_legale::common::UniteLegale; -use chrono::{DateTime, Utc}; +use chrono::{Duration, NaiveDate, NaiveDateTime, Utc}; use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; impl Connector { + fn get_minimum_timestamp_for_request(&self, timestamp: NaiveDateTime) -> NaiveDate { + let minimum_date: NaiveDate = (Utc::now().naive_local() - Duration::days(60)).date(); + + minimum_date.max(timestamp.date()) + } + pub fn get_daily_unites_legales( &self, - _start_timestamp: DateTime, + start_timestamp: NaiveDateTime, ) -> Result, InseeUpdateError> { let mut unites_legales: Vec = vec![]; let client = reqwest::blocking::Client::new(); + let mut has_data = true; let mut current_cursor = String::from("*"); + let base_request_url = format!( + "{}/siren?q=dateDernierTraitementUniteLegale:[{} TO *]&nombre=1000", + BASE_URL, + self.get_minimum_timestamp_for_request(start_timestamp) + ); + + println!("Base Request URL: {}", base_request_url); while has_data { println!("Current cursor: {}", current_cursor); - let url = format!( - "{}/siren?q=dateDernierTraitementUniteLegale:[2020-06-09 TO *]&nombre=1000&curseur={}", - BASE_URL, current_cursor - ); + let url = format!("{}&curseur={}", base_request_url, current_cursor); let response: InseeUniteLegaleResponse = client .get(&url) diff --git a/src/models/common.rs b/src/models/common.rs index 8eb2e23..fd4962e 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -1,6 +1,6 @@ use crate::connectors::insee::error::InseeUpdateError; use crate::connectors::Connectors; -use chrono::{DateTime, Utc}; +use chrono::NaiveDateTime; use custom_error::custom_error; pub trait UpdatableModel { @@ -11,11 +11,11 @@ pub trait UpdatableModel { fn get_last_insee_synced_timestamp( &self, connectors: &Connectors, - ) -> Result>, Error>; + ) -> Result, Error>; fn update_daily_data( &self, connectors: &Connectors, - start_timestamp: DateTime, + start_timestamp: NaiveDateTime, ) -> Result<(), Error>; } diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 6fcb75d..0eafeee 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -5,7 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; use crate::connectors::Connectors; -use chrono::{DateTime, Utc}; +use chrono::NaiveDateTime; use common::Etablissement; use diesel::prelude::*; use diesel::sql_query; @@ -115,14 +115,14 @@ impl UpdatableModel for EtablissementModel { fn get_last_insee_synced_timestamp( &self, _connectors: &Connectors, - ) -> Result>, UpdatableError> { + ) -> Result, UpdatableError> { Ok(None) } fn update_daily_data( &self, connectors: &Connectors, - _start_timestamp: DateTime, + _start_timestamp: NaiveDateTime, ) -> Result<(), UpdatableError> { let insee = connectors .insee diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index dd1bd87..161ba08 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -5,7 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; use crate::connectors::Connectors; -use chrono::{DateTime, Utc}; +use chrono::NaiveDateTime; use common::UniteLegale; use diesel::prelude::*; use diesel::sql_query; @@ -92,15 +92,21 @@ impl UpdatableModel for UniteLegaleModel { // SELECT date_dernier_traitement FROM unite_legale WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; fn get_last_insee_synced_timestamp( &self, - _connectors: &Connectors, - ) -> Result>, UpdatableError> { - Ok(None) + connectors: &Connectors, + ) -> Result, UpdatableError> { + let connection = connectors.local.pool.get()?; + dsl::unite_legale + .select(dsl::date_dernier_traitement) + .order(dsl::date_dernier_traitement.desc()) + .filter(dsl::date_dernier_traitement.is_not_null()) + .first::>(&connection) + .map_err(|error| error.into()) } fn update_daily_data( &self, connectors: &Connectors, - start_timestamp: DateTime, + start_timestamp: NaiveDateTime, ) -> Result<(), UpdatableError> { let insee = connectors .insee diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index e320745..5b6f619 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -26,7 +26,9 @@ impl Action for SyncInseeAction { let model = group_type.get_updatable_model(); - model.update_daily_data(connectors, started_timestamp)?; + if let Some(timestamp) = model.get_last_insee_synced_timestamp(connectors)? { + model.update_daily_data(connectors, timestamp)?; + } } println!("[SyncInsee] Finished for {:#?}", group_type); From a784b69f7a286434bb9aa596fdbc03f56b0a8d63 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Thu, 11 Jun 2020 11:11:08 +0200 Subject: [PATCH 29/69] Update last insee synced timestamp (#134) --- src/models/group_metadata/mod.rs | 13 +++++++++++++ src/update/action/sync_insee.rs | 22 ++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/models/group_metadata/mod.rs b/src/models/group_metadata/mod.rs index f604968..09b2fa6 100644 --- a/src/models/group_metadata/mod.rs +++ b/src/models/group_metadata/mod.rs @@ -68,6 +68,19 @@ pub fn set_last_imported_timestamp( .map_err(|error| error.into()) } +pub fn set_last_insee_synced_timestamp( + connectors: &Connectors, + group_type: GroupType, + timestamp: DateTime, +) -> Result { + let connection = connectors.local.pool.get()?; + diesel::update(dsl::group_metadata.filter(dsl::group_type.eq(group_type))) + .set(dsl::last_insee_synced_timestamp.eq(timestamp)) + .execute(&connection) + .map(|count| count > 0) + .map_err(|error| error.into()) +} + pub fn reset_staging_timestamps( connectors: &Connectors, group_type: GroupType, diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 5b6f619..8e63e13 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -1,6 +1,7 @@ use super::super::error::Error; use super::common::Action; use crate::connectors::Connectors; +use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; use chrono::Utc; @@ -17,8 +18,10 @@ impl Action for SyncInseeAction { group_type: GroupType, connectors: &Connectors, ) -> Result { - println!("[SyncInsee] Starting {:#?}", group_type); + println!("[SyncInsee] Syncing {:#?}", group_type); let started_timestamp = Utc::now(); + let mut updated = false; + let mut status_label = String::from("missing insee connector"); if let Some(insee) = &connectors.insee { // Use Insee connector only if present @@ -26,16 +29,27 @@ impl Action for SyncInseeAction { let model = group_type.get_updatable_model(); + status_label = String::from("missing last treatment date"); + if let Some(timestamp) = model.get_last_insee_synced_timestamp(connectors)? { model.update_daily_data(connectors, timestamp)?; + + group_metadata::set_last_insee_synced_timestamp( + connectors, + group_type, + Utc::now(), + )?; + + updated = true; + status_label = String::from("synced"); } } - println!("[SyncInsee] Finished for {:#?}", group_type); + println!("[SyncInsee] {:#?} synced", group_type); Ok(UpdateGroupSummary { group_type, - updated: true, - status_label: String::from("synced"), + updated, + status_label, started_timestamp, finished_timestamp: Utc::now(), }) From c05d29d9cf98b54e2d4376d0fab15443866f5a85 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Thu, 11 Jun 2020 15:39:48 +0200 Subject: [PATCH 30/69] Upsert daily legal units retrieved from INSEE (#137) --- src/connectors/insee/implementation.rs | 74 +++++++++++++++----------- src/connectors/insee/mod.rs | 2 + src/connectors/insee/types.rs | 6 +-- src/models/common.rs | 2 +- src/models/etablissement/mod.rs | 4 +- src/models/unite_legale/common.rs | 4 +- src/models/unite_legale/mod.rs | 61 +++++++++++++++++++-- src/update/action/sync_insee.rs | 21 ++++---- 8 files changed, 121 insertions(+), 53 deletions(-) diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index 8bdb583..3c5544f 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -2,56 +2,68 @@ use super::error::InseeUpdateError; use super::types::InseeUniteLegaleResponse; use super::Connector; use crate::models::unite_legale::common::UniteLegale; -use chrono::{Duration, NaiveDate, NaiveDateTime, Utc}; +use chrono::{Duration, NaiveDateTime, Utc}; use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; +pub const INITIAL_CURSOR: &str = "*"; impl Connector { - fn get_minimum_timestamp_for_request(&self, timestamp: NaiveDateTime) -> NaiveDate { - let minimum_date: NaiveDate = (Utc::now().naive_local() - Duration::days(60)).date(); - - minimum_date.max(timestamp.date()) + fn get_minimum_timestamp_for_request(&self, timestamp: NaiveDateTime) -> NaiveDateTime { + timestamp.max(Utc::now().naive_local() - Duration::days(31)) } pub fn get_daily_unites_legales( &self, start_timestamp: NaiveDateTime, - ) -> Result, InseeUpdateError> { - let mut unites_legales: Vec = vec![]; - + cursor: String, + ) -> Result<(Option, Vec), InseeUpdateError> { let client = reqwest::blocking::Client::new(); - let mut has_data = true; - let mut current_cursor = String::from("*"); - let base_request_url = format!( - "{}/siren?q=dateDernierTraitementUniteLegale:[{} TO *]&nombre=1000", + let url = format!( + "{}/siren?q=dateDernierTraitementUniteLegale:%7B{} TO *]&nombre=1000&curseur={}", BASE_URL, self.get_minimum_timestamp_for_request(start_timestamp) + .format("%Y-%m-%dT%H:%M:%S"), + cursor ); - println!("Base Request URL: {}", base_request_url); - - while has_data { - println!("Current cursor: {}", current_cursor); - - let url = format!("{}&curseur={}", base_request_url, current_cursor); - - let response: InseeUniteLegaleResponse = client - .get(&url) - .header(AUTHORIZATION, format!("Bearer {}", self.token)) - .header(ACCEPT, HeaderValue::from_static("application/json")) - .send()? - .error_for_status()? - .json()?; + let response = match client + .get(&url) + .header(AUTHORIZATION, format!("Bearer {}", self.token)) + .header(ACCEPT, HeaderValue::from_static("application/json")) + .send()? + .error_for_status() + { + Ok(response) => response + .json::() + .map_err(|error| error.into()), + Err(error) => { + // Insee returns 404 for empty data + if let Some(status) = error.status() { + if status == reqwest::StatusCode::NOT_FOUND { + return Ok((None, vec![])); + } + } - unites_legales.extend(response.unites_legales.iter().filter_map(|u| u.into())); + Err(error) + } + }?; - has_data = response.header.curseur != response.header.curseur_suivant; - current_cursor = response.header.curseur_suivant; - } + let next_cursor = if response.header.curseur == response.header.curseur_suivant { + None + } else { + Some(response.header.curseur_suivant) + }; - Ok(unites_legales) + Ok(( + next_cursor, + response + .unites_legales + .iter() + .filter_map(|u| u.into()) + .collect(), + )) } pub fn get_daily_etablissements(&self) -> Result { diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index aadd6ee..372c4c1 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -8,6 +8,8 @@ use reqwest::header::AUTHORIZATION; use serde::{Deserialize, Serialize}; use std::env; +pub use implementation::INITIAL_CURSOR; + pub struct Connector { pub token: String, } diff --git a/src/connectors/insee/types.rs b/src/connectors/insee/types.rs index 7c623a0..f92778a 100644 --- a/src/connectors/insee/types.rs +++ b/src/connectors/insee/types.rs @@ -179,9 +179,9 @@ where Ok(serde_json::Value::String(s)) => T::from_str(&s) .map_err(serde::de::Error::custom) .map(Option::from), - Ok(_v) => { - // Commenting this print to prevent console spamming - // println!("string expected but found something else: {}", v); + Ok(serde_json::Value::Null) => Ok(None), + Ok(v) => { + println!("string expected but found something else: {}", v); Ok(None) } Err(_) => Ok(None), diff --git a/src/models/common.rs b/src/models/common.rs index fd4962e..43fdc4b 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -16,7 +16,7 @@ pub trait UpdatableModel { &self, connectors: &Connectors, start_timestamp: NaiveDateTime, - ) -> Result<(), Error>; + ) -> Result; } custom_error! { pub Error diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 0eafeee..0e13434 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -123,7 +123,7 @@ impl UpdatableModel for EtablissementModel { &self, connectors: &Connectors, _start_timestamp: NaiveDateTime, - ) -> Result<(), UpdatableError> { + ) -> Result { let insee = connectors .insee .as_ref() @@ -132,6 +132,6 @@ impl UpdatableModel for EtablissementModel { let result = insee.get_daily_etablissements()?; println!("{:#?}", result); - Ok(()) + Ok(0) } } diff --git a/src/models/unite_legale/common.rs b/src/models/unite_legale/common.rs index f2aff92..1d97939 100644 --- a/src/models/unite_legale/common.rs +++ b/src/models/unite_legale/common.rs @@ -1,7 +1,9 @@ +use super::super::schema::unite_legale; use chrono::{NaiveDate, NaiveDateTime}; use serde::Serialize; -#[derive(Queryable, Serialize, Clone, Debug)] +#[derive(Insertable, Queryable, Serialize, Clone, Debug)] +#[table_name = "unite_legale"] pub struct UniteLegale { pub siren: String, pub statut_diffusion: String, diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index 161ba08..cc3f232 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -4,9 +4,10 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; -use crate::connectors::Connectors; +use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; use chrono::NaiveDateTime; use common::UniteLegale; +use diesel::pg::upsert::excluded; use diesel::prelude::*; use diesel::sql_query; use error::Error; @@ -107,15 +108,65 @@ impl UpdatableModel for UniteLegaleModel { &self, connectors: &Connectors, start_timestamp: NaiveDateTime, - ) -> Result<(), UpdatableError> { + ) -> Result { let insee = connectors .insee .as_ref() .ok_or(UpdatableError::MissingInseeConnector)?; - let result = insee.get_daily_unites_legales(start_timestamp)?; + let connection = connectors.local.pool.get()?; + + let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); + let mut updated_count: usize = 0; + + while let Some(cursor) = current_cursor { + let (next_cursor, unites_legales) = + insee.get_daily_unites_legales(start_timestamp, cursor)?; + + current_cursor = next_cursor; + + updated_count += diesel::insert_into(dsl::unite_legale) + .values(&unites_legales) + .on_conflict(dsl::siren) + .do_update() + .set(( + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::unite_purgee.eq(excluded(dsl::unite_purgee)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::sigle.eq(excluded(dsl::sigle)), + dsl::sexe.eq(excluded(dsl::sexe)), + dsl::prenom_1.eq(excluded(dsl::prenom_1)), + dsl::prenom_2.eq(excluded(dsl::prenom_2)), + dsl::prenom_3.eq(excluded(dsl::prenom_3)), + dsl::prenom_4.eq(excluded(dsl::prenom_4)), + dsl::prenom_usuel.eq(excluded(dsl::prenom_usuel)), + dsl::pseudonyme.eq(excluded(dsl::pseudonyme)), + dsl::identifiant_association.eq(excluded(dsl::identifiant_association)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::categorie_entreprise.eq(excluded(dsl::categorie_entreprise)), + dsl::annee_categorie_entreprise.eq(excluded(dsl::annee_categorie_entreprise)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::nom.eq(excluded(dsl::nom)), + dsl::nom_usage.eq(excluded(dsl::nom_usage)), + dsl::denomination.eq(excluded(dsl::denomination)), + dsl::denomination_usuelle_1.eq(excluded(dsl::denomination_usuelle_1)), + dsl::denomination_usuelle_2.eq(excluded(dsl::denomination_usuelle_2)), + dsl::denomination_usuelle_3.eq(excluded(dsl::denomination_usuelle_3)), + dsl::categorie_juridique.eq(excluded(dsl::categorie_juridique)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::nic_siege.eq(excluded(dsl::nic_siege)), + dsl::economie_sociale_solidaire.eq(excluded(dsl::economie_sociale_solidaire)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; + } - println!("[SyncInsee] Done updating {} UnitesLegales", result.len()); - Ok(()) + Ok(updated_count) } } diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 8e63e13..18fb0fb 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -20,19 +20,16 @@ impl Action for SyncInseeAction { ) -> Result { println!("[SyncInsee] Syncing {:#?}", group_type); let started_timestamp = Utc::now(); + let status_label: String; let mut updated = false; - let mut status_label = String::from("missing insee connector"); - - if let Some(insee) = &connectors.insee { - // Use Insee connector only if present - println!("Insee access token: {}", insee.token); + // Use Insee connector only if present + if connectors.insee.is_some() { let model = group_type.get_updatable_model(); - status_label = String::from("missing last treatment date"); - if let Some(timestamp) = model.get_last_insee_synced_timestamp(connectors)? { - model.update_daily_data(connectors, timestamp)?; + let updated_count = model.update_daily_data(connectors, timestamp)?; + println!("[SyncInsee] {} {:#?} synced", updated_count, group_type); group_metadata::set_last_insee_synced_timestamp( connectors, @@ -40,12 +37,16 @@ impl Action for SyncInseeAction { Utc::now(), )?; - updated = true; + updated = updated_count > 0; status_label = String::from("synced"); + } else { + status_label = String::from("missing last treatment date"); } + } else { + status_label = String::from("no insee connector configured"); } - println!("[SyncInsee] {:#?} synced", group_type); + println!("[SyncInsee] Syncing of {:#?} done", group_type); Ok(UpdateGroupSummary { group_type, updated, From c085a452f90580950db13ad837138bedbe1c2f0a Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 11 Jun 2020 16:39:46 +0200 Subject: [PATCH 31/69] Upgrade dependencies --- Cargo.lock | 758 +++++++++++++++++++++++--------------- Cargo.toml | 6 +- src/commands/common.rs | 12 +- src/commands/serve/mod.rs | 14 +- src/commands/update.rs | 1 + 5 files changed, 472 insertions(+), 319 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e68508d..d67484a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,10 +7,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" [[package]] -name = "anyhow" -version = "1.0.26" +name = "aead" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf01b9b56e767bb57b94ebf91a58b338002963785cdd7013e21c0d4679471e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54eb1d8fe354e5fc611daf4f2ea97dd45a765f4f1e4512306ec183ae2e8f20c9" +dependencies = [ + "aes-soft", + "aesni", + "block-cipher-trait", +] + +[[package]] +name = "aes-gcm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "834a6bda386024dbb7c8fc51322856c10ffe69559f972261c868485f5759c638" +dependencies = [ + "aead", + "aes", + "block-cipher-trait", + "ghash", + "subtle 2.2.3", + "zeroize", +] + +[[package]] +name = "aes-soft" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" +dependencies = [ + "block-cipher-trait", + "byteorder", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" +dependencies = [ + "block-cipher-trait", + "opaque-debug", +] [[package]] name = "atty" @@ -41,30 +90,57 @@ dependencies = [ [[package]] name = "base64" -version = "0.10.1" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d1ccbaf7d9ec9537465a97bf19edc1a4e158ecb49fc16178202238c569cc42" + +[[package]] +name = "bitflags" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ + "block-padding", + "byte-tools", "byteorder", + "generic-array", ] [[package]] -name = "base64" -version = "0.11.0" +name = "block-cipher-trait" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +checksum = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774" +dependencies = [ + "generic-array", +] [[package]] -name = "bitflags" -version = "1.2.1" +name = "block-padding" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] [[package]] name = "bumpalo" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" @@ -90,28 +166,20 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.7" +version = "0.1.9+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" dependencies = [ "cc", "libc", -] - -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -dependencies = [ - "ppv-lite86", + "pkg-config", ] [[package]] name = "cc" -version = "1.0.50" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" [[package]] name = "cfg-if" @@ -134,13 +202,15 @@ dependencies = [ [[package]] name = "clap" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e1d83a78804a271b053d4d21f69b67f7eb01ad01" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "860643c53f980f0d38a5e25dfab6c3c93b2cb3aa1fe192643d17a293c6c41936" dependencies = [ "atty", "bitflags", "clap_derive", "indexmap", "lazy_static", + "os_str_bytes", "strsim", "termcolor", "textwrap", @@ -151,13 +221,14 @@ dependencies = [ [[package]] name = "clap_derive" version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e1d83a78804a271b053d4d21f69b67f7eb01ad01" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb51c9e75b94452505acd21d929323f5a5c6c4735a852adbd39ef5fb1b014f30" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] @@ -171,21 +242,25 @@ dependencies = [ [[package]] name = "cookie" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9fac5e7bdefb6160fb181ee0eaa6f96704b625c70e6d61c465cb35750a4ea12" +checksum = "5795cda0897252e34380a27baf884c53aa7ad9990329cdad96d4c5d027015d44" dependencies = [ - "base64 0.9.3", - "ring", + "aes-gcm", + "base64 0.12.1", + "hkdf", + "hmac", + "percent-encoding 2.1.0", + "rand", + "sha2", "time", - "url 1.7.2", ] [[package]] name = "core-foundation" -version = "0.6.4" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" dependencies = [ "core-foundation-sys", "libc", @@ -193,9 +268,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" [[package]] name = "crc32fast" @@ -206,6 +281,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle 1.0.0", +] + [[package]] name = "custom_error" version = "1.7.1" @@ -246,9 +331,9 @@ dependencies = [ [[package]] name = "diesel" -version = "1.4.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d7ca63eb2efea87a7f56a283acc49e2ce4b2bd54adf7465dc1d81fef13d8fc" +checksum = "3e2de9deab977a153492a1468d1b1c0662c1cf39e5ea87d0c060ecd59ef18d8c" dependencies = [ "bitflags", "byteorder", @@ -265,9 +350,9 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] @@ -280,6 +365,15 @@ dependencies = [ "migrations_macros", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -294,18 +388,24 @@ checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" [[package]] name = "encoding_rs" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" +checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" dependencies = [ "cfg-if", ] +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "filetime" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +checksum = "affc17579b132fc2461adf7c575cc6e8b134ebca52c51f5411388965227dc695" dependencies = [ "cfg-if", "libc", @@ -315,9 +415,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", @@ -327,9 +427,9 @@ dependencies = [ [[package]] name = "fnv" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foreign-types" @@ -383,51 +483,64 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures-channel" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" [[package]] name = "futures-io" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" [[package]] name = "futures-sink" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" [[package]] name = "futures-task" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] [[package]] name = "futures-util" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" dependencies = [ "futures-core", "futures-io", "futures-task", "memchr", + "pin-project", "pin-utils", "slab", ] +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.1.14" @@ -439,6 +552,15 @@ dependencies = [ "wasi", ] +[[package]] +name = "ghash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f0930ed19a7184089ea46d2fedead2f6dc2b674c5db4276b7da336c7cd83252" +dependencies = [ + "polyval", +] + [[package]] name = "glob" version = "0.3.0" @@ -447,9 +569,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5c295d1c0c68e4e42003d75f908f5e16a1edd1cbe0b0d02e4dc2006a384f47" +checksum = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" dependencies = [ "bytes", "fnv", @@ -475,18 +597,38 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.7" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c55f143919fbc0bc77e427fe2d74cf23786d7c1875666f2fde3ac3c659bb67" +checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" dependencies = [ "libc", ] +[[package]] +name = "hkdf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa08a006102488bd9cd5b8013aabe84955cf5ae22e304c2caf655b633aefae3" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "http" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ "bytes", "fnv", @@ -530,9 +672,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.3" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5" +checksum = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" dependencies = [ "bytes", "futures-channel", @@ -544,8 +686,8 @@ dependencies = [ "httparse", "itoa", "log 0.4.8", - "net2", "pin-project", + "socket2", "time", "tokio", "tower-service", @@ -559,7 +701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" dependencies = [ "bytes", - "hyper 0.13.3", + "hyper 0.13.6", "native-tls", "tokio", "tokio-tls", @@ -589,18 +731,18 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" dependencies = [ "autocfg", ] [[package]] name = "inotify" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e40d6fd5d64e2082e0c796495c8ef5ad667a96d03e5aaa0becfd9d47bcbfb8" +checksum = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f" dependencies = [ "bitflags", "inotify-sys", @@ -633,9 +775,9 @@ checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" [[package]] name = "js-sys" -version = "0.3.35" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7889c7c36282151f6bf465be4700359318aef36baa951462382eae49e9577cf9" +checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" dependencies = [ "wasm-bindgen", ] @@ -670,15 +812,15 @@ checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" [[package]] name = "libc" -version = "0.2.66" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" [[package]] name = "lock_api" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" dependencies = [ "scopeguard", ] @@ -709,29 +851,29 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "memchr" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53445de381a1f436797497c61d851644d0e8e88e6140f22872ad33a704933978" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "migrations_internals" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8089920229070f914b9ce9b07ef60e175b2b9bc2d35c3edd8bf4433604e863b9" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" dependencies = [ "diesel", ] [[package]] name = "migrations_macros" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719ef0bc7f531428764c9b70661c14abd50a7f3d21f355752d9985aa21251c9e" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" dependencies = [ "migrations_internals", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] @@ -751,9 +893,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" dependencies = [ "mime 0.3.16", "unicase 2.6.0", @@ -761,18 +903,18 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" dependencies = [ "adler32", ] [[package]] name = "mio" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ "cfg-if", "fuchsia-zircon", @@ -813,9 +955,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" dependencies = [ "lazy_static", "libc", @@ -831,25 +973,15 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ "cfg-if", "libc", "winapi 0.3.8", ] -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - [[package]] name = "notify" version = "4.0.15" @@ -889,14 +1021,26 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", ] +[[package]] +name = "once_cell" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "openssl" version = "0.10.29" @@ -919,9 +1063,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.55" +version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ "autocfg", "cc", @@ -930,11 +1074,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_str_bytes" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06de47b848347d8c4c94219ad8ecd35eb90231704b067e67e6ae2e36ee023510" + [[package]] name = "parking_lot" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ "lock_api", "parking_lot_core", @@ -942,9 +1092,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", "cloudabi", @@ -990,35 +1140,35 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" +checksum = "e75373ff9037d112bb19bc61333a06a159eaeb217660dcfbea7d88e1db823919" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +checksum = "10b4b44893d3c370407a1d6a5cfde7c41ae0478e31c516c85f67eb3adc51be6d" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] name = "pin-project-lite" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" +checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" @@ -1028,15 +1178,25 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" [[package]] name = "podio" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" +checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" + +[[package]] +name = "polyval" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec3341498978de3bfd12d1b22f1af1de22818f5473a11e8a6ef997989e3a212" +dependencies = [ + "cfg-if", + "universal-hash", +] [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "pq-sys" @@ -1049,28 +1209,28 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052b3c9af39c7e5e94245f820530487d19eb285faedcb40e0c3275132293f242" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.8", - "quote 1.0.2", - "rustversion", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", + "version_check 0.9.2", ] [[package]] name = "proc-macro-error-attr" -version = "0.4.9" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d175bef481c7902e63e3165627123fff3502f06ac043d3ef42d08c1246da9253" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "rustversion", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", "syn-mid", + "version_check 0.9.2", ] [[package]] @@ -1084,9 +1244,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.8" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ "unicode-xid 0.2.0", ] @@ -1102,11 +1262,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.2" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2 1.0.18", ] [[package]] @@ -1135,11 +1295,11 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ - "c2-chacha", + "ppv-lite86", "rand_core", ] @@ -1178,18 +1338,18 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" +checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" dependencies = [ - "base64 0.11.0", + "base64 0.12.1", "bytes", "encoding_rs", "futures-core", "futures-util", "http", "http-body", - "hyper 0.13.3", + "hyper 0.13.6", "hyper-tls", "js-sys", "lazy_static", @@ -1202,7 +1362,6 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "time", "tokio", "tokio-tls", "url 2.1.1", @@ -1212,26 +1371,14 @@ dependencies = [ "winreg", ] -[[package]] -name = "ring" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4db68a2e35f3497146b7e4563df7d4773a2433230c5e4b448328e31740458a" -dependencies = [ - "cc", - "lazy_static", - "libc", - "untrusted", -] - [[package]] name = "rocket" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e20afbad214b001cabbe31dd270b48b3be980a7153ee2ed8392e241f856d651b" +checksum = "6130967b369cfb8411b0b73e96fcba1229c32a9cc6f295d144f879bfced13c6e" dependencies = [ "atty", - "base64 0.10.1", + "base64 0.12.1", "log 0.4.8", "memchr", "num_cpus", @@ -1241,30 +1388,30 @@ dependencies = [ "state", "time", "toml", - "version_check 0.9.1", + "version_check 0.9.2", "yansi 0.5.0", ] [[package]] name = "rocket_codegen" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2108b35e2c3a35759d3f16cc3002ece05523191d884d3ad6523693fd43324dde" +checksum = "cb852e6da168fb948a8f2b798ba2e2f0e4fc860eae0efa9cf2bf0f5466bb0425" dependencies = [ "devise", "glob", "indexmap", "quote 0.6.13", "rocket_http", - "version_check 0.9.1", + "version_check 0.9.2", "yansi 0.5.0", ] [[package]] name = "rocket_contrib" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10e7471279bc2d4a21b6fddd9589016bb119e6fbb547b216dd54ef237f28341" +checksum = "e3946ca815127041d8f64455561031d058c22ae1b135251502c5ea523cf9e14b" dependencies = [ "log 0.4.8", "notify", @@ -1275,9 +1422,9 @@ dependencies = [ [[package]] name = "rocket_http" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce8ca76247376ea21cf271af0f95e3f2014596e3e4c7cc04e44ee6242a40ff2" +checksum = "1aff5a5480175f2f553a876b251e9350c74196128806d176da3a51c82aab5428" dependencies = [ "cookie", "hyper 0.10.16", @@ -1290,22 +1437,11 @@ dependencies = [ "unicode-xid 0.1.0", ] -[[package]] -name = "rustversion" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" -dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", -] - [[package]] name = "ryu" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "safemem" @@ -1324,9 +1460,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507a9e6e8ffe0a4e0ebb9a10293e62fdf7657c06f1b8bb07a8fcf697d2abf295" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static", "winapi 0.3.8", @@ -1334,9 +1470,9 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5de7bc31f28f8e6c28df5e1bf3d10610f5fdc14cc95f272853512c70a2bd779" +checksum = "0988d7fdf88d5e5fcf5923a0f1e8ab345f3e98ab4bc6bc45a2d5ff7f7458fbf6" dependencies = [ "parking_lot", ] @@ -1349,10 +1485,11 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ef2429d7cefe5fd28bd1d2ed41c944547d4ff84776f5935b456da44593a16df" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" dependencies = [ + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -1361,38 +1498,39 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "0.3.3" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31493fc37615debb8c5090a7aeb4a9730bc61e77ab10b9af59f1a202284f895" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" dependencies = [ "core-foundation-sys", + "libc", ] [[package]] name = "serde" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] name = "serde_json" -version = "1.0.53" +version = "1.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993948e75b189211a9b31a7528f950c6adc21f9720b6438ff80a7fa2f864cea2" +checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" dependencies = [ "itoa", "ryu", @@ -1411,6 +1549,18 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "sha2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + [[package]] name = "sirene" version = "1.0.3" @@ -1439,15 +1589,21 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" [[package]] -name = "sourcefile" -version = "0.1.4" +name = "socket2" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.8", +] [[package]] name = "state" @@ -1457,9 +1613,21 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "strsim" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + +[[package]] +name = "subtle" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" @@ -1474,12 +1642,12 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.14" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2 1.0.18", + "quote 1.0.7", "unicode-xid 0.2.0", ] @@ -1489,9 +1657,9 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", ] [[package]] @@ -1528,23 +1696,23 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "redox_syscall", "winapi 0.3.8", ] [[package]] name = "tokio" -version = "0.2.11" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fdd17989496f49cdc57978c96f0c9fe5e4a58a8bddc6813c449a4624f6a030b" +checksum = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" dependencies = [ "bytes", "fnv", + "futures-core", "iovec", "lazy_static", "memchr", @@ -1556,9 +1724,9 @@ dependencies = [ [[package]] name = "tokio-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ "native-tls", "tokio", @@ -1566,9 +1734,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes", "futures-core", @@ -1611,6 +1779,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicase" version = "1.4.2" @@ -1626,7 +1800,7 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.1", + "version_check 0.9.2", ] [[package]] @@ -1672,10 +1846,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] -name = "untrusted" -version = "0.6.2" +name = "universal-hash" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cd1f4b4e96b46aeb8d4855db4a7a9bd96eeeb5c6a1ab54593328761642ce2f" +checksum = "df0c900f2f9b4116803415878ff48b63da9edb268668e08cf9292d7503114a01" +dependencies = [ + "generic-array", + "subtle 2.2.3", +] [[package]] name = "url" @@ -1701,15 +1879,15 @@ dependencies = [ [[package]] name = "vcpkg" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -1719,9 +1897,9 @@ checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" [[package]] name = "version_check" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" [[package]] name = "walkdir" @@ -1752,9 +1930,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.58" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5205e9afdf42282b192e2310a5b463a6d1c1d774e30dc3c791ac37ab42d2616c" +checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" dependencies = [ "cfg-if", "serde", @@ -1764,24 +1942,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.58" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11cdb95816290b525b32587d76419facd99662a07e59d3cdb560488a819d9a45" +checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" dependencies = [ "bumpalo", "lazy_static", "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.8" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbdd49e3e28b40dec6a9ba8d17798245ce32b019513a845369c641b275135d9" +checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6" dependencies = [ "cfg-if", "js-sys", @@ -1791,69 +1969,41 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.58" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3" +checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" dependencies = [ - "quote 1.0.2", + "quote 1.0.7", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.58" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668" +checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.31", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.58" +version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e7e61fc929f4c0dddb748b102ebf9f632e2b8d739f2016542b4de2965a9601" - -[[package]] -name = "wasm-bindgen-webidl" -version = "0.2.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef012a0d93fc0432df126a8eaf547b2dce25a8ce9212e1d3cbeef5c11157975d" -dependencies = [ - "anyhow", - "heck", - "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", - "wasm-bindgen-backend", - "weedle", -] +checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" [[package]] name = "web-sys" -version = "0.3.35" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf97caf6aa8c2b1dac90faf0db529d9d63c93846cca4911856f78a83cebf53b" +checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" dependencies = [ - "anyhow", "js-sys", - "sourcefile", "wasm-bindgen", - "wasm-bindgen-webidl", -] - -[[package]] -name = "weedle" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164" -dependencies = [ - "nom", ] [[package]] @@ -1886,9 +2036,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi 0.3.8", ] @@ -1901,9 +2051,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi 0.3.8", ] @@ -1930,6 +2080,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" + [[package]] name = "zip" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index 9a4a5cf..0ccae23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] chrono = { version = "0.4", features = ["serde"] } -clap = { git = "https://github.com/clap-rs/clap.git" } +clap = "3.0.0-beta" custom_error = "1.7" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2", "serde_json", "64-column-tables"] } diesel_migrations = { version = "1.4", features = ["postgres"] } @@ -16,12 +16,12 @@ dotenv = "0.15" openssl = "0.10" r2d2 = "0.8" reqwest = { version = "0.10", features = ["blocking", "json"] } -rocket = "0.4.4" +rocket = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" zip = "0.5" [dependencies.rocket_contrib] -version = "0.4.4" +version = "0.4" default-features = false features = ["json"] diff --git a/src/commands/common.rs b/src/commands/common.rs index cfcef5f..d735ed6 100644 --- a/src/commands/common.rs +++ b/src/commands/common.rs @@ -7,13 +7,11 @@ pub struct FolderOptions { pub db: String, } -arg_enum! { - #[derive(Debug, Deserialize, Clone, Copy)] - pub enum CmdGroupType { - UnitesLegales, - Etablissements, - All - } +#[derive(Clap, Debug, Deserialize, Clone, Copy)] +pub enum CmdGroupType { + UnitesLegales, + Etablissements, + All } impl From for SyntheticGroupType { diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index 8bf1034..bdc082d 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -8,7 +8,7 @@ use std::env; #[derive(Clap, Debug)] pub struct ServeFlags { /// Production, Staging or Development, will change log level, you can set in environment variable as SIRENE_ENV - #[clap(long = "env")] + #[clap(arg_enum, long = "env")] environment: Option, /// Listen this port, you can set in environment variable as PORT @@ -24,13 +24,11 @@ pub struct ServeFlags { api_key: Option, } -arg_enum! { - #[derive(Debug)] - enum CmdEnvironment { - Development, - Staging, - Production - } +#[derive(Clap, Debug)] +enum CmdEnvironment { + Development, + Staging, + Production } impl From for Environment { diff --git a/src/commands/update.rs b/src/commands/update.rs index 20407b3..c950781 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -6,6 +6,7 @@ use crate::update::{common::Config, update, update_step}; #[derive(Clap, Debug)] pub struct UpdateFlags { /// Configure which part will be updated, UnitesLegales, Etablissements or All + #[clap(arg_enum)] group_type: CmdGroupType, /// Force update even if the source data where not updated From 770f0ed2aebdd79d0c8340ba55ec0d65d766c846 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 11 Jun 2020 16:40:02 +0200 Subject: [PATCH 32/69] Use generics in daily updates --- src/connectors/insee/implementation.rs | 113 +++++++++++++++---------- src/connectors/insee/types.rs | 14 ++- 2 files changed, 81 insertions(+), 46 deletions(-) diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index 3c5544f..8abdd29 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -1,5 +1,5 @@ use super::error::InseeUpdateError; -use super::types::InseeUniteLegaleResponse; +use super::types::{InseeResponse, InseeUniteLegaleResponse}; use super::Connector; use crate::models::unite_legale::common::UniteLegale; use chrono::{Duration, NaiveDateTime, Utc}; @@ -8,61 +8,37 @@ use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; pub const INITIAL_CURSOR: &str = "*"; -impl Connector { - fn get_minimum_timestamp_for_request(&self, timestamp: NaiveDateTime) -> NaiveDateTime { - timestamp.max(Utc::now().naive_local() - Duration::days(31)) - } +struct EndpointConfig { + token: String, + route: String, + query_field: String, +} +impl Connector { pub fn get_daily_unites_legales( &self, start_timestamp: NaiveDateTime, cursor: String, ) -> Result<(Option, Vec), InseeUpdateError> { - let client = reqwest::blocking::Client::new(); - - let url = format!( - "{}/siren?q=dateDernierTraitementUniteLegale:%7B{} TO *]&nombre=1000&curseur={}", - BASE_URL, - self.get_minimum_timestamp_for_request(start_timestamp) - .format("%Y-%m-%dT%H:%M:%S"), - cursor - ); - - let response = match client - .get(&url) - .header(AUTHORIZATION, format!("Bearer {}", self.token)) - .header(ACCEPT, HeaderValue::from_static("application/json")) - .send()? - .error_for_status() - { - Ok(response) => response - .json::() - .map_err(|error| error.into()), - Err(error) => { - // Insee returns 404 for empty data - if let Some(status) = error.status() { - if status == reqwest::StatusCode::NOT_FOUND { - return Ok((None, vec![])); - } - } - - Err(error) - } - }?; - - let next_cursor = if response.header.curseur == response.header.curseur_suivant { - None - } else { - Some(response.header.curseur_suivant) - }; + let (next_cursor, response) = get_daily_data::( + EndpointConfig { + token: self.token.clone(), + route: String::from("siren"), + query_field: String::from("dateDernierTraitementUniteLegale"), + }, + start_timestamp, + cursor, + )?; Ok(( next_cursor, - response - .unites_legales + match response { + Some(resp) => resp.unites_legales .iter() .filter_map(|u| u.into()) .collect(), + None => vec![] + } )) } @@ -70,3 +46,52 @@ impl Connector { Ok(String::from("Etablissement")) } } + +fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime { + timestamp.max(Utc::now().naive_local() - Duration::days(31)) +} + +fn get_daily_data( + config: EndpointConfig, + start_timestamp: NaiveDateTime, + cursor: String, +) -> Result<(Option, Option), InseeUpdateError> { + let client = reqwest::blocking::Client::new(); + + let url = format!( + "{}/{}?q={}:%7B{} TO *]&nombre=1000&curseur={}", + BASE_URL, + config.route, + config.query_field, + get_minimum_timestamp_for_request(start_timestamp).format("%Y-%m-%dT%H:%M:%S"), + cursor + ); + + let response = match client + .get(&url) + .header(AUTHORIZATION, format!("Bearer {}", config.token)) + .header(ACCEPT, HeaderValue::from_static("application/json")) + .send()? + .error_for_status() + { + Ok(response) => response.json::().map_err(|error| error.into()), + Err(error) => { + // Insee returns 404 for empty data + if let Some(status) = error.status() { + if status == reqwest::StatusCode::NOT_FOUND { + return Ok((None, None)); + } + } + + Err(error) + } + }?; + + let next_cursor = if response.header().curseur == response.header().curseur_suivant { + None + } else { + Some(response.header().curseur_suivant) + }; + + Ok((next_cursor, Some(response))) +} diff --git a/src/connectors/insee/types.rs b/src/connectors/insee/types.rs index f92778a..a980f3b 100644 --- a/src/connectors/insee/types.rs +++ b/src/connectors/insee/types.rs @@ -1,6 +1,6 @@ use crate::models::unite_legale::common::UniteLegale; use chrono::{NaiveDate, NaiveDateTime}; -use serde::Deserialize; +use serde::{de::DeserializeOwned, Deserialize}; use std::fmt::Display; use std::str::FromStr; @@ -8,6 +8,10 @@ fn default_as_false() -> bool { false } +pub trait InseeResponse: DeserializeOwned { + fn header(&self) -> Header; +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct InseeUniteLegaleResponse { @@ -15,7 +19,13 @@ pub struct InseeUniteLegaleResponse { pub unites_legales: Vec, } -#[derive(Deserialize, Debug)] +impl InseeResponse for InseeUniteLegaleResponse { + fn header(&self) -> Header { + self.header.clone() + } +} + +#[derive(Clone, Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct Header { total: u32, From 9e9fcfc2270e64165ea086f3d5a1a44526676503 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Fri, 12 Jun 2020 15:03:38 +0200 Subject: [PATCH 33/69] Updates "Etablissement" from INSEE (#138) --- .github/workflows/rust.yml | 7 +- Cargo.lock | 30 ++- README.md | 4 +- src/commands/serve/mod.rs | 4 +- src/commands/update.rs | 2 +- src/connectors/insee/implementation.rs | 52 ++++- src/connectors/insee/types/etablissement.rs | 201 ++++++++++++++++++ src/connectors/insee/types/mod.rs | 44 ++++ .../insee/{types.rs => types/unite_legale.rs} | 49 +---- src/models/etablissement/common.rs | 4 +- src/models/etablissement/mod.rs | 87 +++++++- 11 files changed, 395 insertions(+), 89 deletions(-) create mode 100644 src/connectors/insee/types/etablissement.rs create mode 100644 src/connectors/insee/types/mod.rs rename src/connectors/insee/{types.rs => types/unite_legale.rs} (84%) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9ab22a5..8537566 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - name: Install minimal nightly uses: actions-rs/toolchain@v1 with: @@ -17,9 +17,10 @@ jobs: override: true - name: Build uses: actions-rs/cargo@v1 - with: + with: command: build + args: --release - name: Run tests uses: actions-rs/cargo@v1 - with: + with: command: test diff --git a/Cargo.lock b/Cargo.lock index d67484a..33062fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,9 +1002,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ "autocfg", "num-traits", @@ -1012,9 +1012,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ "autocfg", ] @@ -1106,24 +1106,24 @@ dependencies = [ [[package]] name = "pear" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26d2b92e47063ffce70d3e3b1bd097af121a9e0db07ca38a6cc1cf0cc85ff25" +checksum = "5320f212db967792b67cfe12bd469d08afd6318a249bd917d5c19bc92200ab8a" dependencies = [ "pear_codegen", ] [[package]] name = "pear_codegen" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336db4a192cc7f54efeb0c4e11a9245394824cc3bcbd37ba3ff51240c35d7a6e" +checksum = "bfc1c836fdc3d1ef87c348b237b5b5c4dff922156fb2d968f57734f9669768ca" dependencies = [ "proc-macro2 0.4.30", "quote 0.6.13", "syn 0.15.44", - "version_check 0.1.5", - "yansi 0.4.0", + "version_check 0.9.2", + "yansi", ] [[package]] @@ -1389,7 +1389,7 @@ dependencies = [ "time", "toml", "version_check 0.9.2", - "yansi 0.5.0", + "yansi", ] [[package]] @@ -1404,7 +1404,7 @@ dependencies = [ "quote 0.6.13", "rocket_http", "version_check 0.9.2", - "yansi 0.5.0", + "yansi", ] [[package]] @@ -2068,12 +2068,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "yansi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60c3b48c9cdec42fb06b3b84b5b087405e1fa1c644a1af3930e4dfafe93de48" - [[package]] name = "yansi" version = "0.5.0" diff --git a/README.md b/README.md index 1240cf3..0944a15 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ FLAGS: OPTIONS: -k, --api-key API key needed to allow maintenance operation from HTTP, you can set in environment variable as API_KEY - --env Production, Staging or Development, will change log level, you can set in environment + --env production, staging or development, will change log level, you can set in environment variable as SIRENE_ENV -h, --host Listen this host, you can set in environment variable as HOST -p, --port Listen this port, you can set in environment variable as PORT @@ -82,7 +82,7 @@ USAGE: sirene update [FLAGS] [SUBCOMMAND] ARGS: - Configure which part will be updated, UnitesLegales, Etablissements or All + Configure which part will be updated, unites-legales, etablissements or all FLAGS: --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete it diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index bdc082d..c97ed63 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -7,7 +7,7 @@ use std::env; #[derive(Clap, Debug)] pub struct ServeFlags { - /// Production, Staging or Development, will change log level, you can set in environment variable as SIRENE_ENV + /// production, staging or development, will change log level, you can set in environment variable as SIRENE_ENV #[clap(arg_enum, long = "env")] environment: Option, @@ -28,7 +28,7 @@ pub struct ServeFlags { enum CmdEnvironment { Development, Staging, - Production + Production, } impl From for Environment { diff --git a/src/commands/update.rs b/src/commands/update.rs index c950781..02695d5 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -5,7 +5,7 @@ use crate::update::{common::Config, update, update_step}; #[derive(Clap, Debug)] pub struct UpdateFlags { - /// Configure which part will be updated, UnitesLegales, Etablissements or All + /// Configure which part will be updated, unites-legales, etablissements or all #[clap(arg_enum)] group_type: CmdGroupType, diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index 8abdd29..4b0bac0 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -1,6 +1,10 @@ use super::error::InseeUpdateError; -use super::types::{InseeResponse, InseeUniteLegaleResponse}; +use super::types::{ + etablissement::InseeEtablissementResponse, unite_legale::InseeUniteLegaleResponse, + InseeResponse, +}; use super::Connector; +use crate::models::etablissement::common::Etablissement; use crate::models::unite_legale::common::UniteLegale; use chrono::{Duration, NaiveDateTime, Utc}; use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; @@ -33,17 +37,42 @@ impl Connector { Ok(( next_cursor, match response { - Some(resp) => resp.unites_legales - .iter() - .filter_map(|u| u.into()) - .collect(), - None => vec![] - } + Some(resp) => resp + .unites_legales + .iter() + .filter_map(|u| u.into()) + .collect(), + None => vec![], + }, )) } - pub fn get_daily_etablissements(&self) -> Result { - Ok(String::from("Etablissement")) + pub fn get_daily_etablissements( + &self, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, Vec), InseeUpdateError> { + let (next_cursor, response) = get_daily_data::( + EndpointConfig { + token: self.token.clone(), + route: String::from("siret"), + query_field: String::from("dateDernierTraitementEtablissement"), + }, + start_timestamp, + cursor, + )?; + + Ok(( + next_cursor, + match response { + Some(resp) => resp + .etablissements + .iter() + .filter_map(|u| u.into()) + .collect(), + None => vec![], + }, + )) } } @@ -87,10 +116,11 @@ fn get_daily_data( } }?; - let next_cursor = if response.header().curseur == response.header().curseur_suivant { + let header = response.header(); + let next_cursor = if header.curseur == header.curseur_suivant { None } else { - Some(response.header().curseur_suivant) + Some(header.curseur_suivant) }; Ok((next_cursor, Some(response))) diff --git a/src/connectors/insee/types/etablissement.rs b/src/connectors/insee/types/etablissement.rs new file mode 100644 index 0000000..8633e70 --- /dev/null +++ b/src/connectors/insee/types/etablissement.rs @@ -0,0 +1,201 @@ +use super::{Header, InseeResponse}; +use crate::models::etablissement::common::Etablissement; +use chrono::{NaiveDate, NaiveDateTime}; +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissementResponse { + pub header: Header, + pub etablissements: Vec, +} + +impl InseeResponse for InseeEtablissementResponse { + fn header(&self) -> Header { + self.header.clone() + } +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissementInner { + pub siren: String, + pub nic: String, + pub siret: String, + pub statut_diffusion_etablissement: String, + pub date_creation_etablissement: Option, + pub tranche_effectifs_etablissement: Option, + #[serde(deserialize_with = "super::from_str_optional")] + pub annee_effectifs_etablissement: Option, + pub activite_principale_registre_metiers_etablissement: Option, + pub date_dernier_traitement_etablissement: Option, + pub etablissement_siege: bool, + pub nombre_periodes_etablissement: Option, + pub adresse_etablissement: InseeAdresseEtablissement, + pub adresse2_etablissement: InseeAdresse2Etablissement, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissement { + #[serde(flatten)] + pub content: InseeEtablissementInner, + pub periodes_etablissement: Vec, +} + +#[derive(Debug)] +pub struct InseeEtablissementWithPeriode { + pub content: InseeEtablissementInner, + pub periode: InseePeriodeEtablissement, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeAdresseEtablissement { + pub complement_adresse_etablissement: Option, + pub numero_voie_etablissement: Option, + pub indice_repetition_etablissement: Option, + pub type_voie_etablissement: Option, + pub libelle_voie_etablissement: Option, + pub code_postal_etablissement: Option, + pub libelle_commune_etablissement: Option, + pub libelle_commune_etranger_etablissement: Option, + pub distribution_speciale_etablissement: Option, + pub code_commune_etablissement: Option, + pub code_cedex_etablissement: Option, + pub libelle_cedex_etablissement: Option, + pub code_pays_etranger_etablissement: Option, + pub libelle_pays_etranger_etablissement: Option, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeAdresse2Etablissement { + complement_adresse2_etablissement: Option, + numero_voie2_etablissement: Option, + indice_repetition2_etablissement: Option, + type_voie2_etablissement: Option, + libelle_voie2_etablissement: Option, + code_postal2_etablissement: Option, + libelle_commune2_etablissement: Option, + libelle_commune_etranger2_etablissement: Option, + distribution_speciale2_etablissement: Option, + code_commune2_etablissement: Option, + code_cedex2_etablissement: Option, + libelle_cedex2_etablissement: Option, + code_pays_etranger2_etablissement: Option, + libelle_pays_etranger2_etablissement: Option, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseePeriodeEtablissement { + date_fin: Option, + date_debut: Option, + #[serde(deserialize_with = "deserialize_etat_administratif")] + etat_administratif_etablissement: String, + changement_etat_administratif_etablissement: bool, + enseigne1_etablissement: Option, + enseigne2_etablissement: Option, + enseigne3_etablissement: Option, + changement_enseigne_etablissement: bool, + denomination_usuelle_etablissement: Option, + changement_denomination_usuelle_etablissement: bool, + activite_principale_etablissement: Option, + nomenclature_activite_principale_etablissement: Option, + changement_activite_principale_etablissement: bool, + caractere_employeur_etablissement: Option, + changement_caractere_employeur_etablissement: bool, +} + +impl From<&InseeEtablissement> for Option { + fn from(u: &InseeEtablissement) -> Self { + match u + .periodes_etablissement + .iter() + .find(|p| p.date_fin.is_none()) + { + Some(periode) => { + // Convert + Some( + InseeEtablissementWithPeriode { + content: u.content.clone(), + periode: periode.clone(), + } + .into(), + ) + } + None => None, + } + } +} + +impl From for Etablissement { + fn from(e: InseeEtablissementWithPeriode) -> Self { + let adresse = e.content.adresse_etablissement; + let adresse2 = e.content.adresse2_etablissement; + + Etablissement { + siret: e.content.siret, + siren: e.content.siren, + nic: e.content.nic, + statut_diffusion: e.content.statut_diffusion_etablissement, + date_creation: e.content.date_creation_etablissement, + tranche_effectifs: e.content.tranche_effectifs_etablissement, + annee_effectifs: e.content.annee_effectifs_etablissement, + activite_principale_registre_metiers: e + .content + .activite_principale_registre_metiers_etablissement, + date_dernier_traitement: e.content.date_dernier_traitement_etablissement, + etablissement_siege: e.content.etablissement_siege, + nombre_periodes: e.content.nombre_periodes_etablissement, + complement_adresse: adresse.complement_adresse_etablissement, + numero_voie: adresse.numero_voie_etablissement, + indice_repetition: adresse.indice_repetition_etablissement, + type_voie: adresse.type_voie_etablissement, + libelle_voie: adresse.libelle_voie_etablissement, + code_postal: adresse.code_postal_etablissement, + libelle_commune: adresse.libelle_commune_etablissement, + libelle_commune_etranger: adresse.libelle_commune_etranger_etablissement, + distribution_speciale: adresse.distribution_speciale_etablissement, + code_commune: adresse.code_commune_etablissement, + code_cedex: adresse.code_cedex_etablissement, + libelle_cedex: adresse.libelle_cedex_etablissement, + code_pays_etranger: adresse.code_pays_etranger_etablissement, + libelle_pays_etranger: adresse.libelle_pays_etranger_etablissement, + complement_adresse2: adresse2.complement_adresse2_etablissement, + numero_voie_2: adresse2.numero_voie2_etablissement, + indice_repetition_2: adresse2.indice_repetition2_etablissement, + type_voie_2: adresse2.type_voie2_etablissement, + libelle_voie_2: adresse2.libelle_voie2_etablissement, + code_postal_2: adresse2.code_postal2_etablissement, + libelle_commune_2: adresse2.libelle_commune2_etablissement, + libelle_commune_etranger_2: adresse2.libelle_commune_etranger2_etablissement, + distribution_speciale_2: adresse2.distribution_speciale2_etablissement, + code_commune_2: adresse2.code_commune2_etablissement, + code_cedex_2: adresse2.code_cedex2_etablissement, + libelle_cedex_2: adresse2.libelle_cedex2_etablissement, + code_pays_etranger_2: adresse2.code_pays_etranger2_etablissement, + libelle_pays_etranger_2: adresse2.libelle_pays_etranger2_etablissement, + date_debut: e.periode.date_debut, + etat_administratif: e.periode.etat_administratif_etablissement, + enseigne_1: e.periode.enseigne1_etablissement, + enseigne_2: e.periode.enseigne2_etablissement, + enseigne_3: e.periode.enseigne3_etablissement, + denomination_usuelle: e.periode.denomination_usuelle_etablissement, + activite_principale: e.periode.activite_principale_etablissement, + nomenclature_activite_principale: e + .periode + .nomenclature_activite_principale_etablissement, + caractere_employeur: e.periode.caractere_employeur_etablissement, + } + } +} + +fn deserialize_etat_administratif<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let opt = Option::deserialize(deserializer)?; + Ok(opt.unwrap_or(String::from("F"))) +} diff --git a/src/connectors/insee/types/mod.rs b/src/connectors/insee/types/mod.rs new file mode 100644 index 0000000..8fc3bfb --- /dev/null +++ b/src/connectors/insee/types/mod.rs @@ -0,0 +1,44 @@ +pub mod etablissement; +pub mod unite_legale; + +use serde::{de::DeserializeOwned, Deserialize}; +use std::fmt::Display; +use std::str::FromStr; + +pub trait InseeResponse: DeserializeOwned { + fn header(&self) -> Header; +} + +#[derive(Clone, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Header { + total: u32, + debut: u32, + nombre: u32, + pub curseur: String, + pub curseur_suivant: String, +} + +fn from_str_optional<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: FromStr, + T::Err: Display, + D: serde::Deserializer<'de>, +{ + let deser_res: Result = serde::Deserialize::deserialize(deserializer); + match deser_res { + Ok(serde_json::Value::String(s)) => T::from_str(&s) + .map_err(serde::de::Error::custom) + .map(Option::from), + Ok(serde_json::Value::Null) => Ok(None), + Ok(v) => { + println!("string expected but found something else: {}", v); + Ok(None) + } + Err(_) => Ok(None), + } +} + +fn default_as_false() -> bool { + false +} diff --git a/src/connectors/insee/types.rs b/src/connectors/insee/types/unite_legale.rs similarity index 84% rename from src/connectors/insee/types.rs rename to src/connectors/insee/types/unite_legale.rs index a980f3b..dea7da0 100644 --- a/src/connectors/insee/types.rs +++ b/src/connectors/insee/types/unite_legale.rs @@ -1,16 +1,7 @@ +use super::{Header, InseeResponse}; use crate::models::unite_legale::common::UniteLegale; use chrono::{NaiveDate, NaiveDateTime}; -use serde::{de::DeserializeOwned, Deserialize}; -use std::fmt::Display; -use std::str::FromStr; - -fn default_as_false() -> bool { - false -} - -pub trait InseeResponse: DeserializeOwned { - fn header(&self) -> Header; -} +use serde::Deserialize; #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] @@ -25,23 +16,13 @@ impl InseeResponse for InseeUniteLegaleResponse { } } -#[derive(Clone, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Header { - total: u32, - debut: u32, - nombre: u32, - pub curseur: String, - pub curseur_suivant: String, -} - #[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct InseeUniteLegaleInner { pub siren: String, pub statut_diffusion_unite_legale: String, - #[serde(default = "default_as_false")] + #[serde(default = "super::default_as_false")] pub unite_purgee_unite_legale: bool, pub date_creation_unite_legale: Option, @@ -56,14 +37,14 @@ pub struct InseeUniteLegaleInner { pub identifiant_association_unite_legale: Option, pub tranche_effectifs_unite_legale: Option, - #[serde(deserialize_with = "from_str_optional")] + #[serde(deserialize_with = "super::from_str_optional")] pub annee_effectifs_unite_legale: Option, pub date_dernier_traitement_unite_legale: Option, pub nombre_periodes_unite_legale: Option, pub categorie_entreprise: Option, - #[serde(deserialize_with = "from_str_optional")] + #[serde(deserialize_with = "super::from_str_optional")] pub annee_categorie_entreprise: Option, } @@ -178,26 +159,6 @@ impl From for UniteLegale { } } -fn from_str_optional<'de, T, D>(deserializer: D) -> Result, D::Error> -where - T: FromStr, - T::Err: Display, - D: serde::Deserializer<'de>, -{ - let deser_res: Result = serde::Deserialize::deserialize(deserializer); - match deser_res { - Ok(serde_json::Value::String(s)) => T::from_str(&s) - .map_err(serde::de::Error::custom) - .map(Option::from), - Ok(serde_json::Value::Null) => Ok(None), - Ok(v) => { - println!("string expected but found something else: {}", v); - Ok(None) - } - Err(_) => Ok(None), - } -} - fn deserialize_etat_administratif<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, diff --git a/src/models/etablissement/common.rs b/src/models/etablissement/common.rs index ce77aff..9033126 100644 --- a/src/models/etablissement/common.rs +++ b/src/models/etablissement/common.rs @@ -1,7 +1,9 @@ +use super::super::schema::etablissement; use chrono::{NaiveDate, NaiveDateTime}; use serde::Serialize; -#[derive(Queryable, Serialize, Clone, Debug)] +#[derive(Insertable, Queryable, Serialize, Clone, Debug)] +#[table_name = "etablissement"] pub struct Etablissement { pub siret: String, pub siren: String, diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 0e13434..7ae578c 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -4,9 +4,10 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; -use crate::connectors::Connectors; +use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; use chrono::NaiveDateTime; use common::Etablissement; +use diesel::pg::upsert::excluded; use diesel::prelude::*; use diesel::sql_query; use error::Error; @@ -114,24 +115,96 @@ impl UpdatableModel for EtablissementModel { // SELECT date_dernier_traitement FROM etablissement WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; fn get_last_insee_synced_timestamp( &self, - _connectors: &Connectors, + connectors: &Connectors, ) -> Result, UpdatableError> { - Ok(None) + let connection = connectors.local.pool.get()?; + dsl::etablissement + .select(dsl::date_dernier_traitement) + .order(dsl::date_dernier_traitement.desc()) + .filter(dsl::date_dernier_traitement.is_not_null()) + .first::>(&connection) + .map_err(|error| error.into()) } fn update_daily_data( &self, connectors: &Connectors, - _start_timestamp: NaiveDateTime, + start_timestamp: NaiveDateTime, ) -> Result { let insee = connectors .insee .as_ref() .ok_or(UpdatableError::MissingInseeConnector)?; - let result = insee.get_daily_etablissements()?; + let connection = connectors.local.pool.get()?; + + let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); + let mut updated_count: usize = 0; + + while let Some(cursor) = current_cursor { + let (next_cursor, etablissements) = + insee.get_daily_etablissements(start_timestamp, cursor)?; + + current_cursor = next_cursor; + + updated_count += diesel::insert_into(dsl::etablissement) + .values(&etablissements) + .on_conflict(dsl::siret) + .do_update() + .set(( + dsl::nic.eq(excluded(dsl::nic)), + dsl::siren.eq(excluded(dsl::siren)), + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::activite_principale_registre_metiers + .eq(excluded(dsl::activite_principale_registre_metiers)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::etablissement_siege.eq(excluded(dsl::etablissement_siege)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::complement_adresse.eq(excluded(dsl::complement_adresse)), + dsl::numero_voie.eq(excluded(dsl::numero_voie)), + dsl::indice_repetition.eq(excluded(dsl::indice_repetition)), + dsl::type_voie.eq(excluded(dsl::type_voie)), + dsl::libelle_voie.eq(excluded(dsl::libelle_voie)), + dsl::code_postal.eq(excluded(dsl::code_postal)), + dsl::libelle_commune.eq(excluded(dsl::libelle_commune)), + dsl::libelle_commune_etranger.eq(excluded(dsl::libelle_commune_etranger)), + dsl::distribution_speciale.eq(excluded(dsl::distribution_speciale)), + dsl::code_commune.eq(excluded(dsl::code_commune)), + dsl::code_cedex.eq(excluded(dsl::code_cedex)), + dsl::libelle_cedex.eq(excluded(dsl::libelle_cedex)), + dsl::code_pays_etranger.eq(excluded(dsl::code_pays_etranger)), + dsl::libelle_pays_etranger.eq(excluded(dsl::libelle_pays_etranger)), + dsl::complement_adresse2.eq(excluded(dsl::complement_adresse2)), + dsl::numero_voie_2.eq(excluded(dsl::numero_voie_2)), + dsl::indice_repetition_2.eq(excluded(dsl::indice_repetition_2)), + dsl::type_voie_2.eq(excluded(dsl::type_voie_2)), + dsl::libelle_voie_2.eq(excluded(dsl::libelle_voie_2)), + dsl::code_postal_2.eq(excluded(dsl::code_postal_2)), + dsl::libelle_commune_2.eq(excluded(dsl::libelle_commune_2)), + dsl::libelle_commune_etranger_2.eq(excluded(dsl::libelle_commune_etranger_2)), + dsl::distribution_speciale_2.eq(excluded(dsl::distribution_speciale_2)), + dsl::code_commune_2.eq(excluded(dsl::code_commune_2)), + dsl::code_cedex_2.eq(excluded(dsl::code_cedex_2)), + dsl::libelle_cedex_2.eq(excluded(dsl::libelle_cedex_2)), + dsl::code_pays_etranger_2.eq(excluded(dsl::code_pays_etranger_2)), + dsl::libelle_pays_etranger_2.eq(excluded(dsl::libelle_pays_etranger_2)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::enseigne_1.eq(excluded(dsl::enseigne_1)), + dsl::enseigne_2.eq(excluded(dsl::enseigne_2)), + dsl::enseigne_3.eq(excluded(dsl::enseigne_3)), + dsl::denomination_usuelle.eq(excluded(dsl::denomination_usuelle)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; + } - println!("{:#?}", result); - Ok(0) + Ok(updated_count) } } From 76e2d8e779329398e8cd6c725a22a70a86ba1907 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Wed, 17 Jun 2020 16:22:04 +0200 Subject: [PATCH 34/69] Updates api server from Rocket to Warp (#141) --- Cargo.lock | 1024 +++++++++++++-------------- Cargo.toml | 10 +- src/commands/common.rs | 3 +- src/commands/mod.rs | 4 +- src/commands/serve/mod.rs | 36 +- src/commands/serve/runner/common.rs | 1 + src/commands/serve/runner/error.rs | 125 +++- src/commands/serve/runner/mod.rs | 132 ++-- src/connectors/insee/mod.rs | 1 + src/connectors/local.rs | 1 + src/connectors/mod.rs | 1 + src/main.rs | 12 +- 12 files changed, 698 insertions(+), 652 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33062fa..9337a09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,58 +7,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" [[package]] -name = "aead" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf01b9b56e767bb57b94ebf91a58b338002963785cdd7013e21c0d4679471e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54eb1d8fe354e5fc611daf4f2ea97dd45a765f4f1e4512306ec183ae2e8f20c9" -dependencies = [ - "aes-soft", - "aesni", - "block-cipher-trait", -] - -[[package]] -name = "aes-gcm" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "834a6bda386024dbb7c8fc51322856c10ffe69559f972261c868485f5759c638" -dependencies = [ - "aead", - "aes", - "block-cipher-trait", - "ghash", - "subtle 2.2.3", - "zeroize", -] - -[[package]] -name = "aes-soft" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" -dependencies = [ - "block-cipher-trait", - "byteorder", - "opaque-debug", -] - -[[package]] -name = "aesni" -version = "0.6.0" +name = "aho-corasick" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" dependencies = [ - "block-cipher-trait", - "opaque-debug", + "memchr", ] [[package]] @@ -72,6 +26,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.0" @@ -80,13 +40,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "base64" -version = "0.9.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" -dependencies = [ - "byteorder", - "safemem", -] +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "base64" @@ -113,21 +69,22 @@ dependencies = [ ] [[package]] -name = "block-cipher-trait" -version = "0.6.2" +name = "block-padding" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" dependencies = [ - "generic-array", + "byte-tools", ] [[package]] -name = "block-padding" -version = "0.1.5" +name = "buf_redux" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" dependencies = [ - "byte-tools", + "memchr", + "safemem", ] [[package]] @@ -226,9 +183,9 @@ checksum = "fb51c9e75b94452505acd21d929323f5a5c6c4735a852adbd39ef5fb1b014f30" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -240,22 +197,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "cookie" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5795cda0897252e34380a27baf884c53aa7ad9990329cdad96d4c5d027015d44" -dependencies = [ - "aes-gcm", - "base64 0.12.1", - "hkdf", - "hmac", - "percent-encoding 2.1.0", - "rand", - "sha2", - "time", -] - [[package]] name = "core-foundation" version = "0.7.0" @@ -281,54 +222,12 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crypto-mac" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" -dependencies = [ - "generic-array", - "subtle 1.0.0", -] - [[package]] name = "custom_error" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93a0fc65739ae998afc8d68e64bdac2efd1bc4ffa1a0703d171ef2defae3792f" -[[package]] -name = "devise" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74e04ba2d03c5fa0d954c061fc8c9c288badadffc272ebb87679a89846de3ed3" -dependencies = [ - "devise_codegen", - "devise_core", -] - -[[package]] -name = "devise_codegen" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066ceb7928ca93a9bedc6d0e612a8a0424048b0ab1f75971b203d01420c055d7" -dependencies = [ - "devise_core", - "quote 0.6.13", -] - -[[package]] -name = "devise_core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf41c59b22b5e3ec0ea55c7847e5f358d340f3a8d6d53a5cf4f1564967f96487" -dependencies = [ - "bitflags", - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", -] - [[package]] name = "diesel" version = "1.4.5" @@ -350,9 +249,9 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -396,22 +295,23 @@ dependencies = [ ] [[package]] -name = "fake-simd" -version = "0.1.2" +name = "env_logger" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log 0.4.8", + "regex", + "termcolor", +] [[package]] -name = "filetime" -version = "0.2.10" +name = "fake-simd" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affc17579b132fc2461adf7c575cc6e8b134ebca52c51f5411388965227dc695" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "winapi 0.3.8", -] +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" @@ -447,23 +347,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] -name = "fsevent" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6" -dependencies = [ - "bitflags", - "fsevent-sys", -] - -[[package]] -name = "fsevent-sys" -version = "2.0.1" +name = "fuchsia-cprng" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41b048a94555da0f42f1d632e2e19510084fb8e303b0daa2816e733fb3644a0" -dependencies = [ - "libc", -] +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "fuchsia-zircon" @@ -481,6 +368,21 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futures" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.5" @@ -488,6 +390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -496,12 +399,35 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +[[package]] +name = "futures-executor" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.5" @@ -523,12 +449,17 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project", "pin-utils", + "proc-macro-hack", + "proc-macro-nested", "slab", ] @@ -552,21 +483,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "ghash" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f0930ed19a7184089ea46d2fedead2f6dc2b674c5db4276b7da336c7cd83252" -dependencies = [ - "polyval", -] - -[[package]] -name = "glob" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" - [[package]] name = "h2" version = "0.2.5" @@ -587,41 +503,46 @@ dependencies = [ ] [[package]] -name = "heck" -version = "0.3.1" +name = "headers" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" dependencies = [ - "unicode-segmentation", + "base64 0.12.1", + "bitflags", + "bytes", + "headers-core", + "http", + "mime 0.3.16", + "sha-1", + "time", ] [[package]] -name = "hermit-abi" -version = "0.1.13" +name = "headers-core" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "libc", + "http", ] [[package]] -name = "hkdf" -version = "0.8.0" +name = "heck" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fa08a006102488bd9cd5b8013aabe84955cf5ae22e304c2caf655b633aefae3" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" dependencies = [ - "digest", - "hmac", + "unicode-segmentation", ] [[package]] -name = "hmac" -version = "0.7.1" +name = "hermit-abi" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" dependencies = [ - "crypto-mac", - "digest", + "libc", ] [[package]] @@ -652,22 +573,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] -name = "hyper" -version = "0.10.16" +name = "humantime" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ - "base64 0.9.3", - "httparse", - "language-tags", - "log 0.3.9", - "mime 0.2.6", - "num_cpus", - "time", - "traitobject", - "typeable", - "unicase 1.4.2", - "url 1.7.2", + "quick-error", ] [[package]] @@ -701,23 +612,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" dependencies = [ "bytes", - "hyper 0.13.6", + "hyper", "native-tls", "tokio", "tokio-tls", ] -[[package]] -name = "idna" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.2.0" @@ -735,27 +635,16 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" dependencies = [ - "autocfg", -] - -[[package]] -name = "inotify" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4816c66d2c8ae673df83366c18341538f234a26d65a9ecea5c348b453ac1d02f" -dependencies = [ - "bitflags", - "inotify-sys", - "libc", + "autocfg 1.0.0", ] [[package]] -name = "inotify-sys" -version = "0.1.3" +name = "input_buffer" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0" +checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" dependencies = [ - "libc", + "bytes", ] [[package]] @@ -792,24 +681,12 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" - [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" - [[package]] name = "libc" version = "0.2.71" @@ -871,9 +748,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" dependencies = [ "migrations_internals", - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -891,6 +768,18 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "mime_guess" +version = "1.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" +dependencies = [ + "mime 0.2.6", + "phf", + "phf_codegen", + "unicase 1.4.2", +] + [[package]] name = "mime_guess" version = "2.0.3" @@ -929,18 +818,6 @@ dependencies = [ "winapi 0.2.8", ] -[[package]] -name = "mio-extras" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" -dependencies = [ - "lazycell", - "log 0.4.8", - "mio", - "slab", -] - [[package]] name = "miow" version = "0.2.1" @@ -953,6 +830,24 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "multipart" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" +dependencies = [ + "buf_redux", + "httparse", + "log 0.4.8", + "mime 0.2.6", + "mime_guess 1.8.8", + "quick-error", + "rand 0.6.5", + "safemem", + "tempfile", + "twoway", +] + [[package]] name = "native-tls" version = "0.2.4" @@ -982,31 +877,13 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "notify" -version = "4.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" -dependencies = [ - "bitflags", - "filetime", - "fsevent", - "fsevent-sys", - "inotify", - "libc", - "mio", - "mio-extras", - "walkdir", - "winapi 0.3.8", -] - [[package]] name = "num-integer" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ - "autocfg", + "autocfg 1.0.0", "num-traits", ] @@ -1016,7 +893,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ - "autocfg", + "autocfg 1.0.0", ] [[package]] @@ -1067,7 +944,7 @@ version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ - "autocfg", + "autocfg 1.0.0", "cc", "libc", "pkg-config", @@ -1105,38 +982,49 @@ dependencies = [ ] [[package]] -name = "pear" -version = "0.1.4" +name = "percent-encoding" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5320f212db967792b67cfe12bd469d08afd6318a249bd917d5c19bc92200ab8a" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" dependencies = [ - "pear_codegen", + "phf_shared", ] [[package]] -name = "pear_codegen" -version = "0.1.4" +name = "phf_codegen" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfc1c836fdc3d1ef87c348b237b5b5c4dff922156fb2d968f57734f9669768ca" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", - "version_check 0.9.2", - "yansi", + "phf_generator", + "phf_shared", ] [[package]] -name = "percent-encoding" -version = "1.0.1" +name = "phf_generator" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] [[package]] -name = "percent-encoding" -version = "2.1.0" +name = "phf_shared" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", + "unicase 1.4.2", +] [[package]] name = "pin-project" @@ -1153,9 +1041,9 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10b4b44893d3c370407a1d6a5cfde7c41ae0478e31c516c85f67eb3adc51be6d" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1182,16 +1070,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" -[[package]] -name = "polyval" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec3341498978de3bfd12d1b22f1af1de22818f5473a11e8a6ef997989e3a212" -dependencies = [ - "cfg-if", - "universal-hash", -] - [[package]] name = "ppv-lite86" version = "0.2.8" @@ -1207,6 +1085,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger", + "log 0.4.8", +] + [[package]] name = "proc-macro-error" version = "0.4.12" @@ -1214,9 +1102,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", "version_check 0.9.2", ] @@ -1226,21 +1114,24 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", "syn-mid", "version_check 0.9.2", ] [[package]] -name = "proc-macro2" -version = "0.4.30" +name = "proc-macro-hack" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" + +[[package]] +name = "proc-macro-nested" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0afe1bd463b9e9ed51d0e0f0b50b6b146aec855c56fd182bb242388710a9b6de" [[package]] name = "proc-macro2" @@ -1248,17 +1139,14 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ - "unicode-xid 0.2.0", + "unicode-xid", ] [[package]] -name = "quote" -version = "0.6.13" +name = "quick-error" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" @@ -1266,7 +1154,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.18", + "proc-macro2", ] [[package]] @@ -1280,6 +1168,25 @@ dependencies = [ "scheduled-thread-pool", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.8", +] + [[package]] name = "rand" version = "0.7.3" @@ -1288,9 +1195,19 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom", "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", ] [[package]] @@ -1300,9 +1217,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.5.1", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -1312,13 +1244,84 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.8", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.8", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", ] [[package]] @@ -1327,6 +1330,24 @@ version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + [[package]] name = "remove_dir_all" version = "0.5.2" @@ -1349,94 +1370,28 @@ dependencies = [ "futures-util", "http", "http-body", - "hyper 0.13.6", + "hyper", "hyper-tls", "js-sys", "lazy_static", "log 0.4.8", "mime 0.3.16", - "mime_guess", + "mime_guess 2.0.3", "native-tls", - "percent-encoding 2.1.0", + "percent-encoding", "pin-project-lite", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-tls", - "url 2.1.1", + "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "winreg", ] -[[package]] -name = "rocket" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6130967b369cfb8411b0b73e96fcba1229c32a9cc6f295d144f879bfced13c6e" -dependencies = [ - "atty", - "base64 0.12.1", - "log 0.4.8", - "memchr", - "num_cpus", - "pear", - "rocket_codegen", - "rocket_http", - "state", - "time", - "toml", - "version_check 0.9.2", - "yansi", -] - -[[package]] -name = "rocket_codegen" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb852e6da168fb948a8f2b798ba2e2f0e4fc860eae0efa9cf2bf0f5466bb0425" -dependencies = [ - "devise", - "glob", - "indexmap", - "quote 0.6.13", - "rocket_http", - "version_check 0.9.2", - "yansi", -] - -[[package]] -name = "rocket_contrib" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3946ca815127041d8f64455561031d058c22ae1b135251502c5ea523cf9e14b" -dependencies = [ - "log 0.4.8", - "notify", - "rocket", - "serde", - "serde_json", -] - -[[package]] -name = "rocket_http" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aff5a5480175f2f553a876b251e9350c74196128806d176da3a51c82aab5428" -dependencies = [ - "cookie", - "hyper 0.10.16", - "indexmap", - "pear", - "percent-encoding 1.0.1", - "smallvec", - "state", - "time", - "unicode-xid 0.1.0", -] - [[package]] name = "ryu" version = "1.0.5" @@ -1449,15 +1404,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.19" @@ -1477,6 +1423,12 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1521,9 +1473,9 @@ version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1546,14 +1498,14 @@ dependencies = [ "dtoa", "itoa", "serde", - "url 2.1.1", + "url", ] [[package]] -name = "sha2" +name = "sha-1" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ "block-buffer", "digest", @@ -1561,6 +1513,12 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + [[package]] name = "sirene" version = "1.0.3" @@ -1571,13 +1529,15 @@ dependencies = [ "diesel", "diesel_migrations", "dotenv", + "log 0.4.8", "openssl", + "pretty_env_logger", "r2d2", "reqwest", - "rocket", - "rocket_contrib", "serde", "serde_json", + "tokio", + "warp", "zip", ] @@ -1605,50 +1565,21 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "state" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" - [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "subtle" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" - -[[package]] -name = "subtle" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" - -[[package]] -name = "syn" -version = "0.15.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", -] - [[package]] name = "syn" version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "unicode-xid 0.2.0", + "proc-macro2", + "quote", + "unicode-xid", ] [[package]] @@ -1657,9 +1588,9 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1670,7 +1601,7 @@ checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ "cfg-if", "libc", - "rand", + "rand 0.7.3", "redox_syscall", "remove_dir_all", "winapi 0.3.8", @@ -1694,6 +1625,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.43" @@ -1720,6 +1660,18 @@ dependencies = [ "num_cpus", "pin-project-lite", "slab", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1732,6 +1684,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b8fe88007ebc363512449868d7da4389c9400072a3f666f212c7280082882a" +dependencies = [ + "futures", + "log 0.4.8", + "pin-project", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.3.1" @@ -1746,27 +1711,12 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" -dependencies = [ - "serde", -] - [[package]] name = "tower-service" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" -[[package]] -name = "traitobject" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" - [[package]] name = "try-lock" version = "0.2.2" @@ -1774,10 +1724,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] -name = "typeable" -version = "0.1.2" +name = "tungstenite" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +checksum = "cfea31758bf674f990918962e8e5f07071a3161bd7c4138ed23e416e1ac4264e" +dependencies = [ + "base64 0.11.0", + "byteorder", + "bytes", + "http", + "httparse", + "input_buffer", + "log 0.4.8", + "rand 0.7.3", + "sha-1", + "url", + "utf-8", +] + +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr", +] [[package]] name = "typenum" @@ -1833,12 +1805,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" -[[package]] -name = "unicode-xid" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" - [[package]] name = "unicode-xid" version = "0.2.0" @@ -1846,36 +1812,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] -name = "universal-hash" -version = "0.3.0" +name = "url" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0c900f2f9b4116803415878ff48b63da9edb268668e08cf9292d7503114a01" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" dependencies = [ - "generic-array", - "subtle 2.2.3", + "idna", + "matches", + "percent-encoding", ] [[package]] -name = "url" -version = "1.7.2" +name = "urlencoding" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] +checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed" [[package]] -name = "url" -version = "2.1.1" +name = "utf-8" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" -dependencies = [ - "idna 0.2.0", - "matches", - "percent-encoding 2.1.0", -] +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" [[package]] name = "vcpkg" @@ -1902,24 +1859,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" [[package]] -name = "walkdir" -version = "2.3.1" +name = "want" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "same-file", - "winapi 0.3.8", - "winapi-util", + "log 0.4.8", + "try-lock", ] [[package]] -name = "want" -version = "0.3.0" +name = "warp" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "0e95175b7a927258ecbb816bdada3cc469cb68593e7940b96a60f4af366a9970" dependencies = [ + "bytes", + "futures", + "headers", + "http", + "hyper", "log 0.4.8", - "try-lock", + "mime 0.3.16", + "mime_guess 2.0.3", + "multipart", + "pin-project", + "scoped-tls", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-tungstenite", + "tower-service", + "urlencoding", ] [[package]] @@ -1949,9 +1921,9 @@ dependencies = [ "bumpalo", "lazy_static", "log 0.4.8", - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-shared", ] @@ -1973,7 +1945,7 @@ version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" dependencies = [ - "quote 1.0.7", + "quote", "wasm-bindgen-macro-support", ] @@ -1983,9 +1955,9 @@ version = "0.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" dependencies = [ - "proc-macro2 1.0.18", - "quote 1.0.7", - "syn 1.0.31", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2068,18 +2040,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "yansi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" - -[[package]] -name = "zeroize" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" - [[package]] name = "zip" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index 0ccae23..1448633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,15 +13,13 @@ custom_error = "1.7" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2", "serde_json", "64-column-tables"] } diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" +log = "0.4" openssl = "0.10" +pretty_env_logger = "0.4" r2d2 = "0.8" reqwest = { version = "0.10", features = ["blocking", "json"] } -rocket = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +tokio = { version = "0.2", features = ["macros"] } +warp = "0.2" zip = "0.5" - -[dependencies.rocket_contrib] -version = "0.4" -default-features = false -features = ["json"] diff --git a/src/commands/common.rs b/src/commands/common.rs index d735ed6..f79e02b 100644 --- a/src/commands/common.rs +++ b/src/commands/common.rs @@ -1,6 +1,7 @@ use crate::models::update_metadata::common::SyntheticGroupType; use serde::Deserialize; +#[derive(Clone)] pub struct FolderOptions { pub temp: String, pub file: String, @@ -11,7 +12,7 @@ pub struct FolderOptions { pub enum CmdGroupType { UnitesLegales, Etablissements, - All + All, } impl From for SyntheticGroupType { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index f472294..39c3aa1 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -43,7 +43,7 @@ enum MainCommand { Serve(ServeFlags), } -pub fn run(builders: ConnectorsBuilders) { +pub async fn run(builders: ConnectorsBuilders) { let opts = Opts::parse(); let temp_folder = opts @@ -66,6 +66,6 @@ pub fn run(builders: ConnectorsBuilders) { match opts.main_command { MainCommand::Update(update_flags) => update::run(update_flags, folder_options, builders), - MainCommand::Serve(serve_flags) => serve::run(serve_flags, folder_options, builders), + MainCommand::Serve(serve_flags) => serve::run(serve_flags, folder_options, builders).await, } } diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index c97ed63..a48e2b6 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -2,8 +2,9 @@ mod runner; use super::common::FolderOptions; use crate::connectors::ConnectorsBuilders; -use rocket::config::{Config, Environment}; +use runner::common::Context; use std::env; +use std::net::ToSocketAddrs; #[derive(Clap, Debug)] pub struct ServeFlags { @@ -31,16 +32,6 @@ enum CmdEnvironment { Production, } -impl From for Environment { - fn from(env: CmdEnvironment) -> Self { - match env { - CmdEnvironment::Development => Environment::Development, - CmdEnvironment::Staging => Environment::Staging, - CmdEnvironment::Production => Environment::Production, - } - } -} - impl CmdEnvironment { pub fn from_str(s: String) -> Option { match s.as_str() { @@ -52,7 +43,7 @@ impl CmdEnvironment { } } -pub fn run(flags: ServeFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { +pub async fn run(flags: ServeFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { let env = flags.environment.unwrap_or_else(|| { CmdEnvironment::from_str(env::var("SIRENE_ENV").expect("Missing SIRENE_ENV")) .expect("Invalid SIRENE_ENV") @@ -69,15 +60,26 @@ pub fn run(flags: ServeFlags, folder_options: FolderOptions, builders: Connector .host .unwrap_or_else(|| env::var("HOST").expect("Missing HOST")); + let addr = format!("{}:{}", host, port) + .to_socket_addrs() + .expect("Unable to resolve domain") + .next() + .expect("No address available"); + let api_key = match flags.api_key { Some(key) => Some(key), None => env::var("API_KEY").ok(), }; - let config = Config::build(env.into()) - .address(host) - .port(port) - .finalize(); + log::info!("[Warp] Configuring for {:#?}", env); - runner::run(config.unwrap(), api_key, folder_options, builders) + runner::run( + addr, + Context { + builders, + api_key, + folder_options, + }, + ) + .await; } diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index b79905d..bcdeb2f 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -5,6 +5,7 @@ use crate::models::unite_legale::common::UniteLegale; use crate::models::update_metadata::common::{SyntheticGroupType, UpdateSummary}; use serde::{Deserialize, Serialize}; +#[derive(Clone)] pub struct Context { pub builders: ConnectorsBuilders, pub api_key: Option, diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index a15bfa8..a0bc843 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -2,10 +2,9 @@ use crate::connectors::Error as ConnectorError; use crate::models::{etablissement, unite_legale}; use crate::update::error::Error as InternalUpdateError; use custom_error::custom_error; -use rocket::http::Status; -use rocket::request::Request; -use rocket::response::{self, content, Responder, Response}; use serde::Serialize; +use std::convert::Infallible; +use warp::{http::StatusCode, Rejection, Reply}; custom_error! { pub Error InvalidData = "Invalid data", @@ -17,46 +16,96 @@ custom_error! { pub Error EtablissementError {source: etablissement::error::Error} = "[Etablissement] {source}", } +impl warp::reject::Reject for Error {} + +impl From for Rejection { + fn from(e: Error) -> Self { + warp::reject::custom(e) + } +} + +impl From for Rejection { + fn from(e: ConnectorError) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: InternalUpdateError) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: unite_legale::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: etablissement::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } +} + #[derive(Debug, Serialize)] struct ErrorResponse { + code: u16, message: String, } -impl<'r> Responder<'r> for Error { - fn respond_to(self, req: &Request) -> response::Result<'r> { - // Log error - println!("{}", self); - - let status = match &self { - Error::InvalidData => Status::BadRequest, - Error::MissingApiKeyError => return Err(Status::Unauthorized), - Error::ApiKeyError => return Err(Status::Unauthorized), - Error::UpdateConnectorError { source: _ } => Status::InternalServerError, - Error::UpdateError { source: _ } => Status::InternalServerError, - Error::UniteLegaleError { source } => match source { - unite_legale::error::Error::UniteLegaleNotFound => Status::NotFound, - _ => Status::InternalServerError, - }, - Error::EtablissementError { source } => match source { - etablissement::error::Error::EtablissementNotFound => Status::NotFound, - _ => Status::InternalServerError, +pub async fn handle_rejection(err: Rejection) -> Result { + let (code, message) = if let Some(e) = err.find::() { + log::debug!("[Warp][Error] {:?}", e); + + ( + match e { + Error::InvalidData => StatusCode::BAD_REQUEST, + Error::MissingApiKeyError => StatusCode::UNAUTHORIZED, + Error::ApiKeyError => StatusCode::UNAUTHORIZED, + Error::UpdateConnectorError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, + Error::UpdateError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, + Error::UniteLegaleError { source } => match source { + unite_legale::error::Error::UniteLegaleNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, + Error::EtablissementError { source } => match source { + etablissement::error::Error::EtablissementNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, }, - }; - - let error_response = ErrorResponse { - message: self.to_string(), - }; - - let json_result = serde_json::to_string(&error_response) - .map(|string| content::Json(string).respond_to(req).unwrap()) - .map_err(|e| { - eprintln!("JSON failed to serialize: {:?}", e); - Status::InternalServerError - }); - - match json_result { - Ok(json) => Response::build_from(json).status(status).ok(), - Err(status) => Err(status), - } + e.to_string(), + ) + } else if let Some(body_error) = err.find::() { + log::debug!("[Warp][Json] {}", body_error); + + (StatusCode::BAD_REQUEST, body_error.to_string()) + } else if let Some(e) = err.find::() { + log::debug!("[Warp][Method] {}", e); + + (StatusCode::NOT_FOUND, String::from("Not found")) + } else { + log::debug!("[Warp][Rejection] Unhandled error {:?}", err); + + ( + StatusCode::INTERNAL_SERVER_ERROR, + String::from("Internal server error"), + ) + }; + + if code == StatusCode::INTERNAL_SERVER_ERROR { + log::error!("[Warp][InternalServerError] {}", message); } + + Ok(warp::reply::with_status( + warp::reply::json(&ErrorResponse { + code: code.as_u16(), + message, + }), + code, + )) } diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 645cae7..d1e6868 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -1,8 +1,7 @@ -mod common; mod error; -use super::super::common::FolderOptions; -use crate::connectors::ConnectorsBuilders; +pub mod common; + use crate::models; use crate::update::{common::Config as DataConfig, update as update_data}; use common::{ @@ -11,63 +10,57 @@ use common::{ UpdateOptions, UpdateResponse, }; use error::Error; -use rocket::config::Config; -use rocket::State; -use rocket_contrib::json::Json; +use std::convert::Infallible; +use std::net::SocketAddr; +use warp::{ + http::{header, Method}, + Filter, Rejection, Reply, +}; -#[get("/")] fn index() -> &'static str { "SIRENE API v3" } -#[post("/update", format = "application/json", data = "")] -fn update( - state: State, - options: Json, -) -> Result, Error> { - let api_key = match &state.api_key { +async fn update(options: UpdateOptions, context: Context) -> Result { + let api_key = match &context.api_key { Some(key) => key, - None => return Err(Error::MissingApiKeyError), + None => return Err(Error::MissingApiKeyError.into()), }; if &options.api_key != api_key { - return Err(Error::ApiKeyError); + return Err(Error::ApiKeyError.into()); } - let connectors = state.builders.create_with_insee()?; + let connectors = context.builders.create_with_insee()?; let summary = update_data( options.group_type, DataConfig { force: options.force, data_only: options.data_only, - temp_folder: state.folder_options.temp.clone(), - file_folder: state.folder_options.file.clone(), - db_folder: state.folder_options.db.clone(), + temp_folder: context.folder_options.temp.clone(), + file_folder: context.folder_options.file.clone(), + db_folder: context.folder_options.db.clone(), }, &connectors, )?; - Ok(Json(UpdateResponse { summary })) + Ok(warp::reply::json(&UpdateResponse { summary })) } -#[get("/unites_legales/")] -fn unites_legales( - state: State, - siren: String, -) -> Result, Error> { +async fn unites_legales(siren: String, context: Context) -> Result { if siren.len() != 9 { - return Err(Error::InvalidData); + return Err(Error::InvalidData.into()); } - let connectors = state.builders.create(); + let connectors = context.builders.create(); let unite_legale = models::unite_legale::get(&connectors, &siren)?; let etablissements = models::etablissement::get_with_siren(&connectors, &siren)?; let etablissement_siege = models::etablissement::get_siege_with_siren(&connectors, &unite_legale.siren)?; - Ok(Json(UniteLegaleResponse { + Ok(warp::reply::json(&UniteLegaleResponse { unite_legale: UniteLegaleInnerResponse { unite_legale, etablissements, @@ -76,23 +69,19 @@ fn unites_legales( })) } -#[get("/etablissements/")] -fn etablissements( - state: State, - siret: String, -) -> Result, Error> { +async fn etablissements(siret: String, context: Context) -> Result { if siret.len() != 14 { - return Err(Error::InvalidData); + return Err(Error::InvalidData.into()); } - let connectors = state.builders.create(); + let connectors = context.builders.create(); let etablissement = models::etablissement::get(&connectors, &siret)?; let unite_legale = models::unite_legale::get(&connectors, &etablissement.siren)?; let etablissement_siege = models::etablissement::get_siege_with_siren(&connectors, &etablissement.siren)?; - Ok(Json(EtablissementResponse { + Ok(warp::reply::json(&EtablissementResponse { etablissement: EtablissementInnerResponse { etablissement, unite_legale: UniteLegaleEtablissementInnerResponse { @@ -103,19 +92,60 @@ fn etablissements( })) } -pub fn run( - config: Config, - api_key: Option, - folder_options: FolderOptions, - builders: ConnectorsBuilders, -) { - rocket::custom(config) - .mount("/v3", routes![index, unites_legales, etablissements]) - .mount("/admin", routes![update]) - .manage(Context { - builders, - api_key, - folder_options, - }) - .launch(); +pub async fn run(addr: SocketAddr, context: Context) { + // GET / -> OK + let health_route = warp::get() + .and(warp::path::end()) + .map(|| warp::reply::with_status("OK", warp::http::StatusCode::OK)); + log::info!("[Warp] Mount GET /"); + + let v3_route = warp::path!("v3" / ..); + + // GET /v3 -> "SIRENE API v3" + let v3_index = warp::path::end().map(index); + log::info!("[Warp] Mount GET /v3"); + + // GET /unites_legales/ + let v3_unites_legales_route = warp::get() + .and(warp::path!("unites_legales" / String)) + .and(with_context(context.clone())) + .and_then(unites_legales); + log::info!("[Warp] Mount GET /v3/unites_legales/"); + + // GET /etablissements/ + let v3_etablissement_route = warp::get() + .and(warp::path!("etablissements" / String)) + .and(with_context(context.clone())) + .and_then(etablissements); + log::info!("[Warp] Mount GET /v3/etablissements/"); + + // POST /admin/update {json} + let admin_update_route = warp::post() + .and(warp::path!("admin" / "update")) + .and(warp::body::json::()) + .and(with_context(context)) + .and_then(update); + log::info!("[Warp] Mount POST /admin/update {{json}}"); + + // Cors + let cors = warp::cors() + .allow_methods(&[Method::GET, Method::POST]) + .allow_headers(vec![header::CONTENT_TYPE]) + .allow_any_origin(); + + let routes = health_route + .or(v3_route.and( + v3_unites_legales_route + .or(v3_etablissement_route) + .or(v3_index), + )) + .or(admin_update_route) + .recover(error::handle_rejection) + .with(cors); + + warp::serve(routes).run(addr).await; +} + +fn with_context(context: Context) -> impl Filter + Clone { + warp::any().map(move || context.clone()) } diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index 372c4c1..af814e0 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -14,6 +14,7 @@ pub struct Connector { pub token: String, } +#[derive(Clone)] pub struct ConnectorBuilder { pub credentials: String, } diff --git a/src/connectors/local.rs b/src/connectors/local.rs index c7773ed..ac0b141 100644 --- a/src/connectors/local.rs +++ b/src/connectors/local.rs @@ -9,6 +9,7 @@ pub struct Connector { pub pool: Pool>, } +#[derive(Clone)] pub struct ConnectorBuilder { pool: Pool>, } diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index 583417e..b287239 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -7,6 +7,7 @@ custom_error! { pub Error InseeError { source: insee::error::InseeTokenError } = "{source}", } +#[derive(Clone)] pub struct ConnectorsBuilders { local: local::ConnectorBuilder, insee: Option, diff --git a/src/main.rs b/src/main.rs index ce67aff..6525462 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,11 +12,9 @@ extern crate diesel_migrations; extern crate dotenv; extern crate r2d2; extern crate reqwest; -#[macro_use] -extern crate rocket; -extern crate rocket_contrib; extern crate serde; extern crate serde_json; +extern crate warp; extern crate zip; mod commands; @@ -28,10 +26,14 @@ use chrono::Utc; use connectors::ConnectorsBuilders; use dotenv::dotenv; -fn main() { +#[tokio::main] +async fn main() { // Load configuration dotenv().ok(); + // Load Logger + pretty_env_logger::init(); + // Load database let connectors_builders = ConnectorsBuilders::new(); @@ -45,5 +47,5 @@ fn main() { .unwrap(); // Fail launch in case of error // Run command - commands::run(connectors_builders); + commands::run(connectors_builders).await; } From 68778e09383a4feb84c5701cdbf777ee2fbbde0c Mon Sep 17 00:00:00 2001 From: LDouziech Date: Fri, 19 Jun 2020 11:18:10 +0200 Subject: [PATCH 35/69] Replace blocking reqwest by async one (#142) --- Cargo.lock | 26 ++++++++------ Cargo.toml | 7 ++-- src/commands/mod.rs | 4 ++- src/commands/serve/runner/mod.rs | 5 +-- src/commands/update.rs | 10 +++--- src/connectors/insee/implementation.rs | 19 +++++----- src/connectors/insee/mod.rs | 16 +++++---- src/connectors/mod.rs | 4 +-- src/main.rs | 5 ++- src/models/common.rs | 6 ++-- src/models/etablissement/mod.rs | 9 +++-- src/models/unite_legale/mod.rs | 9 +++-- src/update/action/clean.rs | 4 ++- src/update/action/common.rs | 6 ++-- src/update/action/download_stock.rs | 35 +++++++++++++----- src/update/action/insert_stock.rs | 4 ++- src/update/action/mod.rs | 19 ++++++---- src/update/action/swap.rs | 4 ++- src/update/action/sync_insee.rs | 6 ++-- src/update/action/unzip_stock.rs | 4 ++- src/update/mod.rs | 50 +++++++++++++++----------- src/update/summary.rs | 16 +++++++++ 22 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 src/update/summary.rs diff --git a/Cargo.lock b/Cargo.lock index 9337a09..25e0f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "async-trait" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cb5d814ab2a47fd66d3266e9efccb53ca4c740b7451043b8ffcf9a6208f3f8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -896,16 +907,6 @@ dependencies = [ "autocfg 1.0.0", ] -[[package]] -name = "num_cpus" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "once_cell" version = "1.4.0" @@ -1523,12 +1524,14 @@ checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" name = "sirene" version = "1.0.3" dependencies = [ + "async-trait", "chrono", "clap", "custom_error", "diesel", "diesel_migrations", "dotenv", + "futures", "log 0.4.8", "openssl", "pretty_env_logger", @@ -1537,6 +1540,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "tokio-util", "warp", "zip", ] @@ -1657,7 +1661,6 @@ dependencies = [ "lazy_static", "memchr", "mio", - "num_cpus", "pin-project-lite", "slab", "tokio-macros", @@ -1705,6 +1708,7 @@ checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "log 0.4.8", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index 1448633..ee427e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,19 +7,22 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } clap = "3.0.0-beta" custom_error = "1.7" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2", "serde_json", "64-column-tables"] } diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" +futures = "0.3" log = "0.4" openssl = "0.10" pretty_env_logger = "0.4" r2d2 = "0.8" -reqwest = { version = "0.10", features = ["blocking", "json"] } +reqwest = { version = "0.10", features = ["json", "stream"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "0.2", features = ["macros"] } +tokio = { version = "0.2", features = ["macros", "io-util", "fs"] } +tokio-util = { version = "0.3", features = ["compat"] } warp = "0.2" zip = "0.5" diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 39c3aa1..2ef2b54 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -65,7 +65,9 @@ pub async fn run(builders: ConnectorsBuilders) { }; match opts.main_command { - MainCommand::Update(update_flags) => update::run(update_flags, folder_options, builders), + MainCommand::Update(update_flags) => { + update::run(update_flags, folder_options, builders).await + } MainCommand::Serve(serve_flags) => serve::run(serve_flags, folder_options, builders).await, } } diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index d1e6868..383bb0a 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -31,7 +31,7 @@ async fn update(options: UpdateOptions, context: Context) -> Result Result Step::SyncInsee, }; - update_step(step, synthetic_group_type, config, &connectors) + update_step(step, synthetic_group_type, config, &connectors).await } - None => update(synthetic_group_type, config, &connectors), + None => update(synthetic_group_type, config, &connectors).await, }; match summary_result { - Ok(summary) => println!("{}", serde_json::to_string(&summary).unwrap()), + Ok(summary) => println!("{}", serde_json::to_string(&summary).expect("Unable to stringify summary")), Err(error) => error.exit(), } } diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index 4b0bac0..f358cd1 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -19,7 +19,7 @@ struct EndpointConfig { } impl Connector { - pub fn get_daily_unites_legales( + pub async fn get_daily_unites_legales( &self, start_timestamp: NaiveDateTime, cursor: String, @@ -32,7 +32,8 @@ impl Connector { }, start_timestamp, cursor, - )?; + ) + .await?; Ok(( next_cursor, @@ -47,7 +48,7 @@ impl Connector { )) } - pub fn get_daily_etablissements( + pub async fn get_daily_etablissements( &self, start_timestamp: NaiveDateTime, cursor: String, @@ -60,7 +61,8 @@ impl Connector { }, start_timestamp, cursor, - )?; + ) + .await?; Ok(( next_cursor, @@ -80,12 +82,12 @@ fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime timestamp.max(Utc::now().naive_local() - Duration::days(31)) } -fn get_daily_data( +async fn get_daily_data( config: EndpointConfig, start_timestamp: NaiveDateTime, cursor: String, ) -> Result<(Option, Option), InseeUpdateError> { - let client = reqwest::blocking::Client::new(); + let client = reqwest::Client::new(); let url = format!( "{}/{}?q={}:%7B{} TO *]&nombre=1000&curseur={}", @@ -100,10 +102,11 @@ fn get_daily_data( .get(&url) .header(AUTHORIZATION, format!("Bearer {}", config.token)) .header(ACCEPT, HeaderValue::from_static("application/json")) - .send()? + .send() + .await? .error_for_status() { - Ok(response) => response.json::().map_err(|error| error.into()), + Ok(response) => response.json::().await.map_err(|error| error.into()), Err(error) => { // Insee returns 404 for empty data if let Some(status) = error.status() { diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index af814e0..cba5a7c 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -44,21 +44,23 @@ impl ConnectorBuilder { } } - pub fn create(&self) -> Result { - self.generate_token().map(|token| Connector { token }) + pub async fn create(&self) -> Result { + self.generate_token().await.map(|token| Connector { token }) } - fn generate_token(&self) -> Result { - let client = reqwest::blocking::Client::new(); - let response: InseeTokenResponse = client + async fn generate_token(&self) -> Result { + let client = reqwest::Client::new(); + let response = client .post("https://api.insee.fr/token") .header(AUTHORIZATION, format!("Basic {}", self.credentials)) .form(&InseeTokenParameters { grant_type: String::from("client_credentials"), validity_period: 86400, }) - .send()? - .json()?; + .send() + .await? + .json::() + .await?; Ok(response.access_token) } diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index b287239..e51fa9f 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -33,11 +33,11 @@ impl ConnectorsBuilders { } } - pub fn create_with_insee(&self) -> Result { + pub async fn create_with_insee(&self) -> Result { Ok(Connectors { local: self.local.create(), insee: match self.insee.as_ref() { - Some(insee_builder) => Some(insee_builder.create()?), + Some(insee_builder) => Some(insee_builder.create().await?), None => None, }, }) diff --git a/src/main.rs b/src/main.rs index 6525462..89f611e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,3 @@ -#![feature(proc_macro_hygiene, decl_macro)] - extern crate chrono; #[macro_use] extern crate clap; @@ -14,6 +12,7 @@ extern crate r2d2; extern crate reqwest; extern crate serde; extern crate serde_json; +extern crate tokio; extern crate warp; extern crate zip; @@ -44,7 +43,7 @@ async fn main() { String::from("Program unexpectedly closed"), Utc::now(), ) - .unwrap(); // Fail launch in case of error + .expect("Unable to terminate last unfinished update"); // Fail launch in case of error // Run command commands::run(connectors_builders).await; diff --git a/src/models/common.rs b/src/models/common.rs index 43fdc4b..d5bd151 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -1,9 +1,11 @@ use crate::connectors::insee::error::InseeUpdateError; use crate::connectors::Connectors; +use async_trait::async_trait; use chrono::NaiveDateTime; use custom_error::custom_error; -pub trait UpdatableModel { +#[async_trait] +pub trait UpdatableModel: Sync + Send { fn count(&self, connectors: &Connectors) -> Result; fn count_staging(&self, connectors: &Connectors) -> Result; fn insert_in_staging(&self, connectors: &Connectors, file_path: String) -> Result; @@ -12,7 +14,7 @@ pub trait UpdatableModel { &self, connectors: &Connectors, ) -> Result, Error>; - fn update_daily_data( + async fn update_daily_data( &self, connectors: &Connectors, start_timestamp: NaiveDateTime, diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 7ae578c..37966fe 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -5,6 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; +use async_trait::async_trait; use chrono::NaiveDateTime; use common::Etablissement; use diesel::pg::upsert::excluded; @@ -44,6 +45,7 @@ pub fn get_siege_with_siren( pub struct EtablissementModel {} +#[async_trait] impl UpdatableModel for EtablissementModel { fn count(&self, connectors: &Connectors) -> Result { let connection = connectors.local.pool.get()?; @@ -126,7 +128,7 @@ impl UpdatableModel for EtablissementModel { .map_err(|error| error.into()) } - fn update_daily_data( + async fn update_daily_data( &self, connectors: &Connectors, start_timestamp: NaiveDateTime, @@ -142,8 +144,9 @@ impl UpdatableModel for EtablissementModel { let mut updated_count: usize = 0; while let Some(cursor) = current_cursor { - let (next_cursor, etablissements) = - insee.get_daily_etablissements(start_timestamp, cursor)?; + let (next_cursor, etablissements) = insee + .get_daily_etablissements(start_timestamp, cursor) + .await?; current_cursor = next_cursor; diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index cc3f232..f2ba0c7 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -5,6 +5,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; +use async_trait::async_trait; use chrono::NaiveDateTime; use common::UniteLegale; use diesel::pg::upsert::excluded; @@ -22,6 +23,7 @@ pub fn get(connectors: &Connectors, siren: &String) -> Result Result { let connection = connectors.local.pool.get()?; @@ -104,7 +106,7 @@ impl UpdatableModel for UniteLegaleModel { .map_err(|error| error.into()) } - fn update_daily_data( + async fn update_daily_data( &self, connectors: &Connectors, start_timestamp: NaiveDateTime, @@ -120,8 +122,9 @@ impl UpdatableModel for UniteLegaleModel { let mut updated_count: usize = 0; while let Some(cursor) = current_cursor { - let (next_cursor, unites_legales) = - insee.get_daily_unites_legales(start_timestamp, cursor)?; + let (next_cursor, unites_legales) = insee + .get_daily_unites_legales(start_timestamp, cursor) + .await?; current_cursor = next_cursor; diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index ee3a8df..4a7a2f3 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -4,6 +4,7 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::Utc; use std::fs::remove_file; use std::path::PathBuf; @@ -13,12 +14,13 @@ pub struct CleanAction { pub file_folder: String, } +#[async_trait] impl Action for CleanAction { fn step(&self) -> Step { Step::CleanFile } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, diff --git a/src/update/action/common.rs b/src/update/action/common.rs index d3bc3c1..0228ef2 100644 --- a/src/update/action/common.rs +++ b/src/update/action/common.rs @@ -2,10 +2,12 @@ use super::super::error::Error; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; -pub trait Action { +#[async_trait] +pub trait Action: Sync + Send { fn step(&self) -> Step; - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index 8f2572b..b33e2c7 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -4,23 +4,27 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::{DateTime, Utc}; +use futures::stream::TryStreamExt; use reqwest::header::LAST_MODIFIED; -use std::fs::{create_dir_all, File}; -use std::io; +use std::fs::create_dir_all; use std::path::PathBuf; +use tokio::fs::File; +use tokio_util::compat::FuturesAsyncReadCompatExt; pub struct DownloadAction { pub temp_folder: String, pub force: bool, } +#[async_trait] impl Action for DownloadAction { fn step(&self) -> Step { Step::DownloadFile } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, @@ -40,7 +44,8 @@ impl Action for DownloadAction { zip_path.set_extension("zip"); // Prepare file download - let mut resp = reqwest::blocking::get(metadata.url.as_str()) + let resp = reqwest::get(metadata.url.as_str()) + .await .map_err(|req_error| Error::DownloadError { req_error })?; // Decode Last-Modified header @@ -50,8 +55,10 @@ impl Action for DownloadAction { .ok_or(Error::MissingLastModifiedHeader)? .to_str() .map_err(|head_error| Error::InvalidLastModifiedHeader { head_error })?; + let last_modified = DateTime::parse_from_rfc2822(last_modified_str) .map_err(|date_error| Error::InvalidLastModifiedDate { date_error })?; + let last_modified = last_modified.with_timezone(&Utc); // Test if not already imported or downloaded @@ -83,10 +90,22 @@ impl Action for DownloadAction { } } - // Download data and store it on filesystem - let mut out = - File::create(zip_path).map_err(|io_error| Error::FileCreationError { io_error })?; - io::copy(&mut resp, &mut out).map_err(|io_error| Error::FileCopyError { io_error })?; + // Create an output file into which we will save current stock. + let mut outfile = File::create(zip_path) + .await + .map_err(|io_error| Error::FileCreationError { io_error })?; + + let mut stream = resp + .bytes_stream() // Convert the body of the response into a futures::io::Stream. + .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) // We must first convert the reqwest::Error into an futures::io::Error. + .into_async_read() // Convert the stream into an futures::io::AsyncRead. + .compat(); // Convert the futures::io::AsyncRead into a tokio::io::AsyncRead. + + // Invoke tokio::io::copy to actually perform the download. + tokio::io::copy(&mut stream, &mut outfile) + .await + .map_err(|io_error| Error::FileCopyError { io_error })?; + println!("[Download] Download of {:#?} finished", group_type); // Update staging file timestamp diff --git a/src/update/action/insert_stock.rs b/src/update/action/insert_stock.rs index d19ed38..fcd6710 100644 --- a/src/update/action/insert_stock.rs +++ b/src/update/action/insert_stock.rs @@ -4,6 +4,7 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::Utc; use std::fs::canonicalize; use std::path::PathBuf; @@ -13,12 +14,13 @@ pub struct InsertAction { pub force: bool, } +#[async_trait] impl Action for InsertAction { fn step(&self) -> Step { Step::InsertData } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, diff --git a/src/update/action/mod.rs b/src/update/action/mod.rs index 1727252..0053ba6 100644 --- a/src/update/action/mod.rs +++ b/src/update/action/mod.rs @@ -1,5 +1,5 @@ use super::common::Config; -use super::error::Error; +use super::summary::Summary; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary, UpdateStepSummary}; @@ -14,27 +14,34 @@ pub mod swap; pub mod sync_insee; pub mod unzip_stock; -pub fn execute_step( +pub async fn execute_step( step: Step, config: &Config, groups: &Vec, connectors: &Connectors, -) -> Result { + summary: &mut Summary, +) { let started_timestamp = Utc::now(); let mut groups_summary: Vec = vec![]; let action = build_action(config, step); for group in groups { - groups_summary.push(action.execute(*group, connectors)?); + match action.execute(*group, connectors).await { + Ok(group) => groups_summary.push(group), + Err(error) => { + summary.error = Some(error); + return; + } + } } - Ok(UpdateStepSummary { + summary.steps.push(UpdateStepSummary { step, updated: groups_summary.iter().find(|&g| g.updated).is_some(), started_timestamp, finished_timestamp: Utc::now(), groups: groups_summary, - }) + }); } fn build_action(config: &Config, step: Step) -> Box { diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index 699cedd..d570103 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -4,18 +4,20 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::Utc; pub struct SwapAction { pub force: bool, } +#[async_trait] impl Action for SwapAction { fn step(&self) -> Step { Step::SwapData } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 18fb0fb..8920507 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -4,16 +4,18 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::Utc; pub struct SyncInseeAction {} +#[async_trait] impl Action for SyncInseeAction { fn step(&self) -> Step { Step::SyncInsee } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, @@ -28,7 +30,7 @@ impl Action for SyncInseeAction { let model = group_type.get_updatable_model(); if let Some(timestamp) = model.get_last_insee_synced_timestamp(connectors)? { - let updated_count = model.update_daily_data(connectors, timestamp)?; + let updated_count = model.update_daily_data(connectors, timestamp).await?; println!("[SyncInsee] {} {:#?} synced", updated_count, group_type); group_metadata::set_last_insee_synced_timestamp( diff --git a/src/update/action/unzip_stock.rs b/src/update/action/unzip_stock.rs index 25d41fe..f038d51 100644 --- a/src/update/action/unzip_stock.rs +++ b/src/update/action/unzip_stock.rs @@ -4,6 +4,7 @@ use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; use chrono::Utc; use std::fs::{create_dir_all, set_permissions, File, Permissions}; use std::io; @@ -15,12 +16,13 @@ pub struct UnzipAction { pub force: bool, } +#[async_trait] impl Action for UnzipAction { fn step(&self) -> Step { Step::UnzipFile } - fn execute( + async fn execute( &self, group_type: GroupType, connectors: &Connectors, diff --git a/src/update/mod.rs b/src/update/mod.rs index 9d9f82d..bc814e3 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -1,18 +1,18 @@ use crate::connectors::Connectors; use crate::models::update_metadata; -use crate::models::update_metadata::common::{ - Step, SyntheticGroupType, UpdateStepSummary, UpdateSummary, -}; +use crate::models::update_metadata::common::{Step, SyntheticGroupType, UpdateSummary}; use action::execute_step; use chrono::Utc; use common::Config; use error::Error; +use summary::Summary; pub mod action; pub mod common; pub mod error; +pub mod summary; -pub fn update( +pub async fn update( synthetic_group_type: SyntheticGroupType, config: Config, connectors: &Connectors, @@ -24,19 +24,20 @@ pub fn update( config, connectors, ) + .await } -pub fn update_step( +pub async fn update_step( step: Step, synthetic_group_type: SyntheticGroupType, config: Config, connectors: &Connectors, ) -> Result { // Execute step - execute_workflow(vec![step], synthetic_group_type, config, connectors) + execute_workflow(vec![step], synthetic_group_type, config, connectors).await } -fn execute_workflow( +async fn execute_workflow( workflow: Vec, synthetic_group_type: SyntheticGroupType, config: Config, @@ -55,26 +56,35 @@ fn execute_workflow( let started_timestamp = Utc::now(); // Execute workflow - let result_steps: Result, Error> = workflow - .into_iter() - .map(|step| execute_step(step, &config, &synthetic_group_type.into(), connectors)) - .collect(); - - let steps = match result_steps { - Ok(s) => s, - Err(error) => { - update_metadata::error_update(connectors, error.to_string(), Utc::now())?; - return Err(error); + let mut summary = Summary::new(); + + for step in workflow.into_iter() { + execute_step( + step, + &config, + &synthetic_group_type.into(), + connectors, + &mut summary, + ) + .await; + + if summary.error.is_some() { + break; } - }; + } + + if let Some(error) = summary.error { + update_metadata::error_update(connectors, error.to_string(), Utc::now())?; + return Err(error); + } // End println!("[Update] Finished"); let summary = UpdateSummary { - updated: steps.iter().find(|&s| s.updated).is_some(), + updated: summary.steps.iter().find(|&s| s.updated).is_some(), started_timestamp, finished_timestamp: Utc::now(), - steps, + steps: summary.steps, }; update_metadata::finished_update(connectors, summary.clone())?; diff --git a/src/update/summary.rs b/src/update/summary.rs new file mode 100644 index 0000000..e3fd740 --- /dev/null +++ b/src/update/summary.rs @@ -0,0 +1,16 @@ +use super::error::Error; +use crate::models::update_metadata::common::UpdateStepSummary; + +pub struct Summary { + pub error: Option, + pub steps: Vec, +} + +impl Summary { + pub fn new() -> Self { + Summary { + error: None, + steps: vec![], + } + } +} From c7086e022a8b36fef476fd1e6db907eead651f55 Mon Sep 17 00:00:00 2001 From: LDouziech Date: Tue, 23 Jun 2020 17:06:03 +0200 Subject: [PATCH 36/69] Cooldown on INSEE calls and fix ordering (#143) --- Cargo.toml | 2 +- src/commands/serve/runner/mod.rs | 4 +-- src/commands/update.rs | 14 +++++--- src/connectors/insee/implementation.rs | 47 ++++++++++++++++++++------ src/connectors/insee/mod.rs | 9 ++++- src/connectors/insee/types/mod.rs | 10 +++++- src/models/common.rs | 2 +- src/models/etablissement/mod.rs | 4 +-- src/models/unite_legale/mod.rs | 4 +-- src/update/action/clean.rs | 2 +- src/update/action/common.rs | 2 +- src/update/action/download_stock.rs | 2 +- src/update/action/insert_stock.rs | 2 +- src/update/action/mod.rs | 2 +- src/update/action/swap.rs | 2 +- src/update/action/sync_insee.rs | 2 +- src/update/action/unzip_stock.rs | 2 +- src/update/mod.rs | 6 ++-- 18 files changed, 82 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ee427e2..af01277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ r2d2 = "0.8" reqwest = { version = "0.10", features = ["json", "stream"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "0.2", features = ["macros", "io-util", "fs"] } +tokio = { version = "0.2", features = ["macros", "io-util", "fs", "time"] } tokio-util = { version = "0.3", features = ["compat"] } warp = "0.2" zip = "0.5" diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 383bb0a..66047db 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -31,7 +31,7 @@ async fn update(options: UpdateOptions, context: Context) -> Result Result Step::SyncInsee, }; - update_step(step, synthetic_group_type, config, &connectors).await + update_step(step, synthetic_group_type, config, &mut connectors).await } - None => update(synthetic_group_type, config, &connectors).await, + None => update(synthetic_group_type, config, &mut connectors).await, }; match summary_result { - Ok(summary) => println!("{}", serde_json::to_string(&summary).expect("Unable to stringify summary")), + Ok(summary) => println!( + "{}", + serde_json::to_string(&summary).expect("Unable to stringify summary") + ), Err(error) => error.exit(), } } diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index f358cd1..f9b4e65 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -1,7 +1,7 @@ use super::error::InseeUpdateError; use super::types::{ etablissement::InseeEtablissementResponse, unite_legale::InseeUniteLegaleResponse, - InseeResponse, + InseeQueryParams, InseeResponse, }; use super::Connector; use crate::models::etablissement::common::Etablissement; @@ -9,6 +9,8 @@ use crate::models::unite_legale::common::UniteLegale; use chrono::{Duration, NaiveDateTime, Utc}; use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; +const MAX_CALL: u8 = 20; +const MAX_DURATION: std::time::Duration = std::time::Duration::from_secs(60); const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; pub const INITIAL_CURSOR: &str = "*"; @@ -19,11 +21,29 @@ struct EndpointConfig { } impl Connector { + async fn wait_for_insee_limitation(&mut self) { + if self.calls == 0 { + self.started_at = std::time::Instant::now(); + } + + self.calls += 1; + + let elapsed = self.started_at.elapsed(); + if self.calls > MAX_CALL && elapsed < MAX_DURATION { + tokio::time::delay_for(MAX_DURATION).await; + self.calls = 0; + } else if elapsed >= MAX_DURATION { + self.calls = 0; + } + } + pub async fn get_daily_unites_legales( - &self, + &mut self, start_timestamp: NaiveDateTime, cursor: String, ) -> Result<(Option, Vec), InseeUpdateError> { + self.wait_for_insee_limitation().await; + let (next_cursor, response) = get_daily_data::( EndpointConfig { token: self.token.clone(), @@ -49,10 +69,12 @@ impl Connector { } pub async fn get_daily_etablissements( - &self, + &mut self, start_timestamp: NaiveDateTime, cursor: String, ) -> Result<(Option, Vec), InseeUpdateError> { + self.wait_for_insee_limitation().await; + let (next_cursor, response) = get_daily_data::( EndpointConfig { token: self.token.clone(), @@ -89,19 +111,22 @@ async fn get_daily_data( ) -> Result<(Option, Option), InseeUpdateError> { let client = reqwest::Client::new(); - let url = format!( - "{}/{}?q={}:%7B{} TO *]&nombre=1000&curseur={}", - BASE_URL, - config.route, - config.query_field, - get_minimum_timestamp_for_request(start_timestamp).format("%Y-%m-%dT%H:%M:%S"), - cursor - ); + let url = format!("{}/{}", BASE_URL, config.route); let response = match client .get(&url) .header(AUTHORIZATION, format!("Bearer {}", config.token)) .header(ACCEPT, HeaderValue::from_static("application/json")) + .query(&InseeQueryParams { + q: format!( + "{}:[{} TO *]", + config.query_field, + get_minimum_timestamp_for_request(start_timestamp).format("%Y-%m-%dT%H:%M:%S") + ), + nombre: 1000, + curseur: cursor, + tri: format!("{} asc", config.query_field), + }) .send() .await? .error_for_status() diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index cba5a7c..361dc08 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -7,11 +7,14 @@ use error::InseeTokenError; use reqwest::header::AUTHORIZATION; use serde::{Deserialize, Serialize}; use std::env; +use std::time::Instant; pub use implementation::INITIAL_CURSOR; pub struct Connector { pub token: String, + calls: u8, + started_at: Instant, } #[derive(Clone)] @@ -45,7 +48,11 @@ impl ConnectorBuilder { } pub async fn create(&self) -> Result { - self.generate_token().await.map(|token| Connector { token }) + self.generate_token().await.map(|token| Connector { + token, + calls: 0, + started_at: Instant::now(), + }) } async fn generate_token(&self) -> Result { diff --git a/src/connectors/insee/types/mod.rs b/src/connectors/insee/types/mod.rs index 8fc3bfb..4caede8 100644 --- a/src/connectors/insee/types/mod.rs +++ b/src/connectors/insee/types/mod.rs @@ -1,10 +1,18 @@ pub mod etablissement; pub mod unite_legale; -use serde::{de::DeserializeOwned, Deserialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::fmt::Display; use std::str::FromStr; +#[derive(Serialize)] +pub struct InseeQueryParams { + pub q: String, + pub nombre: u16, + pub curseur: String, + pub tri: String, +} + pub trait InseeResponse: DeserializeOwned { fn header(&self) -> Header; } diff --git a/src/models/common.rs b/src/models/common.rs index d5bd151..b5e2c54 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -16,7 +16,7 @@ pub trait UpdatableModel: Sync + Send { ) -> Result, Error>; async fn update_daily_data( &self, - connectors: &Connectors, + connectors: &mut Connectors, start_timestamp: NaiveDateTime, ) -> Result; } diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 37966fe..3df6c80 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -130,12 +130,12 @@ impl UpdatableModel for EtablissementModel { async fn update_daily_data( &self, - connectors: &Connectors, + connectors: &mut Connectors, start_timestamp: NaiveDateTime, ) -> Result { let insee = connectors .insee - .as_ref() + .as_mut() .ok_or(UpdatableError::MissingInseeConnector)?; let connection = connectors.local.pool.get()?; diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index f2ba0c7..1f93504 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -108,12 +108,12 @@ impl UpdatableModel for UniteLegaleModel { async fn update_daily_data( &self, - connectors: &Connectors, + connectors: &mut Connectors, start_timestamp: NaiveDateTime, ) -> Result { let insee = connectors .insee - .as_ref() + .as_mut() .ok_or(UpdatableError::MissingInseeConnector)?; let connection = connectors.local.pool.get()?; diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index 4a7a2f3..8de39f4 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -23,7 +23,7 @@ impl Action for CleanAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[Clean] Cleaning {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/action/common.rs b/src/update/action/common.rs index 0228ef2..33fdf18 100644 --- a/src/update/action/common.rs +++ b/src/update/action/common.rs @@ -10,6 +10,6 @@ pub trait Action: Sync + Send { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result; } diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index b33e2c7..120f5ae 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -27,7 +27,7 @@ impl Action for DownloadAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[Download] Downloading {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/action/insert_stock.rs b/src/update/action/insert_stock.rs index fcd6710..20d2ce6 100644 --- a/src/update/action/insert_stock.rs +++ b/src/update/action/insert_stock.rs @@ -23,7 +23,7 @@ impl Action for InsertAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[Insert] Insert {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/action/mod.rs b/src/update/action/mod.rs index 0053ba6..86f7303 100644 --- a/src/update/action/mod.rs +++ b/src/update/action/mod.rs @@ -18,7 +18,7 @@ pub async fn execute_step( step: Step, config: &Config, groups: &Vec, - connectors: &Connectors, + connectors: &mut Connectors, summary: &mut Summary, ) { let started_timestamp = Utc::now(); diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index d570103..4582525 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -20,7 +20,7 @@ impl Action for SwapAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[Insert] Swapping {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 8920507..b07dfd6 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -18,7 +18,7 @@ impl Action for SyncInseeAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[SyncInsee] Syncing {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/action/unzip_stock.rs b/src/update/action/unzip_stock.rs index f038d51..b2ea4d9 100644 --- a/src/update/action/unzip_stock.rs +++ b/src/update/action/unzip_stock.rs @@ -25,7 +25,7 @@ impl Action for UnzipAction { async fn execute( &self, group_type: GroupType, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { println!("[Unzip] Unzipping {:#?}", group_type); let started_timestamp = Utc::now(); diff --git a/src/update/mod.rs b/src/update/mod.rs index bc814e3..6744b57 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -15,7 +15,7 @@ pub mod summary; pub async fn update( synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { // Build and execute workflow execute_workflow( @@ -31,7 +31,7 @@ pub async fn update_step( step: Step, synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { // Execute step execute_workflow(vec![step], synthetic_group_type, config, connectors).await @@ -41,7 +41,7 @@ async fn execute_workflow( workflow: Vec, synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, + connectors: &mut Connectors, ) -> Result { // Register start update_metadata::launch_update( From aecaf40bb3c1fc204f71971fdd11b0a0226ee11a Mon Sep 17 00:00:00 2001 From: LDouziech Date: Fri, 26 Jun 2020 16:06:17 +0200 Subject: [PATCH 37/69] Update report in realtime (#146) * Reverse update report summaries * Add get current status endpoint `GET /admin/update/status` * Realtime report saving * Handle timeout * Handle INSEE API limitations * Add support for multiple instances while updating by adding a manual command to finish a staled update * Upgrade stock links to HTTPS Co-authored-by: Julien Blatecky --- README.md | 21 ++- .../down.sql | 3 + .../2020-06-26-132333_switch_to_https/up.sql | 3 + src/commands/serve/runner/common.rs | 5 + src/commands/serve/runner/error.rs | 19 ++- src/commands/serve/runner/mod.rs | 72 +++++++- src/commands/update.rs | 22 ++- src/connectors/insee/error.rs | 1 + src/connectors/insee/implementation.rs | 91 +++++++--- src/connectors/insee/mod.rs | 37 ++++- src/connectors/insee/types/mod.rs | 22 +++ src/main.rs | 10 -- src/models/common.rs | 8 +- src/models/etablissement/mod.rs | 147 ++++++++-------- src/models/unite_legale/mod.rs | 115 +++++++------ src/models/update_metadata/common.rs | 26 ++- src/models/update_metadata/mod.rs | 23 ++- src/update/action/clean.rs | 26 +-- src/update/action/common.rs | 8 +- src/update/action/download_stock.rs | 63 ++++--- src/update/action/insert_stock.rs | 66 ++++---- src/update/action/mod.rs | 40 ++--- src/update/action/swap.rs | 52 +++--- src/update/action/sync_insee.rs | 78 ++++++--- src/update/action/unzip_stock.rs | 64 ++++--- src/update/mod.rs | 41 ++--- src/update/summary.rs | 157 +++++++++++++++++- 27 files changed, 821 insertions(+), 399 deletions(-) create mode 100644 migrations/2020-06-26-132333_switch_to_https/down.sql create mode 100644 migrations/2020-06-26-132333_switch_to_https/up.sql diff --git a/README.md b/README.md index 0944a15..384bf83 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,15 @@ OPTIONS: **> sirene update --help** ``` +sirene-update +Update data from CSV source files + USAGE: sirene update [FLAGS] [SUBCOMMAND] ARGS: - Configure which part will be updated, unites-legales, etablissements or all + Configure which part will be updated, unites-legales, etablissements or all [possible values: + unites-legales, etablissements, all] FLAGS: --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete it @@ -93,11 +97,12 @@ FLAGS: SUBCOMMANDS: clean-file Clean files from FILE_FOLDER download-file Download file in TEMP_FOLDER + finish-error Set a staled update process to error, use only if the process is really stopped help Prints this message or the help of the given subcommand(s) insert-data Load CSV file in database in loader-table from DB_FOLDER swap-data Swap loader-table to production - unzip-file Unzip file from TEMP_FOLDER, and move it to the FILE_FOLDER sync-insee Synchronise daily data from INSEE since the last modification + unzip-file Unzip file from TEMP_FOLDER, and move it to the FILE_FOLDER ``` ### HTTP API @@ -122,6 +127,18 @@ POST /admin/update } ``` +``` +GET /admin/update/status?api_key=string +``` + +``` +POST /admin/update/status/error + +{ + api_key: string, +} +``` + ### Basic usage Serve: diff --git a/migrations/2020-06-26-132333_switch_to_https/down.sql b/migrations/2020-06-26-132333_switch_to_https/down.sql new file mode 100644 index 0000000..9630783 --- /dev/null +++ b/migrations/2020-06-26-132333_switch_to_https/down.sql @@ -0,0 +1,3 @@ +UPDATE "public"."group_metadata" SET "url" = 'http://files.data.gouv.fr/insee-sirene/StockEtablissement_utf8.zip' WHERE "group_type" = 'etablissements'; + +UPDATE "public"."group_metadata" SET "url" = 'http://files.data.gouv.fr/insee-sirene/StockUniteLegale_utf8.zip' WHERE "group_type" = 'unites_legales'; diff --git a/migrations/2020-06-26-132333_switch_to_https/up.sql b/migrations/2020-06-26-132333_switch_to_https/up.sql new file mode 100644 index 0000000..a971bf2 --- /dev/null +++ b/migrations/2020-06-26-132333_switch_to_https/up.sql @@ -0,0 +1,3 @@ +UPDATE "public"."group_metadata" SET "url" = 'https://files.data.gouv.fr/insee-sirene/StockEtablissement_utf8.zip' WHERE "group_type" = 'etablissements'; + +UPDATE "public"."group_metadata" SET "url" = 'https://files.data.gouv.fr/insee-sirene/StockUniteLegale_utf8.zip' WHERE "group_type" = 'unites_legales'; diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index bcdeb2f..eb1c48e 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -20,6 +20,11 @@ pub struct UpdateOptions { pub data_only: bool, } +#[derive(Deserialize)] +pub struct StatusQueryString { + pub api_key: String, +} + #[derive(Serialize)] pub struct UpdateResponse { pub summary: UpdateSummary, diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index a0bc843..fe031df 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -1,5 +1,5 @@ use crate::connectors::Error as ConnectorError; -use crate::models::{etablissement, unite_legale}; +use crate::models::{etablissement, unite_legale, update_metadata}; use crate::update::error::Error as InternalUpdateError; use custom_error::custom_error; use serde::Serialize; @@ -14,6 +14,7 @@ custom_error! { pub Error UpdateError {source: InternalUpdateError} = "[Update] {source}", UniteLegaleError {source: unite_legale::error::Error} = "[UniteLegale] {source}", EtablissementError {source: etablissement::error::Error} = "[Etablissement] {source}", + StatusError {source: update_metadata::error::Error} = "[Status] {source}", } impl warp::reject::Reject for Error {} @@ -52,6 +53,13 @@ impl From for Rejection { } } +impl From for Rejection { + fn from(e: update_metadata::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } +} + #[derive(Debug, Serialize)] struct ErrorResponse { code: u16, @@ -59,7 +67,10 @@ struct ErrorResponse { } pub async fn handle_rejection(err: Rejection) -> Result { - let (code, message) = if let Some(e) = err.find::() { + let (code, message) = + if err.is_not_found() { + (StatusCode::NOT_FOUND, String::from("Not found")) + } else if let Some(e) = err.find::() { log::debug!("[Warp][Error] {:?}", e); ( @@ -77,6 +88,10 @@ pub async fn handle_rejection(err: Rejection) -> Result etablissement::error::Error::EtablissementNotFound => StatusCode::NOT_FOUND, _ => StatusCode::INTERNAL_SERVER_ERROR, }, + Error::StatusError { source } => match source { + update_metadata::error::Error::MetadataNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, }, e.to_string(), ) diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 66047db..dcd572c 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -4,8 +4,9 @@ pub mod common; use crate::models; use crate::update::{common::Config as DataConfig, update as update_data}; +use chrono::Utc; use common::{ - Context, EtablissementInnerResponse, EtablissementResponse, + Context, EtablissementInnerResponse, EtablissementResponse, StatusQueryString, UniteLegaleEtablissementInnerResponse, UniteLegaleInnerResponse, UniteLegaleResponse, UpdateOptions, UpdateResponse, }; @@ -49,6 +50,47 @@ async fn update(options: UpdateOptions, context: Context) -> Result Result { + let api_key = match &context.api_key { + Some(key) => key, + None => return Err(Error::MissingApiKeyError.into()), + }; + + if &query.api_key != api_key { + return Err(Error::ApiKeyError.into()); + } + + let connectors = context.builders.create(); + + let update_metadata = models::update_metadata::current_update(&connectors)?; + + Ok(warp::reply::json(&update_metadata)) +} + +async fn set_status_to_error( + query: StatusQueryString, + context: Context, +) -> Result { + let api_key = match &context.api_key { + Some(key) => key, + None => return Err(Error::MissingApiKeyError.into()), + }; + + if &query.api_key != api_key { + return Err(Error::ApiKeyError.into()); + } + + let connectors = context.builders.create(); + + models::update_metadata::error_update( + &connectors, + String::from("Process stopped manually."), + Utc::now(), + )?; + + Ok(warp::reply()) +} + async fn unites_legales(siren: String, context: Context) -> Result { if siren.len() != 9 { return Err(Error::InvalidData.into()); @@ -120,14 +162,34 @@ pub async fn run(addr: SocketAddr, context: Context) { .and_then(etablissements); log::info!("[Warp] Mount GET /v3/etablissements/"); + let admin_update_route = warp::path!("admin" / "update" / ..); + // POST /admin/update {json} - let admin_update_route = warp::post() - .and(warp::path!("admin" / "update")) + let update_route = warp::post() + .and(warp::path::end()) + .and(warp::body::content_length_limit(1024 * 32)) .and(warp::body::json::()) - .and(with_context(context)) + .and(with_context(context.clone())) .and_then(update); log::info!("[Warp] Mount POST /admin/update {{json}}"); + // GET /admin/update/status?api_key="" + let status_route = warp::get() + .and(warp::path!("status")) + .and(warp::query::()) + .and(with_context(context.clone())) + .and_then(status); + log::info!("[Warp] Mount GET /admin/update/status?api_key="); + + // POST /admin/update/status/error { api_key } + let status_error_route = warp::post() + .and(warp::path!("status" / "error")) + .and(warp::body::content_length_limit(1024 * 32)) + .and(warp::body::json::()) + .and(with_context(context)) + .and_then(set_status_to_error); + log::info!("[Warp] Mount POST /admin/update/status/error {{api_key}}"); + // Cors let cors = warp::cors() .allow_methods(&[Method::GET, Method::POST]) @@ -140,7 +202,7 @@ pub async fn run(addr: SocketAddr, context: Context) { .or(v3_etablissement_route) .or(v3_index), )) - .or(admin_update_route) + .or(admin_update_route.and(status_route.or(update_route).or(status_error_route))) .recover(error::handle_rejection) .with(cors); diff --git a/src/commands/update.rs b/src/commands/update.rs index 2f8c94a..eee1d06 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -1,7 +1,9 @@ use super::common::{CmdGroupType, FolderOptions}; use crate::connectors::ConnectorsBuilders; use crate::models::update_metadata::common::{Step, SyntheticGroupType}; -use crate::update::{common::Config, update, update_step}; +use crate::models::update_metadata::error_update; +use crate::update::{common::Config, error::Error, update, update_step}; +use chrono::Utc; #[derive(Clap, Debug)] pub struct UpdateFlags { @@ -46,6 +48,10 @@ enum UpdateSubCommand { /// Synchronise daily data from INSEE since the last modification #[clap(name = "sync-insee")] SyncInsee, + + /// Set a staled update process to error, use only if the process is really stopped + #[clap(name = "finish-error")] + FinishError, } pub async fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { @@ -73,6 +79,18 @@ pub async fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: Co UpdateSubCommand::SwapData => Step::SwapData, UpdateSubCommand::CleanFile => Step::CleanFile, UpdateSubCommand::SyncInsee => Step::SyncInsee, + UpdateSubCommand::FinishError => { + if let Err(err) = error_update( + &connectors, + "Process stopped manually.".to_string(), + Utc::now(), + ) { + let error: Error = err.into(); + error.exit() + } + + std::process::exit(0); + } }; update_step(step, synthetic_group_type, config, &mut connectors).await @@ -83,7 +101,7 @@ pub async fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: Co match summary_result { Ok(summary) => println!( "{}", - serde_json::to_string(&summary).expect("Unable to stringify summary") + serde_json::to_string_pretty(&summary).expect("Unable to stringify summary") ), Err(error) => error.exit(), } diff --git a/src/connectors/insee/error.rs b/src/connectors/insee/error.rs index 4d9c36e..628c016 100644 --- a/src/connectors/insee/error.rs +++ b/src/connectors/insee/error.rs @@ -4,6 +4,7 @@ custom_error! { pub InseeTokenError NetworkError { source: reqwest::Error } = "Unable to retrieve INSEE token (network error: {source})", ApiError = "Unable to retrieve INSEE token", MalformedError {source: serde_json::Error} = "Unable to read INSEE token ({source})", + InvalidError {source: reqwest::header::InvalidHeaderValue} = "Unable to use INSEE token in header ({source})", } custom_error! { pub InseeUpdateError diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs index f9b4e65..43f3c87 100644 --- a/src/connectors/insee/implementation.rs +++ b/src/connectors/insee/implementation.rs @@ -1,13 +1,12 @@ use super::error::InseeUpdateError; use super::types::{ etablissement::InseeEtablissementResponse, unite_legale::InseeUniteLegaleResponse, - InseeQueryParams, InseeResponse, + InseeCountQueryParams, InseeCountResponse, InseeQueryParams, InseeResponse, }; use super::Connector; use crate::models::etablissement::common::Etablissement; use crate::models::unite_legale::common::UniteLegale; -use chrono::{Duration, NaiveDateTime, Utc}; -use reqwest::header::{HeaderValue, ACCEPT, AUTHORIZATION}; +use chrono::NaiveDateTime; const MAX_CALL: u8 = 20; const MAX_DURATION: std::time::Duration = std::time::Duration::from_secs(60); @@ -15,11 +14,20 @@ const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; pub const INITIAL_CURSOR: &str = "*"; struct EndpointConfig { - token: String, - route: String, - query_field: String, + route: &'static str, + query_field: &'static str, } +const UNITES_LEGALES_ENDPOINT: EndpointConfig = EndpointConfig { + route: "siren", + query_field: "dateDernierTraitementUniteLegale", +}; + +const ETABLISSEMENTS_ENDPOINT: EndpointConfig = EndpointConfig { + route: "siret", + query_field: "dateDernierTraitementEtablissement", +}; + impl Connector { async fn wait_for_insee_limitation(&mut self) { if self.calls == 0 { @@ -37,6 +45,24 @@ impl Connector { } } + pub async fn get_total_unites_legales( + &mut self, + start_timestamp: NaiveDateTime, + ) -> Result { + self.wait_for_insee_limitation().await; + + get_total(&self.client, &UNITES_LEGALES_ENDPOINT, start_timestamp).await + } + + pub async fn get_total_etablissements( + &mut self, + start_timestamp: NaiveDateTime, + ) -> Result { + self.wait_for_insee_limitation().await; + + get_total(&self.client, &ETABLISSEMENTS_ENDPOINT, start_timestamp).await + } + pub async fn get_daily_unites_legales( &mut self, start_timestamp: NaiveDateTime, @@ -45,11 +71,8 @@ impl Connector { self.wait_for_insee_limitation().await; let (next_cursor, response) = get_daily_data::( - EndpointConfig { - token: self.token.clone(), - route: String::from("siren"), - query_field: String::from("dateDernierTraitementUniteLegale"), - }, + &self.client, + &UNITES_LEGALES_ENDPOINT, start_timestamp, cursor, ) @@ -76,11 +99,8 @@ impl Connector { self.wait_for_insee_limitation().await; let (next_cursor, response) = get_daily_data::( - EndpointConfig { - token: self.token.clone(), - route: String::from("siret"), - query_field: String::from("dateDernierTraitementEtablissement"), - }, + &self.client, + &ETABLISSEMENTS_ENDPOINT, start_timestamp, cursor, ) @@ -100,28 +120,21 @@ impl Connector { } } -fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime { - timestamp.max(Utc::now().naive_local() - Duration::days(31)) -} - async fn get_daily_data( - config: EndpointConfig, + client: &reqwest::Client, + config: &EndpointConfig, start_timestamp: NaiveDateTime, cursor: String, ) -> Result<(Option, Option), InseeUpdateError> { - let client = reqwest::Client::new(); - let url = format!("{}/{}", BASE_URL, config.route); let response = match client .get(&url) - .header(AUTHORIZATION, format!("Bearer {}", config.token)) - .header(ACCEPT, HeaderValue::from_static("application/json")) .query(&InseeQueryParams { q: format!( "{}:[{} TO *]", config.query_field, - get_minimum_timestamp_for_request(start_timestamp).format("%Y-%m-%dT%H:%M:%S") + start_timestamp.format("%Y-%m-%dT%H:%M:%S") ), nombre: 1000, curseur: cursor, @@ -153,3 +166,29 @@ async fn get_daily_data( Ok((next_cursor, Some(response))) } + +async fn get_total( + client: &reqwest::Client, + config: &EndpointConfig, + start_timestamp: NaiveDateTime, +) -> Result { + let url = format!("{}/{}", BASE_URL, config.route); + + let response = client + .get(&url) + .query(&InseeCountQueryParams { + q: format!( + "{}:[{} TO *]", + config.query_field, + start_timestamp.format("%Y-%m-%dT%H:%M:%S") + ), + nombre: 1, + champs: config.route, + }) + .send() + .await? + .json::() + .await?; + + Ok(response.header.total) +} diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index 361dc08..fb0471a 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -4,15 +4,15 @@ mod types; pub mod error; use error::InseeTokenError; -use reqwest::header::AUTHORIZATION; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION}; use serde::{Deserialize, Serialize}; use std::env; -use std::time::Instant; +use std::time::{Duration, Instant}; pub use implementation::INITIAL_CURSOR; pub struct Connector { - pub token: String, + client: reqwest::Client, calls: u8, started_at: Instant, } @@ -48,15 +48,36 @@ impl ConnectorBuilder { } pub async fn create(&self) -> Result { - self.generate_token().await.map(|token| Connector { - token, - calls: 0, - started_at: Instant::now(), + self.generate_token().await.and_then(|token| { + // Build headers + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(format!("Bearer {}", token).as_str())?, + ); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + + // Build client + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .default_headers(headers) + .build()?; + + Ok(Connector { + client, + calls: 0, + started_at: Instant::now(), + }) }) } async fn generate_token(&self) -> Result { - let client = reqwest::Client::new(); + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .build()?; + let response = client .post("https://api.insee.fr/token") .header(AUTHORIZATION, format!("Basic {}", self.credentials)) diff --git a/src/connectors/insee/types/mod.rs b/src/connectors/insee/types/mod.rs index 4caede8..4662eee 100644 --- a/src/connectors/insee/types/mod.rs +++ b/src/connectors/insee/types/mod.rs @@ -13,6 +13,13 @@ pub struct InseeQueryParams { pub tri: String, } +#[derive(Serialize)] +pub struct InseeCountQueryParams { + pub q: String, + pub nombre: u16, + pub champs: &'static str, +} + pub trait InseeResponse: DeserializeOwned { fn header(&self) -> Header; } @@ -27,6 +34,17 @@ pub struct Header { pub curseur_suivant: String, } +#[derive(Deserialize, Debug)] +pub struct CountHeader { + #[serde(default = "default_as_zero")] + pub total: u32, +} + +#[derive(Deserialize, Debug)] +pub struct InseeCountResponse { + pub header: CountHeader, +} + fn from_str_optional<'de, T, D>(deserializer: D) -> Result, D::Error> where T: FromStr, @@ -50,3 +68,7 @@ where fn default_as_false() -> bool { false } + +fn default_as_zero() -> u32 { + 0 +} diff --git a/src/main.rs b/src/main.rs index 89f611e..405b642 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,6 @@ mod connectors; mod models; mod update; -use chrono::Utc; use connectors::ConnectorsBuilders; use dotenv::dotenv; @@ -36,15 +35,6 @@ async fn main() { // Load database let connectors_builders = ConnectorsBuilders::new(); - // Close running updates - let connectors = connectors_builders.create(); - models::update_metadata::error_update( - &connectors, - String::from("Program unexpectedly closed"), - Utc::now(), - ) - .expect("Unable to terminate last unfinished update"); // Fail launch in case of error - // Run command commands::run(connectors_builders).await; } diff --git a/src/models/common.rs b/src/models/common.rs index b5e2c54..a619b63 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -10,6 +10,11 @@ pub trait UpdatableModel: Sync + Send { fn count_staging(&self, connectors: &Connectors) -> Result; fn insert_in_staging(&self, connectors: &Connectors, file_path: String) -> Result; fn swap(&self, connectors: &Connectors) -> Result<(), Error>; + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result; fn get_last_insee_synced_timestamp( &self, connectors: &Connectors, @@ -18,7 +23,8 @@ pub trait UpdatableModel: Sync + Send { &self, connectors: &mut Connectors, start_timestamp: NaiveDateTime, - ) -> Result; + cursor: String, + ) -> Result<(Option, usize), Error>; } custom_error! { pub Error diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 3df6c80..9c190c9 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -4,7 +4,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; -use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; +use crate::connectors::Connectors; use async_trait::async_trait; use chrono::NaiveDateTime; use common::Etablissement; @@ -114,6 +114,19 @@ impl UpdatableModel for EtablissementModel { }) } + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + Ok(insee.get_total_etablissements(start_timestamp).await?) + } + // SELECT date_dernier_traitement FROM etablissement WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; fn get_last_insee_synced_timestamp( &self, @@ -132,7 +145,8 @@ impl UpdatableModel for EtablissementModel { &self, connectors: &mut Connectors, start_timestamp: NaiveDateTime, - ) -> Result { + cursor: String, + ) -> Result<(Option, usize), UpdatableError> { let insee = connectors .insee .as_mut() @@ -140,74 +154,67 @@ impl UpdatableModel for EtablissementModel { let connection = connectors.local.pool.get()?; - let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); - let mut updated_count: usize = 0; - - while let Some(cursor) = current_cursor { - let (next_cursor, etablissements) = insee - .get_daily_etablissements(start_timestamp, cursor) - .await?; - - current_cursor = next_cursor; - - updated_count += diesel::insert_into(dsl::etablissement) - .values(&etablissements) - .on_conflict(dsl::siret) - .do_update() - .set(( - dsl::nic.eq(excluded(dsl::nic)), - dsl::siren.eq(excluded(dsl::siren)), - dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), - dsl::date_creation.eq(excluded(dsl::date_creation)), - dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), - dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), - dsl::activite_principale_registre_metiers - .eq(excluded(dsl::activite_principale_registre_metiers)), - dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), - dsl::etablissement_siege.eq(excluded(dsl::etablissement_siege)), - dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), - dsl::complement_adresse.eq(excluded(dsl::complement_adresse)), - dsl::numero_voie.eq(excluded(dsl::numero_voie)), - dsl::indice_repetition.eq(excluded(dsl::indice_repetition)), - dsl::type_voie.eq(excluded(dsl::type_voie)), - dsl::libelle_voie.eq(excluded(dsl::libelle_voie)), - dsl::code_postal.eq(excluded(dsl::code_postal)), - dsl::libelle_commune.eq(excluded(dsl::libelle_commune)), - dsl::libelle_commune_etranger.eq(excluded(dsl::libelle_commune_etranger)), - dsl::distribution_speciale.eq(excluded(dsl::distribution_speciale)), - dsl::code_commune.eq(excluded(dsl::code_commune)), - dsl::code_cedex.eq(excluded(dsl::code_cedex)), - dsl::libelle_cedex.eq(excluded(dsl::libelle_cedex)), - dsl::code_pays_etranger.eq(excluded(dsl::code_pays_etranger)), - dsl::libelle_pays_etranger.eq(excluded(dsl::libelle_pays_etranger)), - dsl::complement_adresse2.eq(excluded(dsl::complement_adresse2)), - dsl::numero_voie_2.eq(excluded(dsl::numero_voie_2)), - dsl::indice_repetition_2.eq(excluded(dsl::indice_repetition_2)), - dsl::type_voie_2.eq(excluded(dsl::type_voie_2)), - dsl::libelle_voie_2.eq(excluded(dsl::libelle_voie_2)), - dsl::code_postal_2.eq(excluded(dsl::code_postal_2)), - dsl::libelle_commune_2.eq(excluded(dsl::libelle_commune_2)), - dsl::libelle_commune_etranger_2.eq(excluded(dsl::libelle_commune_etranger_2)), - dsl::distribution_speciale_2.eq(excluded(dsl::distribution_speciale_2)), - dsl::code_commune_2.eq(excluded(dsl::code_commune_2)), - dsl::code_cedex_2.eq(excluded(dsl::code_cedex_2)), - dsl::libelle_cedex_2.eq(excluded(dsl::libelle_cedex_2)), - dsl::code_pays_etranger_2.eq(excluded(dsl::code_pays_etranger_2)), - dsl::libelle_pays_etranger_2.eq(excluded(dsl::libelle_pays_etranger_2)), - dsl::date_debut.eq(excluded(dsl::date_debut)), - dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), - dsl::enseigne_1.eq(excluded(dsl::enseigne_1)), - dsl::enseigne_2.eq(excluded(dsl::enseigne_2)), - dsl::enseigne_3.eq(excluded(dsl::enseigne_3)), - dsl::denomination_usuelle.eq(excluded(dsl::denomination_usuelle)), - dsl::activite_principale.eq(excluded(dsl::activite_principale)), - dsl::nomenclature_activite_principale - .eq(excluded(dsl::nomenclature_activite_principale)), - dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), - )) - .execute(&connection)?; - } + let (next_cursor, etablissements) = insee + .get_daily_etablissements(start_timestamp, cursor) + .await?; + + let updated_count = diesel::insert_into(dsl::etablissement) + .values(&etablissements) + .on_conflict(dsl::siret) + .do_update() + .set(( + dsl::nic.eq(excluded(dsl::nic)), + dsl::siren.eq(excluded(dsl::siren)), + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::activite_principale_registre_metiers + .eq(excluded(dsl::activite_principale_registre_metiers)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::etablissement_siege.eq(excluded(dsl::etablissement_siege)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::complement_adresse.eq(excluded(dsl::complement_adresse)), + dsl::numero_voie.eq(excluded(dsl::numero_voie)), + dsl::indice_repetition.eq(excluded(dsl::indice_repetition)), + dsl::type_voie.eq(excluded(dsl::type_voie)), + dsl::libelle_voie.eq(excluded(dsl::libelle_voie)), + dsl::code_postal.eq(excluded(dsl::code_postal)), + dsl::libelle_commune.eq(excluded(dsl::libelle_commune)), + dsl::libelle_commune_etranger.eq(excluded(dsl::libelle_commune_etranger)), + dsl::distribution_speciale.eq(excluded(dsl::distribution_speciale)), + dsl::code_commune.eq(excluded(dsl::code_commune)), + dsl::code_cedex.eq(excluded(dsl::code_cedex)), + dsl::libelle_cedex.eq(excluded(dsl::libelle_cedex)), + dsl::code_pays_etranger.eq(excluded(dsl::code_pays_etranger)), + dsl::libelle_pays_etranger.eq(excluded(dsl::libelle_pays_etranger)), + dsl::complement_adresse2.eq(excluded(dsl::complement_adresse2)), + dsl::numero_voie_2.eq(excluded(dsl::numero_voie_2)), + dsl::indice_repetition_2.eq(excluded(dsl::indice_repetition_2)), + dsl::type_voie_2.eq(excluded(dsl::type_voie_2)), + dsl::libelle_voie_2.eq(excluded(dsl::libelle_voie_2)), + dsl::code_postal_2.eq(excluded(dsl::code_postal_2)), + dsl::libelle_commune_2.eq(excluded(dsl::libelle_commune_2)), + dsl::libelle_commune_etranger_2.eq(excluded(dsl::libelle_commune_etranger_2)), + dsl::distribution_speciale_2.eq(excluded(dsl::distribution_speciale_2)), + dsl::code_commune_2.eq(excluded(dsl::code_commune_2)), + dsl::code_cedex_2.eq(excluded(dsl::code_cedex_2)), + dsl::libelle_cedex_2.eq(excluded(dsl::libelle_cedex_2)), + dsl::code_pays_etranger_2.eq(excluded(dsl::code_pays_etranger_2)), + dsl::libelle_pays_etranger_2.eq(excluded(dsl::libelle_pays_etranger_2)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::enseigne_1.eq(excluded(dsl::enseigne_1)), + dsl::enseigne_2.eq(excluded(dsl::enseigne_2)), + dsl::enseigne_3.eq(excluded(dsl::enseigne_3)), + dsl::denomination_usuelle.eq(excluded(dsl::denomination_usuelle)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; - Ok(updated_count) + Ok((next_cursor, updated_count)) } } diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index 1f93504..bdd947b 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -4,7 +4,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; -use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; +use crate::connectors::Connectors; use async_trait::async_trait; use chrono::NaiveDateTime; use common::UniteLegale; @@ -92,6 +92,19 @@ impl UpdatableModel for UniteLegaleModel { }) } + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + Ok(insee.get_total_unites_legales(start_timestamp).await?) + } + // SELECT date_dernier_traitement FROM unite_legale WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; fn get_last_insee_synced_timestamp( &self, @@ -110,7 +123,8 @@ impl UpdatableModel for UniteLegaleModel { &self, connectors: &mut Connectors, start_timestamp: NaiveDateTime, - ) -> Result { + cursor: String, + ) -> Result<(Option, usize), UpdatableError> { let insee = connectors .insee .as_mut() @@ -118,58 +132,51 @@ impl UpdatableModel for UniteLegaleModel { let connection = connectors.local.pool.get()?; - let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); - let mut updated_count: usize = 0; - - while let Some(cursor) = current_cursor { - let (next_cursor, unites_legales) = insee - .get_daily_unites_legales(start_timestamp, cursor) - .await?; - - current_cursor = next_cursor; - - updated_count += diesel::insert_into(dsl::unite_legale) - .values(&unites_legales) - .on_conflict(dsl::siren) - .do_update() - .set(( - dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), - dsl::unite_purgee.eq(excluded(dsl::unite_purgee)), - dsl::date_creation.eq(excluded(dsl::date_creation)), - dsl::sigle.eq(excluded(dsl::sigle)), - dsl::sexe.eq(excluded(dsl::sexe)), - dsl::prenom_1.eq(excluded(dsl::prenom_1)), - dsl::prenom_2.eq(excluded(dsl::prenom_2)), - dsl::prenom_3.eq(excluded(dsl::prenom_3)), - dsl::prenom_4.eq(excluded(dsl::prenom_4)), - dsl::prenom_usuel.eq(excluded(dsl::prenom_usuel)), - dsl::pseudonyme.eq(excluded(dsl::pseudonyme)), - dsl::identifiant_association.eq(excluded(dsl::identifiant_association)), - dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), - dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), - dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), - dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), - dsl::categorie_entreprise.eq(excluded(dsl::categorie_entreprise)), - dsl::annee_categorie_entreprise.eq(excluded(dsl::annee_categorie_entreprise)), - dsl::date_debut.eq(excluded(dsl::date_debut)), - dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), - dsl::nom.eq(excluded(dsl::nom)), - dsl::nom_usage.eq(excluded(dsl::nom_usage)), - dsl::denomination.eq(excluded(dsl::denomination)), - dsl::denomination_usuelle_1.eq(excluded(dsl::denomination_usuelle_1)), - dsl::denomination_usuelle_2.eq(excluded(dsl::denomination_usuelle_2)), - dsl::denomination_usuelle_3.eq(excluded(dsl::denomination_usuelle_3)), - dsl::categorie_juridique.eq(excluded(dsl::categorie_juridique)), - dsl::activite_principale.eq(excluded(dsl::activite_principale)), - dsl::nomenclature_activite_principale - .eq(excluded(dsl::nomenclature_activite_principale)), - dsl::nic_siege.eq(excluded(dsl::nic_siege)), - dsl::economie_sociale_solidaire.eq(excluded(dsl::economie_sociale_solidaire)), - dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), - )) - .execute(&connection)?; - } + let (next_cursor, unites_legales) = insee + .get_daily_unites_legales(start_timestamp, cursor) + .await?; + + let updated_count = diesel::insert_into(dsl::unite_legale) + .values(&unites_legales) + .on_conflict(dsl::siren) + .do_update() + .set(( + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::unite_purgee.eq(excluded(dsl::unite_purgee)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::sigle.eq(excluded(dsl::sigle)), + dsl::sexe.eq(excluded(dsl::sexe)), + dsl::prenom_1.eq(excluded(dsl::prenom_1)), + dsl::prenom_2.eq(excluded(dsl::prenom_2)), + dsl::prenom_3.eq(excluded(dsl::prenom_3)), + dsl::prenom_4.eq(excluded(dsl::prenom_4)), + dsl::prenom_usuel.eq(excluded(dsl::prenom_usuel)), + dsl::pseudonyme.eq(excluded(dsl::pseudonyme)), + dsl::identifiant_association.eq(excluded(dsl::identifiant_association)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::categorie_entreprise.eq(excluded(dsl::categorie_entreprise)), + dsl::annee_categorie_entreprise.eq(excluded(dsl::annee_categorie_entreprise)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::nom.eq(excluded(dsl::nom)), + dsl::nom_usage.eq(excluded(dsl::nom_usage)), + dsl::denomination.eq(excluded(dsl::denomination)), + dsl::denomination_usuelle_1.eq(excluded(dsl::denomination_usuelle_1)), + dsl::denomination_usuelle_2.eq(excluded(dsl::denomination_usuelle_2)), + dsl::denomination_usuelle_3.eq(excluded(dsl::denomination_usuelle_3)), + dsl::categorie_juridique.eq(excluded(dsl::categorie_juridique)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::nic_siege.eq(excluded(dsl::nic_siege)), + dsl::economie_sociale_solidaire.eq(excluded(dsl::economie_sociale_solidaire)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; - Ok(updated_count) + Ok((next_cursor, updated_count)) } } diff --git a/src/models/update_metadata/common.rs b/src/models/update_metadata/common.rs index b079ca1..75f1906 100644 --- a/src/models/update_metadata/common.rs +++ b/src/models/update_metadata/common.rs @@ -8,6 +8,21 @@ use diesel::sql_types::{Jsonb, Text}; use serde::{Deserialize, Serialize}; use std::io::Write; +#[derive(Queryable, Serialize)] +pub struct UpdateMetadata { + pub id: i32, + pub synthetic_group_type: SyntheticGroupType, + pub force: bool, + pub data_only: bool, + pub status: String, + pub summary: Option, + pub error: Option, + pub launched_timestamp: DateTime, + pub finished_timestamp: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + #[derive(Insertable)] #[table_name = "update_metadata"] pub struct LaunchUpdateMetadata { @@ -23,7 +38,7 @@ pub struct LaunchUpdateMetadata { pub struct FinishedUpdateMetadata { pub status: UpdateStatus, pub summary: UpdateSummary, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, } #[derive(AsChangeset)] @@ -57,7 +72,10 @@ pub struct UpdateGroupSummary { pub updated: bool, pub status_label: String, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, + pub planned_count: u32, + pub done_count: u32, + pub reference_timestamp: Option>, } #[derive(Deserialize, Serialize, Clone, Copy, Debug)] @@ -75,7 +93,7 @@ pub struct UpdateStepSummary { pub step: Step, pub updated: bool, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, pub groups: Vec, } @@ -84,7 +102,7 @@ pub struct UpdateStepSummary { pub struct UpdateSummary { pub updated: bool, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, pub steps: Vec, } diff --git a/src/models/update_metadata/mod.rs b/src/models/update_metadata/mod.rs index d1120cd..755665a 100644 --- a/src/models/update_metadata/mod.rs +++ b/src/models/update_metadata/mod.rs @@ -6,7 +6,7 @@ use crate::connectors::Connectors; use chrono::{DateTime, Utc}; use common::{ ErrorUpdateMetadata, FinishedUpdateMetadata, LaunchUpdateMetadata, SyntheticGroupType, - UpdateStatus, UpdateSummary, + UpdateMetadata, UpdateStatus, UpdateSummary, }; use diesel::prelude::*; use error::Error; @@ -51,6 +51,16 @@ pub fn launch_update( } } +pub fn progress_update(connectors: &Connectors, summary: UpdateSummary) -> Result { + let connection = connectors.local.pool.get()?; + + diesel::update(dsl::update_metadata.filter(dsl::status.eq(UpdateStatus::Launched))) + .set(dsl::summary.eq(summary)) + .execute(&connection) + .map(|count| count > 0) + .map_err(|error| error.into()) +} + pub fn finished_update(connectors: &Connectors, summary: UpdateSummary) -> Result { let connection = connectors.local.pool.get()?; let finished_timestamp = summary.finished_timestamp; @@ -58,7 +68,7 @@ pub fn finished_update(connectors: &Connectors, summary: UpdateSummary) -> Resul diesel::update(dsl::update_metadata.filter(dsl::status.eq(UpdateStatus::Launched))) .set(&FinishedUpdateMetadata { status: UpdateStatus::Finished, - summary: summary, + summary, finished_timestamp, }) .execute(&connection) @@ -83,3 +93,12 @@ pub fn error_update( .map(|count| count > 0) .map_err(|error| error.into()) } + +pub fn current_update(connectors: &Connectors) -> Result { + let connection = connectors.local.pool.get()?; + + dsl::update_metadata + .order(dsl::launched_timestamp.desc()) + .first::(&connection) + .map_err(|error| error.into()) +} diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index 8de39f4..6ebe329 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -1,11 +1,11 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; -use chrono::Utc; use std::fs::remove_file; use std::path::PathBuf; @@ -20,14 +20,18 @@ impl Action for CleanAction { Step::CleanFile } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { println!("[Clean] Cleaning {:#?}", group_type); - let started_timestamp = Utc::now(); + + summary_delegate.start(connectors, None, 2)?; + let mut updated = true; + let mut done_count = 2; let mut status_label = String::from("cleaned"); let metadata = group_metadata::get(connectors, group_type)?; @@ -45,25 +49,23 @@ impl Action for CleanAction { if let Err(error) = remove_file(zip_path) { println!("[Clean] Zip not deleted ({})", error); updated = false; + done_count -= 1; status_label = String::from("zip not deleted"); } if let Err(error) = remove_file(csv_path) { println!("[Clean] CSV not deleted ({})", error); updated = false; + done_count -= 1; status_label = String::from("csv not deleted"); } group_metadata::reset_staging_timestamps(connectors, group_type)?; + summary_delegate.finish(connectors, status_label, done_count, updated)?; + println!("[Clean] Finished cleaning of {:#?}", group_type); - Ok(UpdateGroupSummary { - group_type, - updated, - status_label, - started_timestamp, - finished_timestamp: Utc::now(), - }) + Ok(()) } } diff --git a/src/update/action/common.rs b/src/update/action/common.rs index 33fdf18..a315986 100644 --- a/src/update/action/common.rs +++ b/src/update/action/common.rs @@ -1,15 +1,17 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; #[async_trait] pub trait Action: Sync + Send { fn step(&self) -> Step; - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result; + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error>; } diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index 120f5ae..af96ff6 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -1,15 +1,17 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::stream::TryStreamExt; use reqwest::header::LAST_MODIFIED; use std::fs::create_dir_all; use std::path::PathBuf; +use std::time::Duration; use tokio::fs::File; use tokio_util::compat::FuturesAsyncReadCompatExt; @@ -24,13 +26,14 @@ impl Action for DownloadAction { Step::DownloadFile } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { println!("[Download] Downloading {:#?}", group_type); - let started_timestamp = Utc::now(); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -44,7 +47,15 @@ impl Action for DownloadAction { zip_path.set_extension("zip"); // Prepare file download - let resp = reqwest::get(metadata.url.as_str()) + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(3600)) + .build() + .map_err(|req_error| Error::DownloadError { req_error })?; + + let resp = client + .get(&metadata.url) + .send() .await .map_err(|req_error| Error::DownloadError { req_error })?; @@ -66,26 +77,30 @@ impl Action for DownloadAction { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if last_modified.le(&last_imported_timestamp) { println!("[Download] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } if let Some(staging_file_timestamp) = metadata.staging_file_timestamp { if last_modified.le(&staging_file_timestamp) { println!("[Download] {:#?} already downloaded", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already downloaded"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already downloaded"), + 0, + false, + )?; + + return Ok(()); } } } @@ -111,12 +126,8 @@ impl Action for DownloadAction { // Update staging file timestamp group_metadata::set_staging_file_timestamp(connectors, group_type, last_modified)?; - return Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("downloaded"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + summary_delegate.finish(connectors, String::from("downloaded"), 1, true)?; + + Ok(()) } } diff --git a/src/update/action/insert_stock.rs b/src/update/action/insert_stock.rs index 20d2ce6..fe04b7c 100644 --- a/src/update/action/insert_stock.rs +++ b/src/update/action/insert_stock.rs @@ -1,11 +1,11 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; -use chrono::Utc; use std::fs::canonicalize; use std::path::PathBuf; @@ -20,13 +20,14 @@ impl Action for InsertAction { Step::InsertData } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { println!("[Insert] Insert {:#?}", group_type); - let started_timestamp = Utc::now(); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -35,13 +36,10 @@ impl Action for InsertAction { Some(staging_csv_file_timestamp) => staging_csv_file_timestamp, None => { println!("[Insert] Nothing to insert for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to insert"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish(connectors, String::from("nothing to insert"), 0, false)?; + + return Ok(()); } }; @@ -51,25 +49,29 @@ impl Action for InsertAction { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { println!("[Insert] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } if staging_csv_file_timestamp.le(&staging_imported_timestamp) { println!("[Insert] {:#?} already inserted", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already inserted"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already inserted"), + 0, + false, + )?; + + return Ok(()); } } } @@ -78,8 +80,10 @@ impl Action for InsertAction { let mut csv_path = PathBuf::from(self.db_folder.clone()); csv_path.push(metadata.file_name); csv_path.set_extension("csv"); + let absolute_csv_path = canonicalize(csv_path) .map_err(|io_error| Error::InvalidComponentInCSVPath { io_error })?; + let csv_path_str = absolute_csv_path .into_os_string() .into_string() @@ -97,12 +101,8 @@ impl Action for InsertAction { println!("[Insert] Finished insert of {:#?}", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("inserted"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("inserted"), 1, true)?; + + Ok(()) } } diff --git a/src/update/action/mod.rs b/src/update/action/mod.rs index 86f7303..cff54bb 100644 --- a/src/update/action/mod.rs +++ b/src/update/action/mod.rs @@ -1,9 +1,9 @@ use super::common::Config; -use super::summary::Summary; +use super::error::Error; +use super::summary::SummaryStepDelegate; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary, UpdateStepSummary}; -use chrono::Utc; +use crate::models::update_metadata::common::Step; use common::Action; pub mod clean; @@ -14,34 +14,30 @@ pub mod swap; pub mod sync_insee; pub mod unzip_stock; -pub async fn execute_step( +pub async fn execute_step<'a>( step: Step, config: &Config, groups: &Vec, connectors: &mut Connectors, - summary: &mut Summary, -) { - let started_timestamp = Utc::now(); - let mut groups_summary: Vec = vec![]; + summary_delegate: &'a mut SummaryStepDelegate<'a>, +) -> Result<(), Error> { let action = build_action(config, step); + summary_delegate.start(connectors)?; + for group in groups { - match action.execute(*group, connectors).await { - Ok(group) => groups_summary.push(group), - Err(error) => { - summary.error = Some(error); - return; - } - } + action + .execute( + *group, + connectors, + &mut summary_delegate.group_delegate(*group), + ) + .await?; } - summary.steps.push(UpdateStepSummary { - step, - updated: groups_summary.iter().find(|&g| g.updated).is_some(), - started_timestamp, - finished_timestamp: Utc::now(), - groups: groups_summary, - }); + summary_delegate.finish(connectors)?; + + Ok(()) } fn build_action(config: &Config, step: Step) -> Box { diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index 4582525..32209bb 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -1,11 +1,11 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; -use chrono::Utc; pub struct SwapAction { pub force: bool, @@ -17,13 +17,14 @@ impl Action for SwapAction { Step::SwapData } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { - println!("[Insert] Swapping {:#?}", group_type); - let started_timestamp = Utc::now(); + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + println!("[Swap] Swapping {:#?}", group_type); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -32,13 +33,10 @@ impl Action for SwapAction { Some(staging_imported_timestamp) => staging_imported_timestamp, None => { println!("[Swap] Nothing to swap for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to swap"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish(connectors, String::from("nothing to swap"), 0, false)?; + + return Ok(()); } }; @@ -47,13 +45,15 @@ impl Action for SwapAction { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { println!("[Swap] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } } @@ -82,14 +82,10 @@ impl Action for SwapAction { staging_imported_timestamp, )?; - println!("[Insert] Swap of {:#?} finished", group_type); + println!("[Swap] Swap of {:#?} finished", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("swapped"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("swapped"), 1, true)?; + + Ok(()) } } diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index b07dfd6..6bfe432 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -1,11 +1,12 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; -use crate::connectors::Connectors; +use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; -use chrono::Utc; +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; pub struct SyncInseeAction {} @@ -15,22 +16,42 @@ impl Action for SyncInseeAction { Step::SyncInsee } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { println!("[SyncInsee] Syncing {:#?}", group_type); - let started_timestamp = Utc::now(); - let status_label: String; - let mut updated = false; // Use Insee connector only if present if connectors.insee.is_some() { let model = group_type.get_updatable_model(); - if let Some(timestamp) = model.get_last_insee_synced_timestamp(connectors)? { - let updated_count = model.update_daily_data(connectors, timestamp).await?; + if let Some(last_timestamp) = model.get_last_insee_synced_timestamp(connectors)? { + let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); + let mut updated_count = 0; + let timestamp = get_minimum_timestamp_for_request(last_timestamp); + + let planned_count = model.get_total_count(connectors, timestamp).await?; + + summary_delegate.start( + connectors, + Some(DateTime::::from_utc(timestamp, Utc)), + planned_count, + )?; + + while let Some(cursor) = current_cursor { + let (next_cursor, inserted_count) = model + .update_daily_data(connectors, timestamp, cursor) + .await?; + + current_cursor = next_cursor; + updated_count += inserted_count; + + summary_delegate.progress(connectors, updated_count as u32)?; + } + println!("[SyncInsee] {} {:#?} synced", updated_count, group_type); group_metadata::set_last_insee_synced_timestamp( @@ -39,22 +60,37 @@ impl Action for SyncInseeAction { Utc::now(), )?; - updated = updated_count > 0; - status_label = String::from("synced"); + summary_delegate.finish( + connectors, + String::from("synced"), + updated_count as u32, + updated_count > 0, + )?; } else { - status_label = String::from("missing last treatment date"); + summary_delegate.finish( + connectors, + String::from("missing last treatment date"), + 0, + false, + )?; } } else { - status_label = String::from("no insee connector configured"); + summary_delegate.finish( + connectors, + String::from("no insee connector configured"), + 0, + false, + )?; } println!("[SyncInsee] Syncing of {:#?} done", group_type); - Ok(UpdateGroupSummary { - group_type, - updated, - status_label, - started_timestamp, - finished_timestamp: Utc::now(), - }) + + Ok(()) } } + +fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime { + // Uncomment the next line to force custom date to be used + // return NaiveDateTime::parse_from_str("2020-06-25 12:00:00", "%Y-%m-%d %H:%M:%S").unwrap(); + timestamp.max(Utc::now().naive_local() - Duration::days(31)) +} diff --git a/src/update/action/unzip_stock.rs b/src/update/action/unzip_stock.rs index b2ea4d9..43e26c0 100644 --- a/src/update/action/unzip_stock.rs +++ b/src/update/action/unzip_stock.rs @@ -1,11 +1,11 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use crate::models::update_metadata::common::Step; use async_trait::async_trait; -use chrono::Utc; use std::fs::{create_dir_all, set_permissions, File, Permissions}; use std::io; use std::path::PathBuf; @@ -22,13 +22,14 @@ impl Action for UnzipAction { Step::UnzipFile } - async fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, connectors: &mut Connectors, - ) -> Result { + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { println!("[Unzip] Unzipping {:#?}", group_type); - let started_timestamp = Utc::now(); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -37,13 +38,10 @@ impl Action for UnzipAction { Some(staging_file_timestamp) => staging_file_timestamp, None => { println!("[Unzip] Nothing to unzip for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to unzip"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish(connectors, String::from("nothing to unzip"), 0, false)?; + + return Ok(()); } }; @@ -53,25 +51,29 @@ impl Action for UnzipAction { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_csv_file_timestamp.le(&last_imported_timestamp) { println!("[Unzip] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } if staging_file_timestamp.le(&staging_csv_file_timestamp) { println!("[Unzip] {:#?} already unzipped", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already unzipped"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + + summary_delegate.finish( + connectors, + String::from("already unzipped"), + 0, + false, + )?; + + return Ok(()); } } } @@ -136,12 +138,8 @@ impl Action for UnzipAction { println!("[Unzip] Unzip of {:#?} finished", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("unzipped"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("unzipped"), 1, true)?; + + Ok(()) } } diff --git a/src/update/mod.rs b/src/update/mod.rs index 6744b57..7584b38 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -5,7 +5,6 @@ use action::execute_step; use chrono::Utc; use common::Config; use error::Error; -use summary::Summary; pub mod action; pub mod common; @@ -43,50 +42,38 @@ async fn execute_workflow( config: Config, connectors: &mut Connectors, ) -> Result { - // Register start - update_metadata::launch_update( + // Start + println!("[Update] Starting"); + + // Execute workflow + let mut summary = UpdateSummary::new(); + + summary.start( connectors, synthetic_group_type, config.force, config.data_only, )?; - // Start - println!("[Update] Starting"); - let started_timestamp = Utc::now(); - - // Execute workflow - let mut summary = Summary::new(); - for step in workflow.into_iter() { execute_step( step, &config, &synthetic_group_type.into(), connectors, - &mut summary, + &mut summary.step_delegate(step), ) - .await; - - if summary.error.is_some() { - break; - } + .await + .or_else(|error| { + update_metadata::error_update(connectors, error.to_string(), Utc::now())?; + Err(error) + })?; } - if let Some(error) = summary.error { - update_metadata::error_update(connectors, error.to_string(), Utc::now())?; - return Err(error); - } + summary.finish(connectors)?; // End println!("[Update] Finished"); - let summary = UpdateSummary { - updated: summary.steps.iter().find(|&s| s.updated).is_some(), - started_timestamp, - finished_timestamp: Utc::now(), - steps: summary.steps, - }; - update_metadata::finished_update(connectors, summary.clone())?; Ok(summary) } diff --git a/src/update/summary.rs b/src/update/summary.rs index e3fd740..8454198 100644 --- a/src/update/summary.rs +++ b/src/update/summary.rs @@ -1,16 +1,157 @@ -use super::error::Error; -use crate::models::update_metadata::common::UpdateStepSummary; +use crate::connectors::Connectors; +use crate::models::group_metadata::common::GroupType; +use crate::models::update_metadata::{ + self, + common::{Step, SyntheticGroupType, UpdateGroupSummary, UpdateStepSummary, UpdateSummary}, + error::Error, +}; +use chrono::{DateTime, Utc}; -pub struct Summary { - pub error: Option, - pub steps: Vec, +pub struct SummaryStepDelegate<'a> { + _step: Step, + summary: &'a mut UpdateSummary, } -impl Summary { +pub struct SummaryGroupDelegate<'a, 'b> { + _group: GroupType, + step_delegate: &'b mut SummaryStepDelegate<'a>, +} + +impl UpdateSummary { pub fn new() -> Self { - Summary { - error: None, + UpdateSummary { steps: vec![], + updated: false, + started_timestamp: Utc::now(), + finished_timestamp: None, } } + + pub fn step_delegate<'a>(&'a mut self, step: Step) -> SummaryStepDelegate<'a> { + let step_summary = UpdateStepSummary { + step, + updated: false, + started_timestamp: Utc::now(), + finished_timestamp: None, + groups: vec![], + }; + + self.steps.insert(0, step_summary); + + SummaryStepDelegate { + _step: step, + summary: self, + } + } + + pub fn start( + &mut self, + connectors: &Connectors, + synthetic_group: SyntheticGroupType, + force: bool, + data_only: bool, + ) -> Result<(), Error> { + update_metadata::launch_update(connectors, synthetic_group, force, data_only).map( + |date| { + self.started_timestamp = date; + Ok(()) + }, + )? + } + + pub fn finish(&mut self, connectors: &Connectors) -> Result<(), Error> { + self.finished_timestamp = Some(Utc::now()); + self.updated = self.steps.iter().find(|s| s.updated).is_some(); + + update_metadata::finished_update(connectors, self.clone()).map(|_| Ok(()))? + } +} + +impl<'a> SummaryStepDelegate<'a> { + pub fn group_delegate<'b>(&'b mut self, group: GroupType) -> SummaryGroupDelegate<'a, 'b> { + if let Some(step) = self.summary.steps.first_mut() { + let group_summary = UpdateGroupSummary { + group_type: group, + updated: false, + status_label: String::from("initialized"), + started_timestamp: Utc::now(), + finished_timestamp: None, + planned_count: 0, + done_count: 0, + reference_timestamp: None, + }; + + step.groups.insert(0, group_summary); + } + + SummaryGroupDelegate { + _group: group, + step_delegate: self, + } + } + + pub fn start(&self, connectors: &Connectors) -> Result<(), Error> { + update_metadata::progress_update(connectors, self.summary.clone()).map(|_| Ok(()))? + } + + pub fn finish(&mut self, connectors: &Connectors) -> Result<(), Error> { + if let Some(step_summary) = self.summary.steps.first_mut() { + step_summary.finished_timestamp = Some(Utc::now()); + step_summary.updated = step_summary.groups.iter().find(|g| g.updated).is_some(); + } + + update_metadata::progress_update(connectors, self.summary.clone()).map(|_| Ok(()))? + } +} + +impl<'a, 'b> SummaryGroupDelegate<'a, 'b> { + fn get_current_mut(&mut self) -> Option<&mut UpdateGroupSummary> { + match self.step_delegate.summary.steps.first_mut() { + Some(step_summary) => step_summary.groups.first_mut(), + None => None, + } + } + + pub fn start( + &mut self, + connectors: &Connectors, + reference_timestamp: Option>, + planned_count: u32, + ) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.reference_timestamp = reference_timestamp; + group_summary.planned_count = planned_count; + group_summary.status_label = String::from("in progress") + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } + + pub fn progress(&mut self, connectors: &Connectors, done_count: u32) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.done_count = done_count; + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } + + pub fn finish( + &mut self, + connectors: &Connectors, + status_label: String, + done_count: u32, + updated: bool, + ) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.status_label = status_label; + group_summary.done_count = done_count; + group_summary.updated = updated; + group_summary.finished_timestamp = Some(Utc::now()); + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } } From 506f5bc446461557e0e1e2d1c22e95a58d52c34e Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Fri, 26 Jun 2020 16:11:47 +0200 Subject: [PATCH 38/69] Upgrade dependencies --- Cargo.lock | 185 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 124 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25e0f35..acf282b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,24 +2,24 @@ # It is not intended for manual editing. [[package]] name = "adler32" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" [[package]] name = "aho-corasick" -version = "0.7.10" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" dependencies = [ "memchr", ] [[package]] name = "async-trait" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cb5d814ab2a47fd66d3266e9efccb53ca4c740b7451043b8ffcf9a6208f3f8" +checksum = "a265e3abeffdce30b2e26b7a11b222fe37c6067404001b434101457d0385eb92" dependencies = [ "proc-macro2", "quote", @@ -34,7 +34,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -57,9 +57,9 @@ checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" [[package]] name = "base64" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d1ccbaf7d9ec9537465a97bf19edc1a4e158ecb49fc16178202238c569cc42" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" [[package]] name = "bitflags" @@ -118,9 +118,12 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "bytes" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" +checksum = "118cf036fbb97d0816e3c34b2d7a1e8cfc60f68fcf63d550ddbe9bd5f59c213b" +dependencies = [ + "loom", +] [[package]] name = "bzip2" @@ -145,9 +148,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.54" +version = "1.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" +checksum = "b1be3409f94d7bdceeb5f5fac551039d9b3f00e25da7a74fc4d33400a0d96368" [[package]] name = "cfg-if" @@ -292,9 +295,9 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "dtoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "encoding_rs" @@ -474,6 +477,19 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" +dependencies = [ + "cc", + "libc", + "log 0.4.8", + "rustc_version", + "winapi 0.3.9", +] + [[package]] name = "generic-array" version = "0.12.3" @@ -519,7 +535,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" dependencies = [ - "base64 0.12.1", + "base64 0.12.3", "bitflags", "bytes", "headers-core", @@ -549,9 +565,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91780f809e750b0a89f5544be56617ff6b1227ee485bcb06ebe10cdf89bd3b71" +checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" dependencies = [ "libc", ] @@ -669,9 +685,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" @@ -731,6 +747,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "loom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls 0.1.2", +] + [[package]] name = "matches" version = "0.1.8" @@ -885,7 +912,7 @@ checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" dependencies = [ "cfg-if", "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -921,9 +948,9 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "openssl" -version = "0.10.29" +version = "0.10.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", "cfg-if", @@ -979,7 +1006,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1029,18 +1056,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75373ff9037d112bb19bc61333a06a159eaeb217660dcfbea7d88e1db823919" +checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b4b44893d3c370407a1d6a5cfde7c41ae0478e31c516c85f67eb3adc51be6d" +checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" dependencies = [ "proc-macro2", "quote", @@ -1130,9 +1157,9 @@ checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" [[package]] name = "proc-macro-nested" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afe1bd463b9e9ed51d0e0f0b50b6b146aec855c56fd182bb242388710a9b6de" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" @@ -1185,7 +1212,7 @@ dependencies = [ "rand_os", "rand_pcg", "rand_xorshift", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1280,7 +1307,7 @@ checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" dependencies = [ "libc", "rand_core 0.4.2", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1294,7 +1321,7 @@ dependencies = [ "libc", "rand_core 0.4.2", "rdrand", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1351,11 +1378,11 @@ checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "remove_dir_all" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1364,7 +1391,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" dependencies = [ - "base64 0.12.1", + "base64 0.12.3", "bytes", "encoding_rs", "futures-core", @@ -1393,6 +1420,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1412,7 +1448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1424,6 +1460,12 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + [[package]] name = "scoped-tls" version = "1.0.0" @@ -1459,20 +1501,35 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" -version = "1.0.111" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.111" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1623,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1577,9 +1634,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.31" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6" +checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" dependencies = [ "proc-macro2", "quote", @@ -1608,7 +1665,7 @@ dependencies = [ "rand 0.7.3", "redox_syscall", "remove_dir_all", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1645,9 +1702,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" + [[package]] name = "tokio" version = "0.2.21" @@ -1790,11 +1853,11 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" dependencies = [ - "smallvec", + "tinyvec", ] [[package]] @@ -1811,9 +1874,9 @@ checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "url" @@ -1828,9 +1891,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "1.0.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" [[package]] name = "utf-8" @@ -1888,7 +1951,7 @@ dependencies = [ "mime_guess 2.0.3", "multipart", "pin-project", - "scoped-tls", + "scoped-tls 1.0.0", "serde", "serde_json", "serde_urlencoded", @@ -1990,9 +2053,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -2016,7 +2079,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -2031,7 +2094,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -2046,9 +2109,9 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df134e83b8f0f8153a094c7b0fd79dfebe437f1d76e7715afa18ed95ebe2fd7" +checksum = "58287c28d78507f5f91f2a4cf1e8310e2c76fd4c6932f93ac60fd1ceb402db7d" dependencies = [ "bzip2", "crc32fast", From a96fe1df9759341c150af99cb7f94d7caa1ddafa Mon Sep 17 00:00:00 2001 From: Ludovic Douziech Date: Fri, 26 Jun 2020 16:50:19 +0200 Subject: [PATCH 39/69] Update README --- README.md | 18 +++++++++++------- src/commands/mod.rs | 4 ++-- src/commands/serve/mod.rs | 2 +- src/commands/update.rs | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 384bf83..e97a523 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ createdb --owner=sirene sirene ``` > sirene -h +Sirene service used to update data in database and serve it through a HTTP REST API + USAGE: sirene [OPTIONS] @@ -43,8 +45,8 @@ FLAGS: OPTIONS: --db-folder Path to the file storage folder for the database, you can set in environment - variable as DB_FOLDER Could be the same as FILE_FOLDER if this app and the - database are on the same file system Files copied by this app inside FILE_FOLDER + variable as DB_FOLDER. Could be the same as FILE_FOLDER if this app and the + database are on the same file system. Files copied by this app inside FILE_FOLDER must be visible by the database in DB_FOLDER --file-folder Path to the file storage folder for this app, you can set in environment variable as FILE_FOLDER @@ -59,6 +61,9 @@ SUBCOMMANDS: **> sirene serve --help** ``` +sirene-serve +Serve data from database to /unites_legales/ and /etablissements/ + USAGE: sirene serve [OPTIONS] @@ -69,8 +74,8 @@ FLAGS: OPTIONS: -k, --api-key API key needed to allow maintenance operation from HTTP, you can set in environment variable as API_KEY - --env production, staging or development, will change log level, you can set in environment - variable as SIRENE_ENV + --env Configure log level, you can set in environment variable as SIRENE_ENV [possible values: + development, staging, production] -h, --host Listen this host, you can set in environment variable as HOST -p, --port Listen this port, you can set in environment variable as PORT ``` @@ -78,15 +83,14 @@ OPTIONS: **> sirene update --help** ``` -sirene-update +sirene-update Update data from CSV source files USAGE: sirene update [FLAGS] [SUBCOMMAND] ARGS: - Configure which part will be updated, unites-legales, etablissements or all [possible values: - unites-legales, etablissements, all] + Configure which part will be updated [possible values: unites-legales, etablissements, all] FLAGS: --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete it diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 2ef2b54..9bbf086 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -22,8 +22,8 @@ struct Opts { #[clap(long = "file-folder")] file_folder: Option, - /// Path to the file storage folder for the database, you can set in environment variable as DB_FOLDER - /// Could be the same as FILE_FOLDER if this app and the database are on the same file system + /// Path to the file storage folder for the database, you can set in environment variable as DB_FOLDER. + /// Could be the same as FILE_FOLDER if this app and the database are on the same file system. /// Files copied by this app inside FILE_FOLDER must be visible by the database in DB_FOLDER #[clap(long = "db-folder")] db_folder: Option, diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index a48e2b6..5660b30 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -8,7 +8,7 @@ use std::net::ToSocketAddrs; #[derive(Clap, Debug)] pub struct ServeFlags { - /// production, staging or development, will change log level, you can set in environment variable as SIRENE_ENV + /// Configure log level, you can set in environment variable as SIRENE_ENV #[clap(arg_enum, long = "env")] environment: Option, diff --git a/src/commands/update.rs b/src/commands/update.rs index eee1d06..aba18a9 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -7,7 +7,7 @@ use chrono::Utc; #[derive(Clap, Debug)] pub struct UpdateFlags { - /// Configure which part will be updated, unites-legales, etablissements or all + /// Configure which part will be updated #[clap(arg_enum)] group_type: CmdGroupType, From 46042da5834513d709d725f6de01cdc5a7d57a0a Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 30 Jun 2020 15:14:43 +0200 Subject: [PATCH 40/69] Use configurable pool size --- .env.sample | 3 ++- src/connectors/local.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index d21bd92..7e79ab9 100644 --- a/.env.sample +++ b/.env.sample @@ -4,7 +4,8 @@ HOST=localhost PORT=3000 API_KEY= DATABASE_URL=postgresql://sirene:sirenepw@127.0.0.1:5432/sirene +DATABASE_POOL_SIZE=100 TEMP_FOLDER=./data/temp FILE_FOLDER=./data/files DB_FOLDER=./data/files -INSEE_CREDENTIALS=Base64(consumer-key:consumer-secret) \ No newline at end of file +INSEE_CREDENTIALS=Base64(consumer-key:consumer-secret) diff --git a/src/connectors/local.rs b/src/connectors/local.rs index ac0b141..7670fd4 100644 --- a/src/connectors/local.rs +++ b/src/connectors/local.rs @@ -17,11 +17,12 @@ pub struct ConnectorBuilder { impl ConnectorBuilder { pub fn new() -> ConnectorBuilder { let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let pool_size = env::var("DATABASE_POOL_SIZE").ok().and_then(|s| s.parse::().ok()).unwrap_or(15); let manager = ConnectionManager::::new(database_url.clone()); let builder = ConnectorBuilder { pool: Pool::builder() - .max_size(15) + .max_size(pool_size) .build(manager) .expect(&format!("Error connecting to {}", database_url)), }; From 95e8f8ccc49efb972c3a3355aeb180bdd90add24 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 30 Jun 2020 15:18:23 +0200 Subject: [PATCH 41/69] Share database connection --- src/commands/serve/runner/error.rs | 5 +++-- src/commands/serve/runner/mod.rs | 26 ++++++++++++++++++-------- src/connectors/local.rs | 7 ++++++- src/models/etablissement/mod.rs | 17 +++++++---------- src/models/unite_legale/mod.rs | 7 +++---- 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index fe031df..24b6639 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -10,6 +10,7 @@ custom_error! { pub Error InvalidData = "Invalid data", MissingApiKeyError = "[Admin] Missing API key in configuration", ApiKeyError = "[Admin] Wrong API key", + LocalConnectionFailed{source: r2d2::Error} = "Unable to connect to local database ({source}).", UpdateConnectorError {source: ConnectorError} = "[Update] Error while creating connector: {source}", UpdateError {source: InternalUpdateError} = "[Update] {source}", UniteLegaleError {source: unite_legale::error::Error} = "[UniteLegale] {source}", @@ -67,8 +68,7 @@ struct ErrorResponse { } pub async fn handle_rejection(err: Rejection) -> Result { - let (code, message) = - if err.is_not_found() { + let (code, message) = if err.is_not_found() { (StatusCode::NOT_FOUND, String::from("Not found")) } else if let Some(e) = err.find::() { log::debug!("[Warp][Error] {:?}", e); @@ -78,6 +78,7 @@ pub async fn handle_rejection(err: Rejection) -> Result Error::InvalidData => StatusCode::BAD_REQUEST, Error::MissingApiKeyError => StatusCode::UNAUTHORIZED, Error::ApiKeyError => StatusCode::UNAUTHORIZED, + Error::LocalConnectionFailed { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, Error::UpdateConnectorError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, Error::UpdateError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, Error::UniteLegaleError { source } => match source { diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index dcd572c..281cabf 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -97,11 +97,16 @@ async fn unites_legales(siren: String, context: Context) -> Result Result>; + pub struct Connector { pub pool: Pool>, } @@ -17,7 +19,10 @@ pub struct ConnectorBuilder { impl ConnectorBuilder { pub fn new() -> ConnectorBuilder { let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let pool_size = env::var("DATABASE_POOL_SIZE").ok().and_then(|s| s.parse::().ok()).unwrap_or(15); + let pool_size = env::var("DATABASE_POOL_SIZE") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(15); let manager = ConnectionManager::::new(database_url.clone()); let builder = ConnectorBuilder { diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 9c190c9..505600c 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -4,7 +4,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; -use crate::connectors::Connectors; +use crate::connectors::{local::Connection, Connectors}; use async_trait::async_trait; use chrono::NaiveDateTime; use common::Etablissement; @@ -13,33 +13,30 @@ use diesel::prelude::*; use diesel::sql_query; use error::Error; -pub fn get(connectors: &Connectors, siret: &String) -> Result { - let connection = connectors.local.pool.get()?; +pub fn get(connection: &Connection, siret: &String) -> Result { dsl::etablissement .find(siret) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } pub fn get_with_siren( - connectors: &Connectors, + connection: &Connection, siren: &String, ) -> Result, Error> { - let connection = connectors.local.pool.get()?; dsl::etablissement .filter(dsl::siren.eq(siren)) - .load::(&connection) + .load::(connection) .map_err(|error| error.into()) } pub fn get_siege_with_siren( - connectors: &Connectors, + connection: &Connection, siren: &String, ) -> Result { - let connection = connectors.local.pool.get()?; dsl::etablissement .filter(dsl::siren.eq(siren).and(dsl::etablissement_siege.eq(true))) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index bdd947b..2681d3a 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -4,7 +4,7 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; -use crate::connectors::Connectors; +use crate::connectors::{local::Connection, Connectors}; use async_trait::async_trait; use chrono::NaiveDateTime; use common::UniteLegale; @@ -13,11 +13,10 @@ use diesel::prelude::*; use diesel::sql_query; use error::Error; -pub fn get(connectors: &Connectors, siren: &String) -> Result { - let connection = connectors.local.pool.get()?; +pub fn get(connection: &Connection, siren: &String) -> Result { dsl::unite_legale .find(siren) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } From 9c64ba392e2d01e533543567932b56bd842797b7 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 30 Jun 2020 15:41:10 +0200 Subject: [PATCH 42/69] Upgrade dependencies --- Cargo.lock | 60 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acf282b..2837956 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,10 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] -name = "adler32" -version = "1.1.0" +name = "adler" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" +checksum = "ccc9a9dd069569f212bc4330af9f17c4afb5e8ce185e83dbb14f1349dda18b10" [[package]] name = "aho-corasick" @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1be3409f94d7bdceeb5f5fac551039d9b3f00e25da7a74fc4d33400a0d96368" +checksum = "77c1f1d60091c1b73e2b1f4560ab419204b178e625fa945ded7b660becd2bd46" [[package]] name = "cfg-if" @@ -329,9 +329,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" dependencies = [ "cfg-if", "crc32fast", @@ -691,9 +691,9 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" +checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" dependencies = [ "wasm-bindgen", ] @@ -830,11 +830,11 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.3.7" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" dependencies = [ - "adler32", + "adler", ] [[package]] @@ -1538,9 +1538,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2c5d7e739bc07a3e73381a39d61fdb5f671c60c1df26a130690665803d8226" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" dependencies = [ "itoa", "ryu", @@ -1868,9 +1868,9 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" @@ -1969,9 +1969,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" +checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" dependencies = [ "cfg-if", "serde", @@ -1981,9 +1981,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" +checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" dependencies = [ "bumpalo", "lazy_static", @@ -1996,9 +1996,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6" +checksum = "dba48d66049d2a6cc8488702e7259ab7afc9043ad0dc5448444f46f2a453b362" dependencies = [ "cfg-if", "js-sys", @@ -2008,9 +2008,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" +checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2018,9 +2018,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" +checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" dependencies = [ "proc-macro2", "quote", @@ -2031,15 +2031,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.63" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" +checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" [[package]] name = "web-sys" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" +checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" dependencies = [ "js-sys", "wasm-bindgen", From 33193de3b9774ff6baf6d053fbdd2259d55017b9 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 30 Jun 2020 16:08:24 +0200 Subject: [PATCH 43/69] Enable threading --- Cargo.lock | 11 +++++++++++ Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 2837956..b95e86b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -934,6 +934,16 @@ dependencies = [ "autocfg 1.0.0", ] +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.4.0" @@ -1724,6 +1734,7 @@ dependencies = [ "lazy_static", "memchr", "mio", + "num_cpus", "pin-project-lite", "slab", "tokio-macros", diff --git a/Cargo.toml b/Cargo.toml index af01277..e9fefa5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ r2d2 = "0.8" reqwest = { version = "0.10", features = ["json", "stream"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -tokio = { version = "0.2", features = ["macros", "io-util", "fs", "time"] } +tokio = { version = "0.2", features = ["macros", "io-util", "fs", "rt-threaded", "time"] } tokio-util = { version = "0.3", features = ["compat"] } warp = "0.2" zip = "0.5" From a716acdef6d27d39490b2e7b1f41e987a9124422 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 14:50:22 +0200 Subject: [PATCH 44/69] Upgrade dependencies --- Cargo.lock | 376 ++++++++++++++++++++++++++++------------------------- 1 file changed, 196 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b95e86b..8b05b2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "adler" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc9a9dd069569f212bc4330af9f17c4afb5e8ce185e83dbb14f1349dda18b10" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" [[package]] name = "aho-corasick" @@ -17,9 +17,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.36" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a265e3abeffdce30b2e26b7a11b222fe37c6067404001b434101457d0385eb92" +checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" dependencies = [ "proc-macro2", "quote", @@ -45,9 +45,9 @@ checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" [[package]] name = "autocfg" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "base64" @@ -118,12 +118,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "bytes" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118cf036fbb97d0816e3c34b2d7a1e8cfc60f68fcf63d550ddbe9bd5f59c213b" -dependencies = [ - "loom", -] +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bzip2" @@ -148,9 +145,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c1f1d60091c1b73e2b1f4560ab419204b178e625fa945ded7b660becd2bd46" +checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" [[package]] name = "cfg-if" @@ -160,9 +157,9 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" dependencies = [ "num-integer", "num-traits", @@ -211,6 +208,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -301,9 +307,9 @@ checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "encoding_rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ "cfg-if", ] @@ -316,7 +322,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", - "log 0.4.8", + "log 0.4.11", "regex", "termcolor", ] @@ -329,9 +335,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" +checksum = "766d0e77a2c1502169d4a93ff3b8c15a71fd946cd0126309752104e5f3c46d94" dependencies = [ "cfg-if", "crc32fast", @@ -477,19 +483,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generator" -version = "0.6.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" -dependencies = [ - "cc", - "libc", - "log 0.4.8", - "rustc_version", - "winapi 0.3.9", -] - [[package]] name = "generic-array" version = "0.12.3" @@ -507,14 +500,14 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] name = "h2" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" +checksum = "993f9e0baeed60001cf565546b0d3dbe6a6ad23f2bd31644a133c641eccf6d53" dependencies = [ "bytes", "fnv", @@ -523,10 +516,19 @@ dependencies = [ "futures-util", "http", "indexmap", - "log 0.4.8", "slab", "tokio", "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" +dependencies = [ + "autocfg 1.0.1", ] [[package]] @@ -565,9 +567,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9586eedd4ce6b3c498bc3b4dd92fc9f11166aa908a914071953768066c67909" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" dependencies = [ "libc", ] @@ -610,9 +612,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.6" +version = "0.13.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f" +checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" dependencies = [ "bytes", "futures-channel", @@ -623,20 +625,20 @@ dependencies = [ "http-body", "httparse", "itoa", - "log 0.4.8", "pin-project", "socket2", "time", "tokio", "tower-service", + "tracing", "want", ] [[package]] name = "hyper-tls" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" dependencies = [ "bytes", "hyper", @@ -658,11 +660,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9" dependencies = [ - "autocfg 1.0.0", + "autocfg 1.0.1", + "hashbrown", ] [[package]] @@ -674,6 +677,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "instant" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" + [[package]] name = "iovec" version = "0.1.4" @@ -683,6 +692,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" version = "0.4.6" @@ -691,9 +706,9 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.41" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" +checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" dependencies = [ "wasm-bindgen", ] @@ -716,15 +731,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.71" +version = "0.2.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" +checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" [[package]] name = "lock_api" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" dependencies = [ "scopeguard", ] @@ -735,27 +750,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.8", + "log 0.4.11", ] [[package]] name = "log" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "loom" -version = "0.3.4" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ "cfg-if", - "generator", - "scoped-tls 0.1.2", ] [[package]] @@ -830,9 +834,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" dependencies = [ "adler", ] @@ -849,7 +853,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.8", + "log 0.4.11", "miow", "net2", "slab", @@ -876,7 +880,7 @@ checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" dependencies = [ "buf_redux", "httparse", - "log 0.4.8", + "log 0.4.11", "mime 0.2.6", "mime_guess 1.8.8", "quick-error", @@ -894,7 +898,7 @@ checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" dependencies = [ "lazy_static", "libc", - "log 0.4.8", + "log 0.4.11", "openssl", "openssl-probe", "openssl-sys", @@ -921,7 +925,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ - "autocfg 1.0.0", + "autocfg 1.0.1", "num-traits", ] @@ -931,7 +935,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ - "autocfg 1.0.0", + "autocfg 1.0.1", ] [[package]] @@ -946,9 +950,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" [[package]] name = "opaque-debug" @@ -982,7 +986,7 @@ version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ - "autocfg 1.0.0", + "autocfg 1.0.1", "cc", "libc", "pkg-config", @@ -991,28 +995,30 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06de47b848347d8c4c94219ad8ecd35eb90231704b067e67e6ae2e36ee023510" +checksum = "2ac6fe3538f701e339953a3ebbe4f39941aababa8a3f6964635b24ab526daeac" [[package]] name = "parking_lot" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" dependencies = [ + "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ "cfg-if", - "cloudabi", + "cloudabi 0.1.0", + "instant", "libc", "redox_syscall", "smallvec", @@ -1066,18 +1072,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" dependencies = [ "proc-macro2", "quote", @@ -1098,9 +1104,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" [[package]] name = "podio" @@ -1110,9 +1116,9 @@ checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" [[package]] name = "ppv-lite86" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "pq-sys" @@ -1130,7 +1136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ "env_logger", - "log 0.4.8", + "log 0.4.11", ] [[package]] @@ -1161,9 +1167,9 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" [[package]] name = "proc-macro-nested" @@ -1173,9 +1179,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ "unicode-xid", ] @@ -1197,11 +1203,11 @@ dependencies = [ [[package]] name = "r2d2" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1497e40855348e4a8a40767d8e55174bce1e445a3ac9254ad44ad468ee0485af" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ - "log 0.4.8", + "log 0.4.11", "parking_lot", "scheduled-thread-pool", ] @@ -1326,7 +1332,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "cloudabi", + "cloudabi 0.0.3", "fuchsia-cprng", "libc", "rand_core 0.4.2", @@ -1364,9 +1370,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "regex" @@ -1397,9 +1403,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ "base64 0.12.3", "bytes", @@ -1410,9 +1416,10 @@ dependencies = [ "http-body", "hyper", "hyper-tls", + "ipnet", "js-sys", "lazy_static", - "log 0.4.8", + "log 0.4.11", "mime 0.3.16", "mime_guess 2.0.3", "native-tls", @@ -1430,15 +1437,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - [[package]] name = "ryu" version = "1.0.5" @@ -1463,19 +1461,13 @@ dependencies = [ [[package]] name = "scheduled-thread-pool" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0988d7fdf88d5e5fcf5923a0f1e8ab345f3e98ab4bc6bc45a2d5ff7f7458fbf6" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ "parking_lot", ] -[[package]] -name = "scoped-tls" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" - [[package]] name = "scoped-tls" version = "1.0.0" @@ -1511,35 +1503,20 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" dependencies = [ "proc-macro2", "quote", @@ -1548,9 +1525,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" dependencies = [ "itoa", "ryu", @@ -1599,7 +1576,7 @@ dependencies = [ "diesel_migrations", "dotenv", "futures", - "log 0.4.8", + "log 0.4.11", "openssl", "pretty_env_logger", "r2d2", @@ -1620,9 +1597,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "socket2" @@ -1644,9 +1621,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.33" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" dependencies = [ "proc-macro2", "quote", @@ -1707,25 +1684,26 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] [[package]] name = "tinyvec" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" [[package]] name = "tokio" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" dependencies = [ "bytes", "fnv", @@ -1768,7 +1746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8b8fe88007ebc363512449868d7da4389c9400072a3f666f212c7280082882a" dependencies = [ "futures", - "log 0.4.8", + "log 0.4.11", "pin-project", "tokio", "tungstenite", @@ -1784,7 +1762,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "log 0.4.8", + "log 0.4.11", "pin-project-lite", "tokio", ] @@ -1795,11 +1773,41 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +[[package]] +name = "tracing" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" +dependencies = [ + "cfg-if", + "log 0.4.11", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "try-lock" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" @@ -1813,7 +1821,7 @@ dependencies = [ "http", "httparse", "input_buffer", - "log 0.4.8", + "log 0.4.11", "rand 0.7.3", "sha-1", "url", @@ -1942,33 +1950,35 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.8", + "log 0.4.11", "try-lock", ] [[package]] name = "warp" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e95175b7a927258ecbb816bdada3cc469cb68593e7940b96a60f4af366a9970" +checksum = "df341dee97c9ae29dfa5e0b0fbbbf620e0d6a36686389bedf83b3daeb8b0d0ac" dependencies = [ "bytes", "futures", "headers", "http", "hyper", - "log 0.4.8", + "log 0.4.11", "mime 0.3.16", "mime_guess 2.0.3", "multipart", "pin-project", - "scoped-tls 1.0.0", + "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-tungstenite", "tower-service", + "tracing", + "tracing-futures", "urlencoding", ] @@ -1978,11 +1988,17 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-bindgen" -version = "0.2.64" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" +checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" dependencies = [ "cfg-if", "serde", @@ -1992,13 +2008,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.64" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" +checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.8", + "log 0.4.11", "proc-macro2", "quote", "syn", @@ -2007,9 +2023,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba48d66049d2a6cc8488702e7259ab7afc9043ad0dc5448444f46f2a453b362" +checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" dependencies = [ "cfg-if", "js-sys", @@ -2019,9 +2035,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.64" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" +checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2029,9 +2045,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.64" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" +checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" dependencies = [ "proc-macro2", "quote", @@ -2042,15 +2058,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.64" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" +checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" [[package]] name = "web-sys" -version = "0.3.41" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" +checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" dependencies = [ "js-sys", "wasm-bindgen", From d90cf2c81339a4a20a0db50981e2b8e68d06b046 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 15:34:19 +0200 Subject: [PATCH 45/69] Upgrade version to 2.0.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b05b2d..fdb8d8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1566,7 +1566,7 @@ checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" [[package]] name = "sirene" -version = "1.0.3" +version = "2.0.0" dependencies = [ "async-trait", "chrono", diff --git a/Cargo.toml b/Cargo.toml index e9fefa5..a46e028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sirene" -version = "1.0.3" +version = "2.0.0" authors = ["Julien Blatecky "] edition = "2018" From 29c2ce314ea7264f80567d608b9411b7a1e56ac9 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 15:36:06 +0200 Subject: [PATCH 46/69] Upgrade version in Clap --- src/commands/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 9bbf086..f7a73e9 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -12,7 +12,7 @@ use update::UpdateFlags; /// Sirene service used to update data in database /// and serve it through a HTTP REST API #[derive(Clap, Debug)] -#[clap(version = "1.0.0", author = "Julien Blatecky")] +#[clap(version = "2.0.0", author = "Julien Blatecky")] struct Opts { /// Path to the temp folder, you can set in environment variable as TEMP_FOLDER #[clap(long = "temp-folder")] From 740478ae46ec09c049bf77c59caa561ac2556f4c Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 17:22:51 +0200 Subject: [PATCH 47/69] Remove useless step method in action --- src/update/action/clean.rs | 5 ----- src/update/action/common.rs | 2 -- src/update/action/download_stock.rs | 5 ----- src/update/action/insert_stock.rs | 5 ----- src/update/action/swap.rs | 5 ----- src/update/action/sync_insee.rs | 5 ----- src/update/action/unzip_stock.rs | 5 ----- 7 files changed, 32 deletions(-) diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index 6ebe329..6d84d40 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; use std::fs::remove_file; use std::path::PathBuf; @@ -16,10 +15,6 @@ pub struct CleanAction { #[async_trait] impl Action for CleanAction { - fn step(&self) -> Step { - Step::CleanFile - } - async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/common.rs b/src/update/action/common.rs index a315986..44da398 100644 --- a/src/update/action/common.rs +++ b/src/update/action/common.rs @@ -2,12 +2,10 @@ use super::super::error::Error; use super::super::summary::SummaryGroupDelegate; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; #[async_trait] pub trait Action: Sync + Send { - fn step(&self) -> Step; async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index af96ff6..462b7ca 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::stream::TryStreamExt; @@ -22,10 +21,6 @@ pub struct DownloadAction { #[async_trait] impl Action for DownloadAction { - fn step(&self) -> Step { - Step::DownloadFile - } - async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/insert_stock.rs b/src/update/action/insert_stock.rs index fe04b7c..cfd7a58 100644 --- a/src/update/action/insert_stock.rs +++ b/src/update/action/insert_stock.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; use std::fs::canonicalize; use std::path::PathBuf; @@ -16,10 +15,6 @@ pub struct InsertAction { #[async_trait] impl Action for InsertAction { - fn step(&self) -> Step { - Step::InsertData - } - async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index 32209bb..8b28ded 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; pub struct SwapAction { @@ -13,10 +12,6 @@ pub struct SwapAction { #[async_trait] impl Action for SwapAction { - fn step(&self) -> Step { - Step::SwapData - } - async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 6bfe432..e88ab45 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; use chrono::{DateTime, Duration, NaiveDateTime, Utc}; @@ -12,10 +11,6 @@ pub struct SyncInseeAction {} #[async_trait] impl Action for SyncInseeAction { - fn step(&self) -> Step { - Step::SyncInsee - } - async fn execute<'a, 'b>( &self, group_type: GroupType, diff --git a/src/update/action/unzip_stock.rs b/src/update/action/unzip_stock.rs index 43e26c0..6cc65b1 100644 --- a/src/update/action/unzip_stock.rs +++ b/src/update/action/unzip_stock.rs @@ -4,7 +4,6 @@ use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::Step; use async_trait::async_trait; use std::fs::{create_dir_all, set_permissions, File, Permissions}; use std::io; @@ -18,10 +17,6 @@ pub struct UnzipAction { #[async_trait] impl Action for UnzipAction { - fn step(&self) -> Step { - Step::UnzipFile - } - async fn execute<'a, 'b>( &self, group_type: GroupType, From 115d8fc125cff76909533b84aa58342c60b68ed5 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 17:23:12 +0200 Subject: [PATCH 48/69] Some small improvements --- src/connectors/insee/mod.rs | 10 +++------- src/models/etablissement/mod.rs | 4 ++-- src/models/unite_legale/mod.rs | 4 ++-- src/update/action/download_stock.rs | 18 +++++++++--------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index fb0471a..33c6112 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -38,13 +38,9 @@ struct InseeTokenResponse { impl ConnectorBuilder { pub fn new() -> Option { - let credentials = env::var("INSEE_CREDENTIALS").ok(); - - if let Some(credentials) = credentials { - Some(ConnectorBuilder { credentials }) - } else { - None - } + env::var("INSEE_CREDENTIALS") + .ok() + .map(|credentials| ConnectorBuilder { credentials }) } pub async fn create(&self) -> Result { diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index 505600c..2e3eabb 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -149,12 +149,12 @@ impl UpdatableModel for EtablissementModel { .as_mut() .ok_or(UpdatableError::MissingInseeConnector)?; - let connection = connectors.local.pool.get()?; - let (next_cursor, etablissements) = insee .get_daily_etablissements(start_timestamp, cursor) .await?; + let connection = connectors.local.pool.get()?; + let updated_count = diesel::insert_into(dsl::etablissement) .values(&etablissements) .on_conflict(dsl::siret) diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index 2681d3a..9b972f1 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -129,12 +129,12 @@ impl UpdatableModel for UniteLegaleModel { .as_mut() .ok_or(UpdatableError::MissingInseeConnector)?; - let connection = connectors.local.pool.get()?; - let (next_cursor, unites_legales) = insee .get_daily_unites_legales(start_timestamp, cursor) .await?; + let connection = connectors.local.pool.get()?; + let updated_count = diesel::insert_into(dsl::unite_legale) .values(&unites_legales) .on_conflict(dsl::siren) diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index 462b7ca..c6ac7af 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -32,15 +32,6 @@ impl Action for DownloadAction { let metadata = group_metadata::get(connectors, group_type)?; - // Create temp path - create_dir_all(self.temp_folder.clone()) - .map_err(|io_error| Error::TempFolderCreationError { io_error })?; - - // Get Zip path - let mut zip_path = PathBuf::from(self.temp_folder.clone()); - zip_path.push(metadata.file_name); - zip_path.set_extension("zip"); - // Prepare file download let client = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) @@ -100,6 +91,15 @@ impl Action for DownloadAction { } } + // Create temp path + create_dir_all(self.temp_folder.clone()) + .map_err(|io_error| Error::TempFolderCreationError { io_error })?; + + // Get Zip path + let mut zip_path = PathBuf::from(self.temp_folder.clone()); + zip_path.push(metadata.file_name); + zip_path.set_extension("zip"); + // Create an output file into which we will save current stock. let mut outfile = File::create(zip_path) .await From 1392dfb3df6b01c6812b48be64f9bb20da8c42fe Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 19:42:35 +0200 Subject: [PATCH 49/69] Remove useless comment --- src/update/action/sync_insee.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index e88ab45..54753b7 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -85,7 +85,5 @@ impl Action for SyncInseeAction { } fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime { - // Uncomment the next line to force custom date to be used - // return NaiveDateTime::parse_from_str("2020-06-25 12:00:00", "%Y-%m-%d %H:%M:%S").unwrap(); timestamp.max(Utc::now().naive_local() - Duration::days(31)) } From f7446c5eff87c9f7f70288a72e18a72cd65b241a Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 27 Aug 2020 20:22:52 +0200 Subject: [PATCH 50/69] Use Github Actions and remove Travis --- .github/workflows/rust.yml | 44 +++++++++++++++------- .travis.yml | 48 ------------------------ Dockerfile | 16 -------- build-musl.sh | 4 +- no-builder.Dockerfile => dist/Dockerfile | 2 +- 5 files changed, 34 insertions(+), 80 deletions(-) delete mode 100644 .travis.yml delete mode 100644 Dockerfile rename no-builder.Dockerfile => dist/Dockerfile (85%) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8537566..cbd2180 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,23 +4,39 @@ on: [push] jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install minimal nightly - uses: actions-rs/toolchain@v1 + - name: Checkout repository + uses: actions/checkout@v2 + - name: Caching + uses: actions/cache@v2 with: - toolchain: nightly - profile: minimal - override: true - - name: Build - uses: actions-rs/cargo@v1 + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-musl-${{ hashFiles('**/Cargo.lock') }} + - name: Build for MUSL + run: | + mkdir -p ~/.cargo/{git,registry} + docker pull clux/muslrust:stable + docker run \ + -v ~/.cargo/registry:/root/.cargo/registry \ + -v ~/.cargo/git:/root/.cargo/git \ + -v $GITHUB_WORKSPACE:/volume \ + --rm -t clux/muslrust:stable \ + sh -c "/volume/build-musl.sh && chown -R $(id -u):$(id -g) ./target /root/.cargo/registry /root/.cargo/git" + - name: Archive production artifact + uses: actions/upload-artifact@v2 with: - command: build - args: --release - - name: Run tests - uses: actions-rs/cargo@v1 + name: sirene + path: dist/sirene + - name: Push to Docker Hub + uses: docker/build-push-action@v1 with: - command: test + path: "./dist" + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: creatiwity/siren + tag_with_ref: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 32d6b42..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: rust - -cache: cargo - -services: - - docker - -env: - global: - - IMAGE_NAME=creatiwity/siren - - secure: eHzYmuj8kykBWbJpsm3PAyHohEmpoLWoewkn3K4OrTyrJwHbzt2OLFTXpaJ3ivtX3BVcJNWiri0pfCt4VUCJTg1kLJpMFcAb4q2drSTKBdpisufn3dy4b4izuRthfildeeRkp7/lR+MUKsTbuiM3BaMQi6lu8u5B8KwOmvsAmwsea06mpajDRANnP7oherHN28YdQD0dn89TEa0/AdHMoLfuTHjX/YY02MzLX+IQ0Fw/Y62M3Ma0FELSNLomv/C94MpDY1l2PxviTx0glMq1mYEDn+sJ+lQcvkYSkSvVe2+K9sRd3LZXN8pAXllM8ZHYbCEtSAsYxkdAj4zpkcO8ekrJiN1A6YkmreVxtNozGi92GfrPy7XPacEJIMNIa7l7EsUtQwbtPnu5vh7YtTL6H0smCYjTYLfY9kWjqU0Vd5s9dS/0q4VPApBNe3NhxITQyUr3y5IdrnJfDUvn3OoBFPl8Q572zFkIC9UhtqmiY+FqjqpVwSpJw12T6Hnf3W3sJknE9LQCojYmQK5wu9A09mb7sfYDqWSSxJ8DDWtbSWtA6+USiZC5tSpFuQr8aMqG4GbRGnvoA7VLDBlYvcidN/u2uo6uLISQrj+uaQNvUwIlwaurt8YTZ5FlUDjxXyjyLOOSmT974Ev3caIjpN58xRTivyTfnREXgU8AcWS3Ma0= - - secure: lVy3FtGBuilGmFwj4E8fXXeQNphq8H1w5ofjrZSJFkuacypShC2ATWijfTZcQO5DUFEP1ODonqtvWJDBgpSnNktwP34bwwUy3uTG6GjNXqY7R+sUMAzzlqQBteqxLvN91OwlaOB0G0gpEDrGTWLc1Wjt7otVo+iRWDN60pukQ5k4+2ePBATF6Mgc0PfcgiBMUh2dh9Q21XTyDw3TyWWqPs76Cs87XwkGiY6lcXRGnBa6ZQYe5KxyR+M9KpffT3ck9AmOCN+QDcaYVgCaXgH3qGYzn2nAXEp8p59cxLyMBKyN/anEJp8TfZ12Aa3BkzmYQeHJSVtYJCyRkci0jXilIwa5Y3mHJqS5Qof/cxayQ3ZogKFObig6I5B3WC45UYansIEKDIp+Z4Mct8cS0IWfyPwMPvv/+MwkbqOKWbSt75DTEyNAn4W+JXAcjhTFUJFDvZVVEJCszghb0UC0RGMEDSNb09Si4njZhpfVmfFCbdKgln/cTkj6AYdZ74CZEI6LX+Zon59FGnPQNs9KjlNBc34bmOMUv1TD/VsoravlYm0mGkiOs1JqHBwdap88wucqbkiuyqa7MSrlvIRYc8V+3T04Nt52fN1hz5TRUG+WGyuwaITB1tlIiO5DmC35CDboHdu+u1GfDYbCD0x7CML4yG9vA873vHDx8JQ4cpxij8E= - -before_script: - - | - if [ "$TRAVIS_BRANCH" = "master" ]; then - DOCKER_TAG=latest - else - DOCKER_TAG="$TRAVIS_BRANCH" - fi - -script: - - docker run -v $TRAVIS_BUILD_DIR:/volume --rm -t clux/muslrust:nightly /volume/build-musl.sh - - docker build --target app --tag $IMAGE_NAME:$DOCKER_TAG --file ./no-builder.Dockerfile "." - -after_success: - - | - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker push $IMAGE_NAME:$DOCKER_TAG - - if [ "$TRAVIS_BRANCH" = "master" ]; then - docker run -v $PWD:/workspace \ - -e DOCKERHUB_USERNAME='$DOCKER_USERNAME' \ - -e DOCKERHUB_PASSWORD='$DOCKER_PASSWORD' \ - -e DOCKERHUB_REPOSITORY='$IMAGE_NAME' \ - -e README_FILEPATH='/workspace/README.md' \ - peterevans/dockerhub-description:2.0.0 - fi - fi - -branches: - only: - - master - - release/* - - develop - - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 082131c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM clux/muslrust:nightly AS builder -RUN rustc --version && cargo --version -COPY Cargo.toml Cargo.lock ./ -COPY migrations ./migrations/ -COPY src ./src/ -RUN cargo build --release --target x86_64-unknown-linux-musl -RUN strip target/x86_64-unknown-linux-musl/release/sirene -RUN cp target/x86_64-unknown-linux-musl/release/sirene / - -FROM alpine AS app -WORKDIR /app -COPY --from=builder /sirene /app/ -ENV HOST 0.0.0.0 -ENV PORT 3000 -EXPOSE 3000 -CMD ["/bin/sh", "-c", "./sirene serve"] diff --git a/build-musl.sh b/build-musl.sh index 42ebdfc..8b9b35a 100755 --- a/build-musl.sh +++ b/build-musl.sh @@ -1,6 +1,8 @@ #! /bin/bash +set -e rustc --version && cargo --version cargo build --release --target x86_64-unknown-linux-musl strip target/x86_64-unknown-linux-musl/release/sirene -cp target/x86_64-unknown-linux-musl/release/sirene . +mkdir -p ./dist +cp target/x86_64-unknown-linux-musl/release/sirene ./dist/ diff --git a/no-builder.Dockerfile b/dist/Dockerfile similarity index 85% rename from no-builder.Dockerfile rename to dist/Dockerfile index 5a67490..7fe6800 100644 --- a/no-builder.Dockerfile +++ b/dist/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine AS app +FROM alpine WORKDIR /app COPY ./sirene /app/ ENV HOST 0.0.0.0 From 30e1d88abff2b990666b315bc3574d486ed1bff1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2020 12:14:20 +0200 Subject: [PATCH 51/69] Bump async-trait from 0.1.38 to 0.1.40 (#162) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.38 to 0.1.40. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.38...0.1.40) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdb8d8f..e70f4e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,9 +17,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" +checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783" dependencies = [ "proc-macro2", "quote", From 2ba5fa4d6f4e2cc493767b17e5de9b14e1b9e066 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2020 09:48:19 +0200 Subject: [PATCH 52/69] Bump warp from 0.2.4 to 0.2.5 (#163) Bumps [warp](https://github.com/seanmonstar/warp) from 0.2.4 to 0.2.5. - [Release notes](https://github.com/seanmonstar/warp/releases) - [Changelog](https://github.com/seanmonstar/warp/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/warp/compare/v0.2.4...v0.2.5) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Cargo.lock | 235 ++++++++++++++++++++++------------------------------- 1 file changed, 96 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e70f4e2..f14dad2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,12 +49,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "base64" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" - [[package]] name = "base64" version = "0.12.3" @@ -76,7 +70,16 @@ dependencies = [ "block-padding", "byte-tools", "byteorder", - "generic-array", + "generic-array 0.12.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.4", ] [[package]] @@ -233,6 +236,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.0" @@ -290,7 +299,16 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array", + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", ] [[package]] @@ -322,7 +340,7 @@ checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", - "log 0.4.11", + "log", "regex", "termcolor", ] @@ -492,6 +510,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.1.14" @@ -537,13 +565,13 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" dependencies = [ - "base64 0.12.3", + "base64", "bitflags", "bytes", "headers-core", "http", - "mime 0.3.16", - "sha-1", + "mime", + "sha-1 0.8.2", "time", ] @@ -744,15 +772,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -dependencies = [ - "log 0.4.11", -] - [[package]] name = "log" version = "0.4.11" @@ -795,41 +814,20 @@ dependencies = [ "syn", ] -[[package]] -name = "mime" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -dependencies = [ - "log 0.3.9", -] - [[package]] name = "mime" version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "1.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" -dependencies = [ - "mime 0.2.6", - "phf", - "phf_codegen", - "unicase 1.4.2", -] - [[package]] name = "mime_guess" version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" dependencies = [ - "mime 0.3.16", - "unicase 2.6.0", + "mime", + "unicase", ] [[package]] @@ -853,7 +851,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.11", + "log", "miow", "net2", "slab", @@ -874,15 +872,15 @@ dependencies = [ [[package]] name = "multipart" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" +checksum = "8209c33c951f07387a8497841122fc6f712165e3f9bda3e6be4645b58188f676" dependencies = [ "buf_redux", "httparse", - "log 0.4.11", - "mime 0.2.6", - "mime_guess 1.8.8", + "log", + "mime", + "mime_guess", "quick-error", "rand 0.6.5", "safemem", @@ -898,7 +896,7 @@ checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" dependencies = [ "lazy_static", "libc", - "log 0.4.11", + "log", "openssl", "openssl-probe", "openssl-sys", @@ -960,6 +958,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.30" @@ -1031,45 +1035,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "phf" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" -dependencies = [ - "phf_shared", - "rand 0.6.5", -] - -[[package]] -name = "phf_shared" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" -dependencies = [ - "siphasher", - "unicase 1.4.2", -] - [[package]] name = "pin-project" version = "0.4.23" @@ -1136,7 +1101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ "env_logger", - "log 0.4.11", + "log", ] [[package]] @@ -1149,7 +1114,7 @@ dependencies = [ "proc-macro2", "quote", "syn", - "version_check 0.9.2", + "version_check", ] [[package]] @@ -1162,7 +1127,7 @@ dependencies = [ "quote", "syn", "syn-mid", - "version_check 0.9.2", + "version_check", ] [[package]] @@ -1207,7 +1172,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ - "log 0.4.11", + "log", "parking_lot", "scheduled-thread-pool", ] @@ -1407,7 +1372,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ - "base64 0.12.3", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1419,9 +1384,9 @@ dependencies = [ "ipnet", "js-sys", "lazy_static", - "log 0.4.11", - "mime 0.3.16", - "mime_guess 2.0.3", + "log", + "mime", + "mime_guess", "native-tls", "percent-encoding", "pin-project-lite", @@ -1552,17 +1517,24 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ - "block-buffer", - "digest", + "block-buffer 0.7.3", + "digest 0.8.1", "fake-simd", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] -name = "siphasher" -version = "0.2.3" +name = "sha-1" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" +checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] [[package]] name = "sirene" @@ -1576,7 +1548,7 @@ dependencies = [ "diesel_migrations", "dotenv", "futures", - "log 0.4.11", + "log", "openssl", "pretty_env_logger", "r2d2", @@ -1741,12 +1713,12 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b8fe88007ebc363512449868d7da4389c9400072a3f666f212c7280082882a" +checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" dependencies = [ - "futures", - "log 0.4.11", + "futures-util", + "log", "pin-project", "tokio", "tungstenite", @@ -1762,7 +1734,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "log 0.4.11", + "log", "pin-project-lite", "tokio", ] @@ -1780,7 +1752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ "cfg-if", - "log 0.4.11", + "log", "tracing-core", ] @@ -1811,19 +1783,19 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfea31758bf674f990918962e8e5f07071a3161bd7c4138ed23e416e1ac4264e" +checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" dependencies = [ - "base64 0.11.0", + "base64", "byteorder", "bytes", "http", "httparse", "input_buffer", - "log 0.4.11", + "log", "rand 0.7.3", - "sha-1", + "sha-1 0.9.1", "url", "utf-8", ] @@ -1843,22 +1815,13 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" -[[package]] -name = "unicase" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" -dependencies = [ - "version_check 0.1.5", -] - [[package]] name = "unicase" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.2", + "version_check", ] [[package]] @@ -1932,12 +1895,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.2" @@ -1950,24 +1907,24 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.11", + "log", "try-lock", ] [[package]] name = "warp" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df341dee97c9ae29dfa5e0b0fbbbf620e0d6a36686389bedf83b3daeb8b0d0ac" +checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" dependencies = [ "bytes", "futures", "headers", "http", "hyper", - "log 0.4.11", - "mime 0.3.16", - "mime_guess 2.0.3", + "log", + "mime", + "mime_guess", "multipart", "pin-project", "scoped-tls", @@ -2014,7 +1971,7 @@ checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.11", + "log", "proc-macro2", "quote", "syn", From dfa63bb8a881a0136ff239e07213e127b90f506a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2020 11:49:15 +0200 Subject: [PATCH 53/69] Bump zip from 0.5.6 to 0.5.7 (#164) Bumps [zip](https://github.com/mvdnes/zip-rs) from 0.5.6 to 0.5.7. - [Release notes](https://github.com/mvdnes/zip-rs/releases) - [Commits](https://github.com/mvdnes/zip-rs/commits/v0.5.7) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Cargo.lock | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f14dad2..ba65669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,10 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] -name = "adler" -version = "0.2.3" +name = "adler32" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "aho-corasick" @@ -353,9 +353,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" -version = "1.0.17" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766d0e77a2c1502169d4a93ff3b8c15a71fd946cd0126309752104e5f3c46d94" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", @@ -832,11 +832,11 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.1" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" dependencies = [ - "adler", + "adler32", ] [[package]] @@ -1073,12 +1073,6 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" -[[package]] -name = "podio" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" - [[package]] name = "ppv-lite86" version = "0.2.9" @@ -1645,6 +1639,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -2093,13 +2107,14 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58287c28d78507f5f91f2a4cf1e8310e2c76fd4c6932f93ac60fd1ceb402db7d" +checksum = "d30de6e58104bb7b9a94f34b52a2bdabb8a40b678a64201cd0069e3d7119b5ff" dependencies = [ + "byteorder", "bzip2", "crc32fast", "flate2", - "podio", + "thiserror", "time", ] From c92446c223250118b010ecab6a495ab9ee3a91d0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 16:25:05 +0200 Subject: [PATCH 54/69] Bump serde from 1.0.115 to 1.0.116 (#166) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.115 to 1.0.116. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.115...v1.0.116) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba65669..d665756 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1464,18 +1464,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" dependencies = [ "proc-macro2", "quote", From da9f7b05b9de9bccd07bfb38dc04da533b031286 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 16:35:38 +0200 Subject: [PATCH 55/69] Bump zip from 0.5.7 to 0.5.8 (#165) Bumps [zip](https://github.com/mvdnes/zip-rs) from 0.5.7 to 0.5.8. - [Release notes](https://github.com/mvdnes/zip-rs/releases) - [Commits](https://github.com/mvdnes/zip-rs/compare/v0.5.7...v0.5.8) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d665756..f3ea6ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2107,9 +2107,9 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30de6e58104bb7b9a94f34b52a2bdabb8a40b678a64201cd0069e3d7119b5ff" +checksum = "543adf038106b64cfca4711c82c917d785e3540e04f7996554488f988ec43124" dependencies = [ "byteorder", "bzip2", From 5e8277e3156bac13b490c1ef8834906963552ab3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2020 14:07:07 +0200 Subject: [PATCH 56/69] Bump custom_error from 1.7.1 to 1.8.0 (#168) Bumps [custom_error](https://github.com/lovasoa/custom_error) from 1.7.1 to 1.8.0. - [Release notes](https://github.com/lovasoa/custom_error/releases) - [Commits](https://github.com/lovasoa/custom_error/compare/v1.7.1...v1.8.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3ea6ab..f3d36bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,9 +253,9 @@ dependencies = [ [[package]] name = "custom_error" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a0fc65739ae998afc8d68e64bdac2efd1bc4ffa1a0703d171ef2defae3792f" +checksum = "51ac5e99a7fea3ee8a03fa4721a47e2efd3fbb38358fc61192a54d4c6f866c12" [[package]] name = "diesel" diff --git a/Cargo.toml b/Cargo.toml index a46e028..2637082 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } clap = "3.0.0-beta" -custom_error = "1.7" +custom_error = "1.8" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2", "serde_json", "64-column-tables"] } diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" From 3ded30fa486c797a7d240ec0aa186b9e11833e00 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Wed, 16 Sep 2020 16:43:42 +0200 Subject: [PATCH 57/69] Add launcher for VSCode --- .vscode/launch.json | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..49d8fc1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,44 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'sirene' serve", + "cargo": { + "args": [ + "build", + "--bin=sirene", + "--package=sirene" + ], + "filter": { + "name": "sirene", + "kind": "bin" + } + }, + "args": ["serve"], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'sirene' update all", + "cargo": { + "args": [ + "build", + "--bin=sirene", + "--package=sirene" + ], + "filter": { + "name": "sirene", + "kind": "bin" + } + }, + "args": ["update", "all"], + "cwd": "${workspaceFolder}" + } + ] +} From 6c4859fdfddc29bdd815d3b69d31d568fbd6b397 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 1 Oct 2020 12:32:33 +0200 Subject: [PATCH 58/69] Upgrade dependencies --- Cargo.lock | 158 +++++++++++++++++++------------------- src/commands/serve/mod.rs | 6 +- 2 files changed, 80 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3d36bf..4378dd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" [[package]] name = "cfg-if" @@ -160,21 +160,23 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "libc", "num-integer", "num-traits", "serde", "time", + "winapi 0.3.9", ] [[package]] name = "clap" -version = "3.0.0-beta.1" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "860643c53f980f0d38a5e25dfab6c3c93b2cb3aa1fe192643d17a293c6c41936" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" dependencies = [ "atty", "bitflags", @@ -191,9 +193,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.0.0-beta.1" +version = "3.0.0-beta.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb51c9e75b94452505acd21d929323f5a5c6c4735a852adbd39ef5fb1b014f30" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" dependencies = [ "heck", "proc-macro-error", @@ -522,9 +524,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -552,12 +554,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" -dependencies = [ - "autocfg 1.0.1", -] +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "headers" @@ -595,9 +594,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "4c30f6d0bc6b00693347368a67d41b58f2fb851215ff1da49e90fe2c5c667151" dependencies = [ "libc", ] @@ -629,6 +628,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + [[package]] name = "humantime" version = "1.3.0" @@ -640,9 +645,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.7" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" dependencies = [ "bytes", "futures-channel", @@ -652,10 +657,10 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project", "socket2", - "time", "tokio", "tower-service", "tracing", @@ -688,9 +693,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ "autocfg 1.0.1", "hashbrown", @@ -707,9 +712,12 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +dependencies = [ + "cfg-if", +] [[package]] name = "iovec" @@ -734,9 +742,9 @@ checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] @@ -759,9 +767,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.76" +version = "0.2.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "aa7087f49d294270db4e1928fc110c976cd4b9e5a16348e0a1df09afa99e6c98" [[package]] name = "lock_api" @@ -908,9 +916,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ "cfg-if", "libc", @@ -1037,18 +1045,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.23" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa" +checksum = "2b9e280448854bd91559252582173b3bd1f8e094a0e644791c0628ca9b1f144f" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.23" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f" +checksum = "c8c8b352676bc6a4c3d71970560b913cea444a7a921cc2e2d920225e4b91edaa" dependencies = [ "proc-macro2", "quote", @@ -1057,9 +1065,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" +checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" [[package]] name = "pin-utils" @@ -1100,9 +1108,9 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.12" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2", @@ -1113,14 +1121,12 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "0.4.12" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "syn", - "syn-mid", "version_check", ] @@ -1138,9 +1144,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.19" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ "unicode-xid", ] @@ -1484,9 +1490,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" +checksum = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" dependencies = [ "itoa", "ryu", @@ -1569,9 +1575,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ "cfg-if", "libc", @@ -1587,26 +1593,15 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.39" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" +checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] -[[package]] -name = "syn-mid" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tempfile" version = "3.1.0" @@ -1632,9 +1627,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" dependencies = [ "unicode-width", ] @@ -1761,20 +1756,21 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" dependencies = [ "cfg-if", "log", + "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" dependencies = [ "lazy_static", ] @@ -1967,9 +1963,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if", "serde", @@ -1979,9 +1975,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" dependencies = [ "bumpalo", "lazy_static", @@ -1994,9 +1990,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ "cfg-if", "js-sys", @@ -2006,9 +2002,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2016,9 +2012,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ "proc-macro2", "quote", @@ -2029,15 +2025,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" [[package]] name = "web-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index 5660b30..a0c8cf5 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -13,15 +13,15 @@ pub struct ServeFlags { environment: Option, /// Listen this port, you can set in environment variable as PORT - #[clap(short = "p", long = "port")] + #[clap(short = 'p', long = "port")] port: Option, /// Listen this host, you can set in environment variable as HOST - #[clap(short = "h", long = "host")] + #[clap(short = 'h', long = "host")] host: Option, /// API key needed to allow maintenance operation from HTTP, you can set in environment variable as API_KEY - #[clap(short = "k", long = "api-key")] + #[clap(short = 'k', long = "api-key")] api_key: Option, } From 97c02435388e20365400f880e2bd97d1938c5021 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Wed, 7 Oct 2020 11:54:32 +0200 Subject: [PATCH 59/69] Fix empty INSEE_CREDENTIALS var --- src/connectors/insee/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs index 33c6112..8f17711 100644 --- a/src/connectors/insee/mod.rs +++ b/src/connectors/insee/mod.rs @@ -40,7 +40,12 @@ impl ConnectorBuilder { pub fn new() -> Option { env::var("INSEE_CREDENTIALS") .ok() - .map(|credentials| ConnectorBuilder { credentials }) + .and_then(|credentials| + match credentials.len() { + 0 => None, + _ => Some(ConnectorBuilder { credentials }) + } + ) } pub async fn create(&self) -> Result { From c74bee72ba6174a6072c8ee449346c60d2a40fb4 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Wed, 7 Oct 2020 11:54:44 +0200 Subject: [PATCH 60/69] Upgrade dependencies --- Cargo.lock | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4378dd3..866f0d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,9 +17,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783" +checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" dependencies = [ "proc-macro2", "quote", @@ -410,9 +410,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +checksum = "5d8e3078b7b2a8a671cb7a3d17b4760e4181ea243227776ba83fd043b4ca034e" dependencies = [ "futures-channel", "futures-core", @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" dependencies = [ "futures-core", "futures-sink", @@ -435,15 +435,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" [[package]] name = "futures-executor" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +checksum = "cc709ca1da6f66143b8c9bec8e6260181869893714e9b5a490b169b0414144ab" dependencies = [ "futures-core", "futures-task", @@ -452,15 +452,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" +checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" [[package]] name = "futures-macro" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -470,24 +470,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" +checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" [[package]] name = "futures-task" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" dependencies = [ "futures-channel", "futures-core", @@ -594,9 +594,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c30f6d0bc6b00693347368a67d41b58f2fb851215ff1da49e90fe2c5c667151" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" dependencies = [ "libc", ] @@ -767,9 +767,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.78" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa7087f49d294270db4e1928fc110c976cd4b9e5a16348e0a1df09afa99e6c98" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" [[package]] name = "lock_api" @@ -1045,18 +1045,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b9e280448854bd91559252582173b3bd1f8e094a0e644791c0628ca9b1f144f" +checksum = "13fbdfd6bdee3dc9be46452f86af4a4072975899cf8592466668620bebfbcc17" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8c8b352676bc6a4c3d71970560b913cea444a7a921cc2e2d920225e4b91edaa" +checksum = "c82fb1329f632c3552cf352d14427d57a511b1cf41db93b3a7d77906a82dcc8e" dependencies = [ "proc-macro2", "quote", @@ -1636,18 +1636,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" dependencies = [ "proc-macro2", "quote", From 4e4b27142424225f54ec7aa8a3d848fd612b941b Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Wed, 7 Oct 2020 16:26:09 +0200 Subject: [PATCH 61/69] Add configuration documentation --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index e97a523..36a6375 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,30 @@ createdb --owner=sirene sirene ## Documentation +### Configuration + +Recommended configuration for production with docker: + +``` +RUST_LOG=api=warn +SIRENE_ENV=production +API_KEY=[Any randomized string, needed to use the HTTP admin endpoint] +DATABASE_URL=postgresql://[USER]:[PASSWORD]@[PG_HOST]:[PG_PORT]/[PG_DATABASE] +DATABASE_POOL_SIZE=100 +INSEE_CREDENTIALS=[Base64(consumer-key:consumer-secret)] +``` + +**How to generate INSEE_CREDENTIALS** + +This variable is only needed if you want to have the daily updates. + +1. Go to https://api.insee.fr/catalogue/ +2. Create an account or sign in +3. Create an application on this portal +4. Subscribe this application to the *Sirene - V3* API +5. Generate a key pair in the application details +6. Copy the key from the `curl` example and paste it in `.env`: `Authorization: Basic [INSEE_CREDENTIALS]` + ### CLI **> sirene** From e1cc0f436db32fc5e8eadd7ff075696ec179dcbe Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 8 Oct 2020 00:46:55 +0200 Subject: [PATCH 62/69] Add asynchronous update --- .env.sample | 1 + README.md | 62 +++++++++++++++++++---------- src/commands/serve/mod.rs | 10 +++++ src/commands/serve/runner/common.rs | 3 +- src/commands/serve/runner/error.rs | 2 + src/commands/serve/runner/mod.rs | 43 ++++++++++++++++++-- src/commands/update.rs | 1 + src/connectors/insee/mod.rs | 11 +++-- src/connectors/insee/types/mod.rs | 2 +- src/connectors/local.rs | 1 + src/connectors/mod.rs | 1 + src/update/action/clean.rs | 8 ++-- src/update/action/download_stock.rs | 8 ++-- src/update/action/insert_stock.rs | 10 ++--- src/update/action/swap.rs | 8 ++-- src/update/action/sync_insee.rs | 6 +-- src/update/action/unzip_stock.rs | 12 +++--- src/update/common.rs | 1 + src/update/error.rs | 3 +- src/update/mod.rs | 48 ++++++++++++++++++---- 20 files changed, 172 insertions(+), 69 deletions(-) diff --git a/.env.sample b/.env.sample index 7e79ab9..680b081 100644 --- a/.env.sample +++ b/.env.sample @@ -2,6 +2,7 @@ RUST_LOG=api=trace SIRENE_ENV=development HOST=localhost PORT=3000 +BASE_URL=http://localhost:3000 API_KEY= DATABASE_URL=postgresql://sirene:sirenepw@127.0.0.1:5432/sirene DATABASE_POOL_SIZE=100 diff --git a/README.md b/README.md index 36a6375..c44ea5f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Recommended configuration for production with docker: ``` RUST_LOG=api=warn SIRENE_ENV=production +BASE_URL=[Your base URL, needed to update asynchronously] API_KEY=[Any randomized string, needed to use the HTTP admin endpoint] DATABASE_URL=postgresql://[USER]:[PASSWORD]@[PG_HOST]:[PG_PORT]/[PG_DATABASE] DATABASE_POOL_SIZE=100 @@ -53,11 +54,11 @@ This variable is only needed if you want to have the daily updates. ### CLI -**> sirene** +**> sirene --help** ``` -> sirene -h - +sirene 2.0.0 +Julien Blatecky Sirene service used to update data in database and serve it through a HTTP REST API USAGE: @@ -68,13 +69,15 @@ FLAGS: -V, --version Prints version information OPTIONS: - --db-folder Path to the file storage folder for the database, you can set in environment - variable as DB_FOLDER. Could be the same as FILE_FOLDER if this app and the - database are on the same file system. Files copied by this app inside FILE_FOLDER - must be visible by the database in DB_FOLDER - --file-folder Path to the file storage folder for this app, you can set in environment variable - as FILE_FOLDER - --temp-folder Path to the temp folder, you can set in environment variable as TEMP_FOLDER + --db-folder Path to the file storage folder for the database, you can set + in environment variable as DB_FOLDER. Could be the same as + FILE_FOLDER if this app and the database are on the same file + system. Files copied by this app inside FILE_FOLDER must be + visible by the database in DB_FOLDER + --file-folder Path to the file storage folder for this app, you can set in + environment variable as FILE_FOLDER + --temp-folder Path to the temp folder, you can set in environment variable + as TEMP_FOLDER SUBCOMMANDS: help Prints this message or the help of the given subcommand(s) @@ -85,7 +88,7 @@ SUBCOMMANDS: **> sirene serve --help** ``` -sirene-serve +sirene-serve Serve data from database to /unites_legales/ and /etablissements/ USAGE: @@ -96,28 +99,32 @@ FLAGS: -V, --version Prints version information OPTIONS: - -k, --api-key API key needed to allow maintenance operation from HTTP, you can set in environment - variable as API_KEY - --env Configure log level, you can set in environment variable as SIRENE_ENV [possible values: - development, staging, production] - -h, --host Listen this host, you can set in environment variable as HOST - -p, --port Listen this port, you can set in environment variable as PORT + -k, --api-key API key needed to allow maintenance operation from HTTP, you can + set in environment variable as API_KEY + -b, --base-url Base URL needed to configure asynchronous polling for updates, you + can set in environment variable as BASE_URL + --env Configure log level, you can set in environment variable as + SIRENE_ENV [possible values: development, staging, production] + -h, --host Listen this host, you can set in environment variable as HOST + -p, --port Listen this port, you can set in environment variable as PORT ``` **> sirene update --help** ``` -sirene-update +sirene-update Update data from CSV source files USAGE: sirene update [FLAGS] [SUBCOMMAND] ARGS: - Configure which part will be updated [possible values: unites-legales, etablissements, all] + Configure which part will be updated [possible values: unites-legales, + etablissements, all] FLAGS: - --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete it + --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete + it --force Force update even if the source data where not updated -h, --help Prints help information -V, --version Prints version information @@ -125,7 +132,8 @@ FLAGS: SUBCOMMANDS: clean-file Clean files from FILE_FOLDER download-file Download file in TEMP_FOLDER - finish-error Set a staled update process to error, use only if the process is really stopped + finish-error Set a staled update process to error, use only if the process is really + stopped help Prints this message or the help of the given subcommand(s) insert-data Load CSV file in database in loader-table from DB_FOLDER swap-data Swap loader-table to production @@ -151,13 +159,23 @@ POST /admin/update api_key: string, group_type: "UnitesLegales" | "Etablissements" | "All", force: bool, - data_only: bool, + asynchronous: bool, } ``` +If `asynchronous` is set to `true`, the update endpoint will immediately return the following: +``` +Status: 202 Accepted +Location: /admin/update/status?api_key=string +Retry-After: 10 + +[Initial status for the started update] +``` + ``` GET /admin/update/status?api_key=string ``` +If an update is in progress, the status code will be 202, otherwise 200. ``` POST /admin/update/status/error diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index a0c8cf5..f33170b 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -23,6 +23,10 @@ pub struct ServeFlags { /// API key needed to allow maintenance operation from HTTP, you can set in environment variable as API_KEY #[clap(short = 'k', long = "api-key")] api_key: Option, + + /// Base URL needed to configure asynchronous polling for updates, you can set in environment variable as BASE_URL + #[clap(short = 'b', long = "base-url")] + base_url: Option, } #[derive(Clap, Debug)] @@ -71,6 +75,11 @@ pub async fn run(flags: ServeFlags, folder_options: FolderOptions, builders: Con None => env::var("API_KEY").ok(), }; + let base_url = match flags.base_url { + Some(key) => Some(key), + None => env::var("BASE_URL").ok(), + }; + log::info!("[Warp] Configuring for {:#?}", env); runner::run( @@ -79,6 +88,7 @@ pub async fn run(flags: ServeFlags, folder_options: FolderOptions, builders: Con builders, api_key, folder_options, + base_url, }, ) .await; diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index eb1c48e..62ad7cf 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -10,6 +10,7 @@ pub struct Context { pub builders: ConnectorsBuilders, pub api_key: Option, pub folder_options: FolderOptions, + pub base_url: Option, } #[derive(Deserialize)] @@ -17,7 +18,7 @@ pub struct UpdateOptions { pub api_key: String, pub group_type: SyntheticGroupType, pub force: bool, - pub data_only: bool, + pub asynchronous: bool, } #[derive(Deserialize)] diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index 24b6639..62586b5 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -10,6 +10,7 @@ custom_error! { pub Error InvalidData = "Invalid data", MissingApiKeyError = "[Admin] Missing API key in configuration", ApiKeyError = "[Admin] Wrong API key", + MissingBaseUrlForAsyncError = "[Admin] No BASE_URL configured, needed for asynchronous updates", LocalConnectionFailed{source: r2d2::Error} = "Unable to connect to local database ({source}).", UpdateConnectorError {source: ConnectorError} = "[Update] Error while creating connector: {source}", UpdateError {source: InternalUpdateError} = "[Update] {source}", @@ -78,6 +79,7 @@ pub async fn handle_rejection(err: Rejection) -> Result Error::InvalidData => StatusCode::BAD_REQUEST, Error::MissingApiKeyError => StatusCode::UNAUTHORIZED, Error::ApiKeyError => StatusCode::UNAUTHORIZED, + Error::MissingBaseUrlForAsyncError => StatusCode::BAD_REQUEST, Error::LocalConnectionFailed { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, Error::UpdateConnectorError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, Error::UpdateError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 281cabf..8975d86 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -14,7 +14,7 @@ use error::Error; use std::convert::Infallible; use std::net::SocketAddr; use warp::{ - http::{header, Method}, + http::{header, Method, StatusCode}, Filter, Rejection, Reply, }; @@ -32,22 +32,48 @@ async fn update(options: UpdateOptions, context: Context) -> Result Result { @@ -64,7 +90,16 @@ async fn status(query: StatusQueryString, context: Context) -> Result Option { env::var("INSEE_CREDENTIALS") .ok() - .and_then(|credentials| - match credentials.len() { - 0 => None, - _ => Some(ConnectorBuilder { credentials }) - } - ) + .and_then(|credentials| match credentials.len() { + 0 => None, + _ => Some(ConnectorBuilder { credentials }), + }) } pub async fn create(&self) -> Result { diff --git a/src/connectors/insee/types/mod.rs b/src/connectors/insee/types/mod.rs index 4662eee..b91ef2a 100644 --- a/src/connectors/insee/types/mod.rs +++ b/src/connectors/insee/types/mod.rs @@ -58,7 +58,7 @@ where .map(Option::from), Ok(serde_json::Value::Null) => Ok(None), Ok(v) => { - println!("string expected but found something else: {}", v); + log::error!("string expected but found something else: {}", v); Ok(None) } Err(_) => Ok(None), diff --git a/src/connectors/local.rs b/src/connectors/local.rs index 0aef95b..202a75a 100644 --- a/src/connectors/local.rs +++ b/src/connectors/local.rs @@ -7,6 +7,7 @@ embed_migrations!("./migrations"); pub type Connection = r2d2::PooledConnection>; +#[derive(Clone)] pub struct Connector { pub pool: Pool>, } diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index e51fa9f..7dbee10 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -13,6 +13,7 @@ pub struct ConnectorsBuilders { insee: Option, } +#[derive(Clone)] pub struct Connectors { pub local: local::Connector, pub insee: Option, diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index 6d84d40..4ae56ca 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -21,7 +21,7 @@ impl Action for CleanAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[Clean] Cleaning {:#?}", group_type); + log::debug!("[Clean] Cleaning {:#?}", group_type); summary_delegate.start(connectors, None, 2)?; @@ -42,14 +42,14 @@ impl Action for CleanAction { csv_path.set_extension("csv"); if let Err(error) = remove_file(zip_path) { - println!("[Clean] Zip not deleted ({})", error); + log::debug!("[Clean] Zip not deleted ({})", error); updated = false; done_count -= 1; status_label = String::from("zip not deleted"); } if let Err(error) = remove_file(csv_path) { - println!("[Clean] CSV not deleted ({})", error); + log::debug!("[Clean] CSV not deleted ({})", error); updated = false; done_count -= 1; status_label = String::from("csv not deleted"); @@ -59,7 +59,7 @@ impl Action for CleanAction { summary_delegate.finish(connectors, status_label, done_count, updated)?; - println!("[Clean] Finished cleaning of {:#?}", group_type); + log::debug!("[Clean] Finished cleaning of {:#?}", group_type); Ok(()) } diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs index c6ac7af..c1e4eab 100644 --- a/src/update/action/download_stock.rs +++ b/src/update/action/download_stock.rs @@ -27,7 +27,7 @@ impl Action for DownloadAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[Download] Downloading {:#?}", group_type); + log::debug!("[Download] Downloading {:#?}", group_type); summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -62,7 +62,7 @@ impl Action for DownloadAction { if !self.force { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if last_modified.le(&last_imported_timestamp) { - println!("[Download] {:#?} already imported", group_type); + log::debug!("[Download] {:#?} already imported", group_type); summary_delegate.finish( connectors, @@ -77,7 +77,7 @@ impl Action for DownloadAction { if let Some(staging_file_timestamp) = metadata.staging_file_timestamp { if last_modified.le(&staging_file_timestamp) { - println!("[Download] {:#?} already downloaded", group_type); + log::debug!("[Download] {:#?} already downloaded", group_type); summary_delegate.finish( connectors, @@ -116,7 +116,7 @@ impl Action for DownloadAction { .await .map_err(|io_error| Error::FileCopyError { io_error })?; - println!("[Download] Download of {:#?} finished", group_type); + log::debug!("[Download] Download of {:#?} finished", group_type); // Update staging file timestamp group_metadata::set_staging_file_timestamp(connectors, group_type, last_modified)?; diff --git a/src/update/action/insert_stock.rs b/src/update/action/insert_stock.rs index cfd7a58..6e18cf6 100644 --- a/src/update/action/insert_stock.rs +++ b/src/update/action/insert_stock.rs @@ -21,7 +21,7 @@ impl Action for InsertAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[Insert] Insert {:#?}", group_type); + log::debug!("[Insert] Insert {:#?}", group_type); summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -30,7 +30,7 @@ impl Action for InsertAction { let staging_csv_file_timestamp = match metadata.staging_csv_file_timestamp { Some(staging_csv_file_timestamp) => staging_csv_file_timestamp, None => { - println!("[Insert] Nothing to insert for {:#?}", group_type); + log::debug!("[Insert] Nothing to insert for {:#?}", group_type); summary_delegate.finish(connectors, String::from("nothing to insert"), 0, false)?; @@ -43,7 +43,7 @@ impl Action for InsertAction { if let Some(staging_imported_timestamp) = metadata.staging_imported_timestamp { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { - println!("[Insert] {:#?} already imported", group_type); + log::debug!("[Insert] {:#?} already imported", group_type); summary_delegate.finish( connectors, @@ -57,7 +57,7 @@ impl Action for InsertAction { } if staging_csv_file_timestamp.le(&staging_imported_timestamp) { - println!("[Insert] {:#?} already inserted", group_type); + log::debug!("[Insert] {:#?} already inserted", group_type); summary_delegate.finish( connectors, @@ -94,7 +94,7 @@ impl Action for InsertAction { staging_csv_file_timestamp, )?; - println!("[Insert] Finished insert of {:#?}", group_type); + log::debug!("[Insert] Finished insert of {:#?}", group_type); summary_delegate.finish(connectors, String::from("inserted"), 1, true)?; diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index 8b28ded..2e87fa8 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -18,7 +18,7 @@ impl Action for SwapAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[Swap] Swapping {:#?}", group_type); + log::debug!("[Swap] Swapping {:#?}", group_type); summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -27,7 +27,7 @@ impl Action for SwapAction { let staging_imported_timestamp = match metadata.staging_imported_timestamp { Some(staging_imported_timestamp) => staging_imported_timestamp, None => { - println!("[Swap] Nothing to swap for {:#?}", group_type); + log::debug!("[Swap] Nothing to swap for {:#?}", group_type); summary_delegate.finish(connectors, String::from("nothing to swap"), 0, false)?; @@ -39,7 +39,7 @@ impl Action for SwapAction { if !self.force { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { - println!("[Swap] {:#?} already imported", group_type); + log::debug!("[Swap] {:#?} already imported", group_type); summary_delegate.finish( connectors, @@ -77,7 +77,7 @@ impl Action for SwapAction { staging_imported_timestamp, )?; - println!("[Swap] Swap of {:#?} finished", group_type); + log::debug!("[Swap] Swap of {:#?} finished", group_type); summary_delegate.finish(connectors, String::from("swapped"), 1, true)?; diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs index 54753b7..ffe299e 100644 --- a/src/update/action/sync_insee.rs +++ b/src/update/action/sync_insee.rs @@ -17,7 +17,7 @@ impl Action for SyncInseeAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[SyncInsee] Syncing {:#?}", group_type); + log::debug!("[SyncInsee] Syncing {:#?}", group_type); // Use Insee connector only if present if connectors.insee.is_some() { @@ -47,7 +47,7 @@ impl Action for SyncInseeAction { summary_delegate.progress(connectors, updated_count as u32)?; } - println!("[SyncInsee] {} {:#?} synced", updated_count, group_type); + log::debug!("[SyncInsee] {} {:#?} synced", updated_count, group_type); group_metadata::set_last_insee_synced_timestamp( connectors, @@ -78,7 +78,7 @@ impl Action for SyncInseeAction { )?; } - println!("[SyncInsee] Syncing of {:#?} done", group_type); + log::debug!("[SyncInsee] Syncing of {:#?} done", group_type); Ok(()) } diff --git a/src/update/action/unzip_stock.rs b/src/update/action/unzip_stock.rs index 6cc65b1..016697c 100644 --- a/src/update/action/unzip_stock.rs +++ b/src/update/action/unzip_stock.rs @@ -23,7 +23,7 @@ impl Action for UnzipAction { connectors: &mut Connectors, summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, ) -> Result<(), Error> { - println!("[Unzip] Unzipping {:#?}", group_type); + log::debug!("[Unzip] Unzipping {:#?}", group_type); summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -32,7 +32,7 @@ impl Action for UnzipAction { let staging_file_timestamp = match metadata.staging_file_timestamp { Some(staging_file_timestamp) => staging_file_timestamp, None => { - println!("[Unzip] Nothing to unzip for {:#?}", group_type); + log::debug!("[Unzip] Nothing to unzip for {:#?}", group_type); summary_delegate.finish(connectors, String::from("nothing to unzip"), 0, false)?; @@ -45,7 +45,7 @@ impl Action for UnzipAction { if let Some(staging_csv_file_timestamp) = metadata.staging_csv_file_timestamp { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_csv_file_timestamp.le(&last_imported_timestamp) { - println!("[Unzip] {:#?} already imported", group_type); + log::debug!("[Unzip] {:#?} already imported", group_type); summary_delegate.finish( connectors, @@ -59,7 +59,7 @@ impl Action for UnzipAction { } if staging_file_timestamp.le(&staging_csv_file_timestamp) { - println!("[Unzip] {:#?} already unzipped", group_type); + log::debug!("[Unzip] {:#?} already unzipped", group_type); summary_delegate.finish( connectors, @@ -103,7 +103,7 @@ impl Action for UnzipAction { .by_index(0) .map_err(|zip_error| Error::ZipAccessFileError { zip_error })?; - println!( + log::debug!( "[Unzip] Unzipping file {:#?} extracted to \"{}\" ({} bytes)", group_type, csv_path.as_path().display(), @@ -131,7 +131,7 @@ impl Action for UnzipAction { staging_file_timestamp, )?; - println!("[Unzip] Unzip of {:#?} finished", group_type); + log::debug!("[Unzip] Unzip of {:#?} finished", group_type); summary_delegate.finish(connectors, String::from("unzipped"), 1, true)?; diff --git a/src/update/common.rs b/src/update/common.rs index bff66fb..5ac71bd 100644 --- a/src/update/common.rs +++ b/src/update/common.rs @@ -5,4 +5,5 @@ pub struct Config { pub temp_folder: String, pub file_folder: String, pub db_folder: String, + pub asynchronous: bool, } diff --git a/src/update/error.rs b/src/update/error.rs index d6e37a3..8ac95b6 100644 --- a/src/update/error.rs +++ b/src/update/error.rs @@ -28,11 +28,12 @@ custom_error! { pub Error InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error}).", SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override.", SyncInseeError {source: InseeUpdateError} = "[SyncInsee] {source}", + WaitThreadError {source: tokio::task::JoinError} = "[Asynchronous] Error while waiting for thread: {source}", } impl Error { pub fn exit(&self) -> ! { - eprintln!("{}", self); + log::error!("{}", self); process::exit(1); } } diff --git a/src/update/mod.rs b/src/update/mod.rs index 7584b38..63e49d8 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -5,6 +5,7 @@ use action::execute_step; use chrono::Utc; use common::Config; use error::Error; +use tokio::task; pub mod action; pub mod common; @@ -42,9 +43,6 @@ async fn execute_workflow( config: Config, connectors: &mut Connectors, ) -> Result { - // Start - println!("[Update] Starting"); - // Execute workflow let mut summary = UpdateSummary::new(); @@ -55,25 +53,59 @@ async fn execute_workflow( config.data_only, )?; + let asynchronous = config.asynchronous; + let started_summary = summary.clone(); + let mut thread_connectors = connectors.clone(); + + let handle = task::spawn(async move { + task::yield_now().await; + + execute_workflow_thread( + workflow, + synthetic_group_type, + config, + &mut thread_connectors, + summary, + ) + .await + }); + + if asynchronous { + Ok(started_summary) + } else { + handle.await? + } +} + +async fn execute_workflow_thread( + workflow: Vec, + synthetic_group_type: SyntheticGroupType, + config: Config, + mut connectors: &mut Connectors, + mut summary: UpdateSummary, +) -> Result { + log::debug!("[Update] Starting"); + for step in workflow.into_iter() { execute_step( step, &config, &synthetic_group_type.into(), - connectors, + &mut connectors, &mut summary.step_delegate(step), ) .await .or_else(|error| { - update_metadata::error_update(connectors, error.to_string(), Utc::now())?; + log::error!("[Update] Errored: {}", error.to_string()); + + update_metadata::error_update(&mut connectors, error.to_string(), Utc::now())?; Err(error) })?; } - summary.finish(connectors)?; + summary.finish(&mut connectors)?; - // End - println!("[Update] Finished"); + log::debug!("[Update] Finished"); Ok(summary) } From a231e7bec49d1ae14bb130bb356c8bd431a5c145 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 8 Oct 2020 00:53:06 +0200 Subject: [PATCH 63/69] Fix log config recommendations --- .env.sample | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index 680b081..6cb3ebf 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,4 @@ -RUST_LOG=api=trace +RUST_LOG=sirene=debug SIRENE_ENV=development HOST=localhost PORT=3000 diff --git a/README.md b/README.md index c44ea5f..62c96da 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ createdb --owner=sirene sirene Recommended configuration for production with docker: ``` -RUST_LOG=api=warn +RUST_LOG=sirene=warn SIRENE_ENV=production BASE_URL=[Your base URL, needed to update asynchronously] API_KEY=[Any randomized string, needed to use the HTTP admin endpoint] From ae1bffbd41bfedff1d2119c9269db2a539a17f59 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 8 Oct 2020 01:15:53 +0200 Subject: [PATCH 64/69] Fix async status --- src/commands/serve/runner/mod.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 8975d86..9e9ea47 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -90,14 +90,26 @@ async fn status(query: StatusQueryString, context: Context) -> Result StatusCode::ACCEPTED, + "error" => StatusCode::INTERNAL_SERVER_ERROR, + _ => StatusCode::OK, }; Ok(warp::reply::with_status( - warp::reply::json(&update_metadata), + warp::reply::with_header( + warp::reply::with_header( + warp::reply::json(&update_metadata), + "Location", + format!( + "{}/admin/update/status?api_key={}", + context.base_url.unwrap_or_default(), + api_key + ), + ), + "Retry-After", + "10", + ), status_code, )) } From 1090dc86231601c804220bb4f4c9de64f4d2c68d Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 8 Oct 2020 09:55:37 +0200 Subject: [PATCH 65/69] Return the same structure on update and on status --- src/commands/serve/runner/common.rs | 7 +----- src/commands/serve/runner/mod.rs | 38 ++++++++++------------------- src/update/mod.rs | 23 ++++++++--------- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index 62ad7cf..225c2f5 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -2,7 +2,7 @@ use super::super::super::common::FolderOptions; use crate::connectors::ConnectorsBuilders; use crate::models::etablissement::common::Etablissement; use crate::models::unite_legale::common::UniteLegale; -use crate::models::update_metadata::common::{SyntheticGroupType, UpdateSummary}; +use crate::models::update_metadata::common::SyntheticGroupType; use serde::{Deserialize, Serialize}; #[derive(Clone)] @@ -26,11 +26,6 @@ pub struct StatusQueryString { pub api_key: String, } -#[derive(Serialize)] -pub struct UpdateResponse { - pub summary: UpdateSummary, -} - #[derive(Serialize)] pub struct UniteLegaleResponse { pub unite_legale: UniteLegaleInnerResponse, diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index 9e9ea47..49d2de1 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -3,12 +3,13 @@ mod error; pub mod common; use crate::models; +use crate::models::update_metadata::common::UpdateMetadata; use crate::update::{common::Config as DataConfig, update as update_data}; use chrono::Utc; use common::{ Context, EtablissementInnerResponse, EtablissementResponse, StatusQueryString, UniteLegaleEtablissementInnerResponse, UniteLegaleInnerResponse, UniteLegaleResponse, - UpdateOptions, UpdateResponse, + UpdateOptions, }; use error::Error; use std::convert::Infallible; @@ -38,7 +39,7 @@ async fn update(options: UpdateOptions, context: Context) -> Result Result Result { @@ -90,6 +70,14 @@ async fn status(query: StatusQueryString, context: Context) -> Result, + api_key: &String, +) -> Result { let status_code = match update_metadata.status.as_str() { "launched" => StatusCode::ACCEPTED, "error" => StatusCode::INTERNAL_SERVER_ERROR, @@ -103,7 +91,7 @@ async fn status(query: StatusQueryString, context: Context) -> Result Result { +) -> Result { // Build and execute workflow execute_workflow( build_workflow(&config), @@ -32,7 +34,7 @@ pub async fn update_step( synthetic_group_type: SyntheticGroupType, config: Config, connectors: &mut Connectors, -) -> Result { +) -> Result { // Execute step execute_workflow(vec![step], synthetic_group_type, config, connectors).await } @@ -42,7 +44,7 @@ async fn execute_workflow( synthetic_group_type: SyntheticGroupType, config: Config, connectors: &mut Connectors, -) -> Result { +) -> Result { // Execute workflow let mut summary = UpdateSummary::new(); @@ -54,7 +56,6 @@ async fn execute_workflow( )?; let asynchronous = config.asynchronous; - let started_summary = summary.clone(); let mut thread_connectors = connectors.clone(); let handle = task::spawn(async move { @@ -70,11 +71,11 @@ async fn execute_workflow( .await }); - if asynchronous { - Ok(started_summary) - } else { - handle.await? + if !asynchronous { + handle.await??; } + + Ok(update_metadata::current_update(&connectors)?) } async fn execute_workflow_thread( @@ -83,7 +84,7 @@ async fn execute_workflow_thread( config: Config, mut connectors: &mut Connectors, mut summary: UpdateSummary, -) -> Result { +) -> Result<(), Error> { log::debug!("[Update] Starting"); for step in workflow.into_iter() { @@ -107,7 +108,7 @@ async fn execute_workflow_thread( log::debug!("[Update] Finished"); - Ok(summary) + Ok(()) } fn build_workflow(config: &Config) -> Vec { From fda557a9e8e535da522531e24bb46086181ffaa1 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Thu, 8 Oct 2020 09:55:46 +0200 Subject: [PATCH 66/69] Improve errors --- src/update/error.rs | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/update/error.rs b/src/update/error.rs index 8ac95b6..07b059c 100644 --- a/src/update/error.rs +++ b/src/update/error.rs @@ -6,27 +6,27 @@ use custom_error::custom_error; use std::process; custom_error! { pub Error - MetadataModelError {source: group_metadata::error::Error} = "Error on Metadata model: {source}.", - UpdateMetadataModelError {source: update_metadata::error::Error} = "Error on UpdateMetadata model: {source}.", - UpdatableModelError {source: models::common::Error} = "Error on UpdatableModel model: {source}.", - TempFolderCreationError {io_error: std::io::Error} = "Unable to create temporary folder ({io_error}).", - FileFolderCreationError {io_error: std::io::Error} = "Unable to create data folder ({io_error}).", - FileCreationError {io_error: std::io::Error} = "Unable to create file for download ({io_error}).", - FileCopyError {io_error: std::io::Error} = "Unable to copy file from download ({io_error}).", - FileCSVCreationError {io_error: std::io::Error} = "Unable to create CSV file for unzip ({io_error}).", - FileCSVCopyError {io_error: std::io::Error} = "Unable to copy CSV file from archive ({io_error}).", - FileCSVPermissionError {io_error: std::io::Error} = "Unable to set permission for CSV file ({io_error}).", - DownloadError {req_error: reqwest::Error} = "Unable to download data from remote server ({req_error}).", - ZipOpenError {io_error: std::io::Error} = "Unable to open data zip file ({io_error}).", - ZipDecodeError {zip_error: zip::result::ZipError} = "Unable to decode zip file ({zip_error}).", - ZipFormatError = "Archive has more than one file inside it, you should review it before running it again.", - ZipAccessFileError {zip_error: zip::result::ZipError} = "Unable to open file in archive ({zip_error}).", - MissingLastModifiedHeader = "Needed header 'Last-Modified' is missing while downloading.", - InvalidLastModifiedHeader {head_error: reqwest::header::ToStrError} = "Needed header 'Last-Modified' is invalid while downloading ({head_error}).", - InvalidLastModifiedDate {date_error: chrono::format::ParseError} = "Needed header 'Last-Modified' is invalid while downloading ({date_error}).", - InvalidCSVPath = "Invalid CSV path, not UTF8 compatible.", - InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error}).", - SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override.", + MetadataModelError {source: group_metadata::error::Error} = "Error on Metadata model: {source}", + UpdateMetadataModelError {source: update_metadata::error::Error} = "Error on UpdateMetadata model: {source}", + UpdatableModelError {source: models::common::Error} = "Error on UpdatableModel model: {source}", + TempFolderCreationError {io_error: std::io::Error} = "Unable to create temporary folder ({io_error})", + FileFolderCreationError {io_error: std::io::Error} = "Unable to create data folder ({io_error})", + FileCreationError {io_error: std::io::Error} = "Unable to create file for download ({io_error})", + FileCopyError {io_error: std::io::Error} = "Unable to copy file from download ({io_error})", + FileCSVCreationError {io_error: std::io::Error} = "Unable to create CSV file for unzip ({io_error})", + FileCSVCopyError {io_error: std::io::Error} = "Unable to copy CSV file from archive ({io_error})", + FileCSVPermissionError {io_error: std::io::Error} = "Unable to set permission for CSV file ({io_error})", + DownloadError {req_error: reqwest::Error} = "Unable to download data from remote server ({req_error})", + ZipOpenError {io_error: std::io::Error} = "Unable to open data zip file ({io_error})", + ZipDecodeError {zip_error: zip::result::ZipError} = "Unable to decode zip file ({zip_error})", + ZipFormatError = "Archive has more than one file inside it, you should review it before running it again", + ZipAccessFileError {zip_error: zip::result::ZipError} = "Unable to open file in archive ({zip_error})", + MissingLastModifiedHeader = "Needed header 'Last-Modified' is missing while downloading", + InvalidLastModifiedHeader {head_error: reqwest::header::ToStrError} = "Needed header 'Last-Modified' is invalid while downloading ({head_error})", + InvalidLastModifiedDate {date_error: chrono::format::ParseError} = "Needed header 'Last-Modified' is invalid while downloading ({date_error})", + InvalidCSVPath = "Invalid CSV path, not UTF8 compatible", + InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error})", + SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override", SyncInseeError {source: InseeUpdateError} = "[SyncInsee] {source}", WaitThreadError {source: tokio::task::JoinError} = "[Asynchronous] Error while waiting for thread: {source}", } From 52e93a420830ece7acdc0e5d2a7264df982a1887 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Fri, 9 Oct 2020 16:34:52 +0200 Subject: [PATCH 67/69] Fix Windows build --- Cargo.toml | 4 +++- src/main.rs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2637082..e869c58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" futures = "0.3" log = "0.4" -openssl = "0.10" pretty_env_logger = "0.4" r2d2 = "0.8" reqwest = { version = "0.10", features = ["json", "stream"] } @@ -26,3 +25,6 @@ tokio = { version = "0.2", features = ["macros", "io-util", "fs", "rt-threaded", tokio-util = { version = "0.3", features = ["compat"] } warp = "0.2" zip = "0.5" + +[target.'cfg(any(unix, linux))'.dependencies] +openssl = "0.10" diff --git a/src/main.rs b/src/main.rs index 405b642..9878e56 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ extern crate chrono; #[macro_use] extern crate clap; extern crate custom_error; +#[cfg(any(target_os = "unix", target_os = "linux"))] extern crate openssl; // Should be before diesel #[macro_use] extern crate diesel; From db6924b2e82da0d7c290260b22029cdd00666be0 Mon Sep 17 00:00:00 2001 From: Julien Blatecky Date: Tue, 20 Oct 2020 14:33:32 +0200 Subject: [PATCH 68/69] Upgrade dependencies --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 866f0d8..67cdbf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,9 +8,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "aho-corasick" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" dependencies = [ "memchr", ] @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" [[package]] name = "cfg-if" @@ -1045,18 +1045,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fbdfd6bdee3dc9be46452f86af4a4072975899cf8592466668620bebfbcc17" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82fb1329f632c3552cf352d14427d57a511b1cf41db93b3a7d77906a82dcc8e" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2", "quote", @@ -1077,9 +1077,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "ppv-lite86" @@ -1341,9 +1341,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "regex" -version = "1.3.9" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" dependencies = [ "aho-corasick", "memchr", @@ -1353,9 +1353,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" [[package]] name = "remove_dir_all" @@ -1470,18 +1470,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ "proc-macro2", "quote", @@ -1490,9 +1490,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" dependencies = [ "itoa", "ryu", @@ -1593,9 +1593,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.42" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556" dependencies = [ "proc-macro2", "quote", From 66b4d7621d4f41cf8d437123c6a5a3a2bb5f331f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 13:02:30 +0200 Subject: [PATCH 69/69] Create Dependabot config file (#167) Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..212fc49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + time: "04:00" + open-pull-requests-limit: 10 + target-branch: develop + reviewers: + - julien1619 + assignees: + - julien1619