diff --git a/.env.sample b/.env.sample index 7a1a670..6cb3ebf 100644 --- a/.env.sample +++ b/.env.sample @@ -1,9 +1,12 @@ -RUST_LOG=api=trace +RUST_LOG=sirene=debug SIRENE_ENV=development HOST=localhost PORT=3000 +BASE_URL=http://localhost:3000 API_KEY= DATABASE_URL=postgresql://sirene:sirenepw@127.0.0.1:5432/sirene +DATABASE_POOL_SIZE=100 TEMP_FOLDER=./data/temp FILE_FOLDER=./data/files DB_FOLDER=./data/files +INSEE_CREDENTIALS=Base64(consumer-key:consumer-secret) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..212fc49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + time: "04:00" + open-pull-requests-limit: 10 + target-branch: develop + reviewers: + - julien1619 + assignees: + - julien1619 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9ab22a5..cbd2180 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,22 +4,39 @@ on: [push] jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Install minimal nightly - uses: actions-rs/toolchain@v1 + - name: Checkout repository + uses: actions/checkout@v2 + - name: Caching + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-musl-${{ hashFiles('**/Cargo.lock') }} + - name: Build for MUSL + run: | + mkdir -p ~/.cargo/{git,registry} + docker pull clux/muslrust:stable + docker run \ + -v ~/.cargo/registry:/root/.cargo/registry \ + -v ~/.cargo/git:/root/.cargo/git \ + -v $GITHUB_WORKSPACE:/volume \ + --rm -t clux/muslrust:stable \ + sh -c "/volume/build-musl.sh && chown -R $(id -u):$(id -g) ./target /root/.cargo/registry /root/.cargo/git" + - name: Archive production artifact + uses: actions/upload-artifact@v2 + with: + name: sirene + path: dist/sirene + - name: Push to Docker Hub + uses: docker/build-push-action@v1 with: - toolchain: nightly - profile: minimal - override: true - - name: Build - uses: actions-rs/cargo@v1 - with: - command: build - - name: Run tests - uses: actions-rs/cargo@v1 - with: - command: test + path: "./dist" + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: creatiwity/siren + tag_with_ref: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 32d6b42..0000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: rust - -cache: cargo - -services: - - docker - -env: - global: - - IMAGE_NAME=creatiwity/siren - - secure: eHzYmuj8kykBWbJpsm3PAyHohEmpoLWoewkn3K4OrTyrJwHbzt2OLFTXpaJ3ivtX3BVcJNWiri0pfCt4VUCJTg1kLJpMFcAb4q2drSTKBdpisufn3dy4b4izuRthfildeeRkp7/lR+MUKsTbuiM3BaMQi6lu8u5B8KwOmvsAmwsea06mpajDRANnP7oherHN28YdQD0dn89TEa0/AdHMoLfuTHjX/YY02MzLX+IQ0Fw/Y62M3Ma0FELSNLomv/C94MpDY1l2PxviTx0glMq1mYEDn+sJ+lQcvkYSkSvVe2+K9sRd3LZXN8pAXllM8ZHYbCEtSAsYxkdAj4zpkcO8ekrJiN1A6YkmreVxtNozGi92GfrPy7XPacEJIMNIa7l7EsUtQwbtPnu5vh7YtTL6H0smCYjTYLfY9kWjqU0Vd5s9dS/0q4VPApBNe3NhxITQyUr3y5IdrnJfDUvn3OoBFPl8Q572zFkIC9UhtqmiY+FqjqpVwSpJw12T6Hnf3W3sJknE9LQCojYmQK5wu9A09mb7sfYDqWSSxJ8DDWtbSWtA6+USiZC5tSpFuQr8aMqG4GbRGnvoA7VLDBlYvcidN/u2uo6uLISQrj+uaQNvUwIlwaurt8YTZ5FlUDjxXyjyLOOSmT974Ev3caIjpN58xRTivyTfnREXgU8AcWS3Ma0= - - secure: lVy3FtGBuilGmFwj4E8fXXeQNphq8H1w5ofjrZSJFkuacypShC2ATWijfTZcQO5DUFEP1ODonqtvWJDBgpSnNktwP34bwwUy3uTG6GjNXqY7R+sUMAzzlqQBteqxLvN91OwlaOB0G0gpEDrGTWLc1Wjt7otVo+iRWDN60pukQ5k4+2ePBATF6Mgc0PfcgiBMUh2dh9Q21XTyDw3TyWWqPs76Cs87XwkGiY6lcXRGnBa6ZQYe5KxyR+M9KpffT3ck9AmOCN+QDcaYVgCaXgH3qGYzn2nAXEp8p59cxLyMBKyN/anEJp8TfZ12Aa3BkzmYQeHJSVtYJCyRkci0jXilIwa5Y3mHJqS5Qof/cxayQ3ZogKFObig6I5B3WC45UYansIEKDIp+Z4Mct8cS0IWfyPwMPvv/+MwkbqOKWbSt75DTEyNAn4W+JXAcjhTFUJFDvZVVEJCszghb0UC0RGMEDSNb09Si4njZhpfVmfFCbdKgln/cTkj6AYdZ74CZEI6LX+Zon59FGnPQNs9KjlNBc34bmOMUv1TD/VsoravlYm0mGkiOs1JqHBwdap88wucqbkiuyqa7MSrlvIRYc8V+3T04Nt52fN1hz5TRUG+WGyuwaITB1tlIiO5DmC35CDboHdu+u1GfDYbCD0x7CML4yG9vA873vHDx8JQ4cpxij8E= - -before_script: - - | - if [ "$TRAVIS_BRANCH" = "master" ]; then - DOCKER_TAG=latest - else - DOCKER_TAG="$TRAVIS_BRANCH" - fi - -script: - - docker run -v $TRAVIS_BUILD_DIR:/volume --rm -t clux/muslrust:nightly /volume/build-musl.sh - - docker build --target app --tag $IMAGE_NAME:$DOCKER_TAG --file ./no-builder.Dockerfile "." - -after_success: - - | - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker push $IMAGE_NAME:$DOCKER_TAG - - if [ "$TRAVIS_BRANCH" = "master" ]; then - docker run -v $PWD:/workspace \ - -e DOCKERHUB_USERNAME='$DOCKER_USERNAME' \ - -e DOCKERHUB_PASSWORD='$DOCKER_PASSWORD' \ - -e DOCKERHUB_REPOSITORY='$IMAGE_NAME' \ - -e README_FILEPATH='/workspace/README.md' \ - peterevans/dockerhub-description:2.0.0 - fi - fi - -branches: - only: - - master - - release/* - - develop - - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..0260fc1 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,9 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + + // List of extensions which should be recommended for users of this workspace. + "recommendations": ["matklad.rust-analyzer"], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [] +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..49d8fc1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,44 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'sirene' serve", + "cargo": { + "args": [ + "build", + "--bin=sirene", + "--package=sirene" + ], + "filter": { + "name": "sirene", + "kind": "bin" + } + }, + "args": ["serve"], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'sirene' update all", + "cargo": { + "args": [ + "build", + "--bin=sirene", + "--package=sirene" + ], + "filter": { + "name": "sirene", + "kind": "bin" + } + }, + "args": ["update", "all"], + "cwd": "${workspaceFolder}" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..02a859c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "editor.codeActionsOnSave": { + "source.fixAll": true, + "source.organizeImports": true + } +} diff --git a/Cargo.lock b/Cargo.lock index 2834d43..67cdbf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,24 +2,29 @@ # It is not intended for manual editing. [[package]] name = "adler32" -version = "1.0.4" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] -name = "ansi_term" -version = "0.12.1" +name = "aho-corasick" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" dependencies = [ - "winapi 0.3.8", + "memchr", ] [[package]] -name = "anyhow" -version = "1.0.26" +name = "async-trait" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "atty" @@ -29,51 +34,84 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] name = "autocfg" -version = "1.0.0" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "base64" -version = "0.9.3" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ + "block-padding", + "byte-tools", "byteorder", - "safemem", + "generic-array 0.12.3", ] [[package]] -name = "base64" -version = "0.10.1" +name = "block-buffer" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "byteorder", + "generic-array 0.14.4", ] [[package]] -name = "base64" -version = "0.11.0" +name = "block-padding" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] [[package]] -name = "bitflags" -version = "1.2.1" +name = "buf_redux" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", +] [[package]] name = "bumpalo" -version = "3.2.0" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" @@ -83,9 +121,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "bytes" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bzip2" @@ -99,28 +137,20 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.7" +version = "0.1.9+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" dependencies = [ "cc", "libc", -] - -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -dependencies = [ - "ppv-lite86", + "pkg-config", ] [[package]] name = "cc" -version = "1.0.50" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" [[package]] name = "cfg-if" @@ -130,28 +160,32 @@ checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "chrono" -version = "0.4.10" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31850b4a4d6bae316f7a09e691c944c28299298837edc0a03f755618c23cbc01" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "libc", "num-integer", "num-traits", "serde", "time", + "winapi 0.3.9", ] [[package]] name = "clap" -version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e4a7f5012855e9dd906caaa7028393cf1926da8e" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" dependencies = [ - "ansi_term", "atty", "bitflags", "clap_derive", "indexmap", "lazy_static", + "os_str_bytes", "strsim", + "termcolor", "textwrap", "unicode-width", "vec_map", @@ -159,14 +193,15 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.0.0-beta.1" -source = "git+https://github.com/clap-rs/clap.git#e4a7f5012855e9dd906caaa7028393cf1926da8e" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -179,22 +214,19 @@ dependencies = [ ] [[package]] -name = "cookie" -version = "0.11.2" +name = "cloudabi" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9fac5e7bdefb6160fb181ee0eaa6f96704b625c70e6d61c465cb35750a4ea12" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" dependencies = [ - "base64 0.9.3", - "ring", - "time", - "url 1.7.2", + "bitflags", ] [[package]] name = "core-foundation" -version = "0.6.4" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" dependencies = [ "core-foundation-sys", "libc", @@ -202,9 +234,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.6.2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" [[package]] name = "crc32fast" @@ -217,47 +255,15 @@ dependencies = [ [[package]] name = "custom_error" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a0fc65739ae998afc8d68e64bdac2efd1bc4ffa1a0703d171ef2defae3792f" - -[[package]] -name = "devise" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74e04ba2d03c5fa0d954c061fc8c9c288badadffc272ebb87679a89846de3ed3" -dependencies = [ - "devise_codegen", - "devise_core", -] - -[[package]] -name = "devise_codegen" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066ceb7928ca93a9bedc6d0e612a8a0424048b0ab1f75971b203d01420c055d7" -dependencies = [ - "devise_core", - "quote 0.6.13", -] - -[[package]] -name = "devise_core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf41c59b22b5e3ec0ea55c7847e5f358d340f3a8d6d53a5cf4f1564967f96487" -dependencies = [ - "bitflags", - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", -] +checksum = "51ac5e99a7fea3ee8a03fa4721a47e2efd3fbb38358fc61192a54d4c6f866c12" [[package]] name = "diesel" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7cc03b910de9935007861dce440881f69102aaaedfd4bc5a6f40340ca5840c" +checksum = "3e2de9deab977a153492a1468d1b1c0662c1cf39e5ea87d0c060ecd59ef18d8c" dependencies = [ "bitflags", "byteorder", @@ -274,9 +280,9 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -289,6 +295,24 @@ dependencies = [ "migrations_macros", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.3", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.4", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -297,36 +321,43 @@ checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] name = "dtoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "encoding_rs" -version = "0.8.22" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ "cfg-if", ] [[package]] -name = "filetime" -version = "0.2.8" +name = "env_logger" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "winapi 0.3.8", + "atty", + "humantime", + "log", + "regex", + "termcolor", ] +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "flate2" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", @@ -336,9 +367,9 @@ dependencies = [ [[package]] name = "fnv" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foreign-types" @@ -356,23 +387,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] -name = "fsevent" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6" -dependencies = [ - "bitflags", - "fsevent-sys", -] - -[[package]] -name = "fsevent-sys" -version = "2.0.1" +name = "fuchsia-cprng" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41b048a94555da0f42f1d632e2e19510084fb8e303b0daa2816e733fb3644a0" -dependencies = [ - "libc", -] +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "fuchsia-zircon" @@ -390,69 +408,136 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futures" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8e3078b7b2a8a671cb7a3d17b4760e4181ea243227776ba83fd043b4ca034e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" dependencies = [ "futures-core", + "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.4" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" + +[[package]] +name = "futures-executor" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +checksum = "cc709ca1da6f66143b8c9bec8e6260181869893714e9b5a490b169b0414144ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] [[package]] name = "futures-io" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" + +[[package]] +name = "futures-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f57ed14da4603b2554682e9f2ff3c65d7567b53188db96cb71538217fc64581b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] [[package]] name = "futures-sink" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" [[package]] name = "futures-task" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" +checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" +dependencies = [ + "once_cell", +] [[package]] name = "futures-util" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", + "pin-project", "pin-utils", + "proc-macro-hack", + "proc-macro-nested", "slab", ] +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] name = "h2" -version = "0.2.1" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1" +checksum = "993f9e0baeed60001cf565546b0d3dbe6a6ad23f2bd31644a133c641eccf6d53" dependencies = [ "bytes", "fnv", @@ -461,10 +546,41 @@ dependencies = [ "futures-util", "http", "indexmap", - "log 0.4.8", "slab", "tokio", "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "headers" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" +dependencies = [ + "base64", + "bitflags", + "bytes", + "headers-core", + "http", + "mime", + "sha-1 0.8.2", + "time", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", ] [[package]] @@ -478,18 +594,18 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.7" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c55f143919fbc0bc77e427fe2d74cf23786d7c1875666f2fde3ac3c659bb67" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" dependencies = [ "libc", ] [[package]] name = "http" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ "bytes", "fnv", @@ -513,29 +629,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] -name = "hyper" -version = "0.10.16" +name = "httpdate" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ - "base64 0.9.3", - "httparse", - "language-tags", - "log 0.3.9", - "mime 0.2.6", - "num_cpus", - "time", - "traitobject", - "typeable", - "unicase 1.4.2", - "url 1.7.2", + "quick-error", ] [[package]] name = "hyper" -version = "0.13.2" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1c527bbc634be72aa7ba31e4e4def9bbb020f5416916279b7c705cd838893e" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" dependencies = [ "bytes", "futures-channel", @@ -545,40 +657,29 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", - "log 0.4.8", - "net2", "pin-project", - "time", + "socket2", "tokio", "tower-service", + "tracing", "want", ] [[package]] name = "hyper-tls" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" dependencies = [ "bytes", - "hyper 0.13.2", + "hyper", "native-tls", "tokio", "tokio-tls", ] -[[package]] -name = "idna" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.2.0" @@ -592,31 +693,30 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.3.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ - "autocfg", + "autocfg 1.0.1", + "hashbrown", ] [[package]] -name = "inotify" -version = "0.7.0" +name = "input_buffer" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e40d6fd5d64e2082e0c796495c8ef5ad667a96d03e5aaa0becfd9d47bcbfb8" +checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" dependencies = [ - "bitflags", - "inotify-sys", - "libc", + "bytes", ] [[package]] -name = "inotify-sys" -version = "0.1.3" +name = "instant" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74a1aa87c59aeff6ef2cc2fa62d41bc43f54952f55652656b18a02fd5e356c0" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" dependencies = [ - "libc", + "cfg-if", ] [[package]] @@ -628,17 +728,23 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" [[package]] name = "js-sys" -version = "0.3.35" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7889c7c36282151f6bf465be4700359318aef36baa951462382eae49e9577cf9" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] @@ -653,53 +759,32 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" - [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" - [[package]] name = "libc" -version = "0.2.66" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" [[package]] name = "lock_api" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -dependencies = [ - "log 0.4.8", -] - -[[package]] -name = "log" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ "cfg-if", ] @@ -710,46 +795,31 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53445de381a1f436797497c61d851644d0e8e88e6140f22872ad33a704933978" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "migrations_internals" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8089920229070f914b9ce9b07ef60e175b2b9bc2d35c3edd8bf4433604e863b9" +checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" dependencies = [ "diesel", ] [[package]] name = "migrations_macros" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719ef0bc7f531428764c9b70661c14abd50a7f3d21f355752d9985aa21251c9e" +checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" dependencies = [ "migrations_internals", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", -] - -[[package]] -name = "mime" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -dependencies = [ - "log 0.3.9", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -760,28 +830,28 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" dependencies = [ - "mime 0.3.16", - "unicase 2.6.0", + "mime", + "unicase", ] [[package]] name = "miniz_oxide" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa679ff6578b1cddee93d7e82e263b94a575e0bfced07284eb0c037c1d2416a5" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" dependencies = [ "adler32", ] [[package]] name = "mio" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ "cfg-if", "fuchsia-zircon", @@ -789,25 +859,13 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.8", + "log", "miow", "net2", "slab", "winapi 0.2.8", ] -[[package]] -name = "mio-extras" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" -dependencies = [ - "lazycell", - "log 0.4.8", - "mio", - "slab", -] - [[package]] name = "miow" version = "0.2.1" @@ -820,15 +878,33 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "multipart" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8209c33c951f07387a8497841122fc6f712165e3f9bda3e6be4645b58188f676" +dependencies = [ + "buf_redux", + "httparse", + "log", + "mime", + "mime_guess", + "quick-error", + "rand 0.6.5", + "safemem", + "tempfile", + "twoway", +] + [[package]] name = "native-tls" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" dependencies = [ "lazy_static", "libc", - "log 0.4.8", + "log", "openssl", "openssl-probe", "openssl-sys", @@ -840,77 +916,67 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.33" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ "cfg-if", "libc", - "winapi 0.3.8", -] - -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - -[[package]] -name = "notify" -version = "4.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" -dependencies = [ - "bitflags", - "filetime", - "fsevent", - "fsevent-sys", - "inotify", - "libc", - "mio", - "mio-extras", - "walkdir", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] name = "num-integer" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", ] +[[package]] +name = "once_cell" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" -version = "0.10.28" +version = "0.10.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973293749822d7dd6370d6da1e523b0d1db19f06c459134c658b2a4261378b52" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", "cfg-if", @@ -928,69 +994,49 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.54" +version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1024c0a59774200a555087a6da3f253a9095a5f344e353b212ac4c8b8e450986" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cc", "libc", "pkg-config", "vcpkg", ] +[[package]] +name = "os_str_bytes" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac6fe3538f701e339953a3ebbe4f39941aababa8a3f6964635b24ab526daeac" + [[package]] name = "parking_lot" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" dependencies = [ + "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ "cfg-if", - "cloudabi", + "cloudabi 0.1.0", + "instant", "libc", "redox_syscall", - "smallvec 1.2.0", - "winapi 0.3.8", -] - -[[package]] -name = "pear" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26d2b92e47063ffce70d3e3b1bd097af121a9e0db07ca38a6cc1cf0cc85ff25" -dependencies = [ - "pear_codegen", -] - -[[package]] -name = "pear_codegen" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336db4a192cc7f54efeb0c4e11a9245394824cc3bcbd37ba3ff51240c35d7a6e" -dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "syn 0.15.44", - "version_check 0.1.5", - "yansi 0.4.0", + "smallvec", + "winapi 0.3.9", ] -[[package]] -name = "percent-encoding" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" - [[package]] name = "percent-encoding" version = "2.1.0" @@ -999,53 +1045,47 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "pin-project-lite" -version = "0.1.4" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" +checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" - -[[package]] -name = "podio" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "pq-sys" @@ -1056,79 +1096,106 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "proc-macro-error" -version = "0.4.9" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052b3c9af39c7e5e94245f820530487d19eb285faedcb40e0c3275132293f242" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.8", - "quote 1.0.2", - "rustversion", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", + "version_check", ] [[package]] name = "proc-macro-error-attr" -version = "0.4.9" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d175bef481c7902e63e3165627123fff3502f06ac043d3ef42d08c1246da9253" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "rustversion", - "syn 1.0.14", - "syn-mid", + "proc-macro2", + "quote", + "version_check", ] [[package]] -name = "proc-macro2" -version = "0.4.30" +name = "proc-macro-hack" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" -dependencies = [ - "unicode-xid 0.1.0", -] +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.8" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid 0.2.0", + "unicode-xid", ] [[package]] -name = "quote" -version = "0.6.13" +name = "quick-error" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" -dependencies = [ - "proc-macro2 0.4.30", -] +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.2" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2", ] [[package]] name = "r2d2" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1497e40855348e4a8a40767d8e55174bce1e445a3ac9254ad44ad468ee0485af" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" dependencies = [ - "log 0.4.8", + "log", "parking_lot", "scheduled-thread-pool", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + [[package]] name = "rand" version = "0.7.3" @@ -1137,21 +1204,46 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom", "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", ] [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" dependencies = [ - "c2-chacha", - "rand_core", + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -1161,158 +1253,160 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", ] [[package]] -name = "redox_syscall" -version = "0.1.56" +name = "rand_isaac" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] [[package]] -name = "remove_dir_all" -version = "0.5.2" +name = "rand_jitter" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" dependencies = [ - "winapi 0.3.8", + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", ] [[package]] -name = "reqwest" -version = "0.10.2" +name = "rand_os" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae3fc32eacd4a5200c6b34bd6c057b07fb64f5a1e55bb67d624cc1393354621" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "base64 0.11.0", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper 0.13.2", - "hyper-tls", - "js-sys", - "lazy_static", - "log 0.4.8", - "mime 0.3.16", - "mime_guess", - "native-tls", - "percent-encoding 2.1.0", - "pin-project-lite", - "serde", - "serde_urlencoded", - "time", - "tokio", - "tokio-tls", - "url 2.1.1", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", + "cloudabi 0.0.3", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", ] [[package]] -name = "ring" -version = "0.13.5" +name = "rand_pcg" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4db68a2e35f3497146b7e4563df7d4773a2433230c5e4b448328e31740458a" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "cc", - "lazy_static", - "libc", - "untrusted", + "autocfg 0.1.7", + "rand_core 0.4.2", ] [[package]] -name = "rocket" -version = "0.4.2" +name = "rand_xorshift" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42c1e9deb3ef4fa430d307bfccd4231434b707ca1328fae339c43ad1201cc6f7" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" dependencies = [ - "atty", - "base64 0.10.1", - "log 0.4.8", - "memchr", - "num_cpus", - "pear", - "rocket_codegen", - "rocket_http", - "state", - "time", - "toml", - "version_check 0.9.1", - "yansi 0.5.0", + "rand_core 0.3.1", ] [[package]] -name = "rocket_codegen" -version = "0.4.2" +name = "rdrand" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79aa1366f9b2eccddc05971e17c5de7bb75a5431eb12c2b5c66545fd348647f4" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" dependencies = [ - "devise", - "indexmap", - "quote 0.6.13", - "rocket_http", - "version_check 0.9.1", - "yansi 0.5.0", + "rand_core 0.3.1", ] [[package]] -name = "rocket_contrib" -version = "0.4.2" +name = "redox_syscall" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0fa5c1392135adc0f96a02ba150ac4c765e27c58dbfd32aa40678e948f6e56f" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" dependencies = [ - "log 0.4.8", - "notify", - "rocket", - "serde", - "serde_json", + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", ] [[package]] -name = "rocket_http" -version = "0.4.2" +name = "regex-syntax" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1391457ee4e80b40d4b57fa5765c0f2836b20d73bcbee4e3f35d93cf3b80817" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "cookie", - "hyper 0.10.16", - "indexmap", - "pear", - "percent-encoding 1.0.1", - "smallvec 0.6.13", - "state", - "time", - "unicode-xid 0.1.0", + "winapi 0.3.9", ] [[package]] -name = "rustversion" -version = "1.0.2" +name = "reqwest" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", ] [[package]] name = "ryu" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "safemem" @@ -1320,34 +1414,31 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507a9e6e8ffe0a4e0ebb9a10293e62fdf7657c06f1b8bb07a8fcf697d2abf295" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static", - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] name = "scheduled-thread-pool" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5de7bc31f28f8e6c28df5e1bf3d10610f5fdc14cc95f272853512c70a2bd779" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" dependencies = [ "parking_lot", ] +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1356,10 +1447,11 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "0.3.4" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ef2429d7cefe5fd28bd1d2ed41c944547d4ff84776f5935b456da44593a16df" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" dependencies = [ + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -1368,38 +1460,39 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "0.3.3" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31493fc37615debb8c5090a7aeb4a9730bc61e77ab10b9af59f1a202284f895" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" dependencies = [ "core-foundation-sys", + "libc", ] [[package]] name = "serde" -version = "1.0.104" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "serde_json" -version = "1.0.48" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" dependencies = [ "itoa", "ryu", @@ -1415,26 +1508,56 @@ dependencies = [ "dtoa", "itoa", "serde", - "url 2.1.1", + "url", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", ] [[package]] name = "sirene" -version = "1.0.3" +version = "2.0.0" dependencies = [ + "async-trait", "chrono", "clap", "custom_error", "diesel", "diesel_migrations", "dotenv", + "futures", + "log", "openssl", + "pretty_env_logger", "r2d2", "reqwest", - "rocket", - "rocket_contrib", "serde", "serde_json", + "tokio", + "tokio-util", + "warp", "zip", ] @@ -1446,112 +1569,126 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "0.6.13" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" -dependencies = [ - "maybe-uninit", -] +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] -name = "smallvec" -version = "1.2.0" +name = "socket2" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.9", +] [[package]] -name = "sourcefile" -version = "0.1.4" +name = "strsim" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "state" -version = "0.4.1" +name = "syn" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] [[package]] -name = "strsim" -version = "0.9.3" +name = "tempfile" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] [[package]] -name = "syn" -version = "0.15.44" +name = "termcolor" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" dependencies = [ - "proc-macro2 0.4.30", - "quote 0.6.13", - "unicode-xid 0.1.0", + "winapi-util", ] [[package]] -name = "syn" -version = "1.0.14" +name = "textwrap" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "unicode-xid 0.2.0", + "unicode-width", ] [[package]] -name = "syn-mid" -version = "0.5.0" +name = "thiserror" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" +checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "thiserror-impl", ] [[package]] -name = "tempfile" -version = "3.1.0" +name = "thiserror-impl" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" dependencies = [ - "cfg-if", - "libc", - "rand", - "redox_syscall", - "remove_dir_all", - "winapi 0.3.8", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "textwrap" -version = "0.11.0" +name = "thread_local" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" dependencies = [ - "unicode-width", + "lazy_static", ] [[package]] name = "time" -version = "0.1.42" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", - "redox_syscall", - "winapi 0.3.8", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" + [[package]] name = "tokio" -version = "0.2.11" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fdd17989496f49cdc57978c96f0c9fe5e4a58a8bddc6813c449a4624f6a030b" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" dependencies = [ "bytes", "fnv", + "futures-core", "iovec", "lazy_static", "memchr", @@ -1559,81 +1696,142 @@ dependencies = [ "num_cpus", "pin-project-lite", "slab", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] name = "tokio-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" dependencies = [ "native-tls", "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" +dependencies = [ + "futures-util", + "log", + "pin-project", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", - "log 0.4.8", + "log", "pin-project-lite", "tokio", ] [[package]] -name = "toml" -version = "0.4.10" +name = "tower-service" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" dependencies = [ - "serde", + "cfg-if", + "log", + "pin-project-lite", + "tracing-core", ] [[package]] -name = "tower-service" -version = "0.3.0" +name = "tracing-core" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] [[package]] -name = "traitobject" -version = "0.1.0" +name = "tracing-futures" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project", + "tracing", +] [[package]] name = "try-lock" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] -name = "typeable" -version = "0.1.2" +name = "tungstenite" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" +dependencies = [ + "base64", + "byteorder", + "bytes", + "http", + "httparse", + "input_buffer", + "log", + "rand 0.7.3", + "sha-1 0.9.1", + "url", + "utf-8", +] [[package]] -name = "unicase" -version = "1.4.2" +name = "twoway" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" dependencies = [ - "version_check 0.1.5", + "memchr", ] +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicase" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.9.1", + "version_check", ] [[package]] @@ -1647,11 +1845,11 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" dependencies = [ - "smallvec 1.2.0", + "tinyvec", ] [[package]] @@ -1662,38 +1860,15 @@ checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" [[package]] name = "unicode-width" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" - -[[package]] -name = "unicode-xid" -version = "0.1.0" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" - -[[package]] -name = "untrusted" -version = "0.6.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cd1f4b4e96b46aeb8d4855db4a7a9bd96eeeb5c6a1ab54593328761642ce2f" - -[[package]] -name = "url" -version = "1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" -dependencies = [ - "idna 0.1.5", - "matches", - "percent-encoding 1.0.1", -] +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" [[package]] name = "url" @@ -1701,45 +1876,40 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" dependencies = [ - "idna 0.2.0", + "idna", "matches", - "percent-encoding 2.1.0", + "percent-encoding", ] [[package]] -name = "vcpkg" -version = "0.2.8" +name = "urlencoding" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" [[package]] -name = "vec_map" -version = "0.8.1" +name = "utf-8" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" [[package]] -name = "version_check" -version = "0.1.5" +name = "vcpkg" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" [[package]] -name = "version_check" -version = "0.9.1" +name = "vec_map" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] -name = "walkdir" -version = "2.3.1" +name = "version_check" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" -dependencies = [ - "same-file", - "winapi 0.3.8", - "winapi-util", -] +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" [[package]] name = "want" @@ -1747,21 +1917,55 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.8", + "log", "try-lock", ] +[[package]] +name = "warp" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" +dependencies = [ + "bytes", + "futures", + "headers", + "http", + "hyper", + "log", + "mime", + "mime_guess", + "multipart", + "pin-project", + "scoped-tls", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-tungstenite", + "tower-service", + "tracing", + "tracing-futures", + "urlencoding", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-bindgen" -version = "0.2.58" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5205e9afdf42282b192e2310a5b463a6d1c1d774e30dc3c791ac37ab42d2616c" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if", "serde", @@ -1771,24 +1975,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.58" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11cdb95816290b525b32587d76419facd99662a07e59d3cdb560488a819d9a45" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "log", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.8" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbdd49e3e28b40dec6a9ba8d17798245ce32b019513a845369c641b275135d9" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ "cfg-if", "js-sys", @@ -1798,69 +2002,41 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.58" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ - "quote 1.0.2", + "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.58" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", + "proc-macro2", + "quote", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.58" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e7e61fc929f4c0dddb748b102ebf9f632e2b8d739f2016542b4de2965a9601" - -[[package]] -name = "wasm-bindgen-webidl" -version = "0.2.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef012a0d93fc0432df126a8eaf547b2dce25a8ce9212e1d3cbeef5c11157975d" -dependencies = [ - "anyhow", - "heck", - "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", - "syn 1.0.14", - "wasm-bindgen-backend", - "weedle", -] +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" [[package]] name = "web-sys" -version = "0.3.35" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf97caf6aa8c2b1dac90faf0db529d9d63c93846cca4911856f78a83cebf53b" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" dependencies = [ - "anyhow", "js-sys", - "sourcefile", "wasm-bindgen", - "wasm-bindgen-webidl", -] - -[[package]] -name = "weedle" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164" -dependencies = [ - "nom", ] [[package]] @@ -1871,9 +2047,9 @@ checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -1893,11 +2069,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1908,11 +2084,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.8", + "winapi 0.3.9", ] [[package]] @@ -1925,27 +2101,16 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "yansi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60c3b48c9cdec42fb06b3b84b5b087405e1fa1c644a1af3930e4dfafe93de48" - -[[package]] -name = "yansi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" - [[package]] name = "zip" -version = "0.5.4" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e41ff37ba788e2169b19fa70253b70cb53d9f2db9fb9aea9bcfc5047e02c3bae" +checksum = "543adf038106b64cfca4711c82c917d785e3540e04f7996554488f988ec43124" dependencies = [ + "byteorder", "bzip2", "crc32fast", "flate2", - "podio", + "thiserror", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 80e28c4..e869c58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,27 +1,30 @@ [package] name = "sirene" -version = "1.0.3" +version = "2.0.0" authors = ["Julien Blatecky "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } -clap = { git = "https://github.com/clap-rs/clap.git" } -custom_error = "1.7" +clap = "3.0.0-beta" +custom_error = "1.8" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2", "serde_json", "64-column-tables"] } diesel_migrations = { version = "1.4", features = ["postgres"] } dotenv = "0.15" -openssl = "0.10" +futures = "0.3" +log = "0.4" +pretty_env_logger = "0.4" r2d2 = "0.8" -reqwest = { version = "0.10", features = ["blocking"] } -rocket = "0.4.2" +reqwest = { version = "0.10", features = ["json", "stream"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +tokio = { version = "0.2", features = ["macros", "io-util", "fs", "rt-threaded", "time"] } +tokio-util = { version = "0.3", features = ["compat"] } +warp = "0.2" zip = "0.5" -[dependencies.rocket_contrib] -version = "0.4.2" -default-features = false -features = ["json"] +[target.'cfg(any(unix, linux))'.dependencies] +openssl = "0.10" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 082131c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM clux/muslrust:nightly AS builder -RUN rustc --version && cargo --version -COPY Cargo.toml Cargo.lock ./ -COPY migrations ./migrations/ -COPY src ./src/ -RUN cargo build --release --target x86_64-unknown-linux-musl -RUN strip target/x86_64-unknown-linux-musl/release/sirene -RUN cp target/x86_64-unknown-linux-musl/release/sirene / - -FROM alpine AS app -WORKDIR /app -COPY --from=builder /sirene /app/ -ENV HOST 0.0.0.0 -ENV PORT 3000 -EXPOSE 3000 -CMD ["/bin/sh", "-c", "./sirene serve"] diff --git a/README.md b/README.md index cd28695..62c96da 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,39 @@ createdb --owner=sirene sirene ## Documentation +### Configuration + +Recommended configuration for production with docker: + +``` +RUST_LOG=sirene=warn +SIRENE_ENV=production +BASE_URL=[Your base URL, needed to update asynchronously] +API_KEY=[Any randomized string, needed to use the HTTP admin endpoint] +DATABASE_URL=postgresql://[USER]:[PASSWORD]@[PG_HOST]:[PG_PORT]/[PG_DATABASE] +DATABASE_POOL_SIZE=100 +INSEE_CREDENTIALS=[Base64(consumer-key:consumer-secret)] +``` + +**How to generate INSEE_CREDENTIALS** + +This variable is only needed if you want to have the daily updates. + +1. Go to https://api.insee.fr/catalogue/ +2. Create an account or sign in +3. Create an application on this portal +4. Subscribe this application to the *Sirene - V3* API +5. Generate a key pair in the application details +6. Copy the key from the `curl` example and paste it in `.env`: `Authorization: Basic [INSEE_CREDENTIALS]` + ### CLI -**> sirene** +**> sirene --help** ``` -> sirene -h +sirene 2.0.0 +Julien Blatecky +Sirene service used to update data in database and serve it through a HTTP REST API USAGE: sirene [OPTIONS] @@ -42,13 +69,15 @@ FLAGS: -V, --version Prints version information OPTIONS: - --db-folder Path to the file storage folder for the database, you can set in environment - variable as DB_FOLDER Could be the same as FILE_FOLDER if this app and the - database are on the same file system Files copied by this app inside FILE_FOLDER - must be visible by the database in DB_FOLDER - --file-folder Path to the file storage folder for this app, you can set in environment variable - as FILE_FOLDER - --temp-folder Path to the temp folder, you can set in environment variable as TEMP_FOLDER + --db-folder Path to the file storage folder for the database, you can set + in environment variable as DB_FOLDER. Could be the same as + FILE_FOLDER if this app and the database are on the same file + system. Files copied by this app inside FILE_FOLDER must be + visible by the database in DB_FOLDER + --file-folder Path to the file storage folder for this app, you can set in + environment variable as FILE_FOLDER + --temp-folder Path to the temp folder, you can set in environment variable + as TEMP_FOLDER SUBCOMMANDS: help Prints this message or the help of the given subcommand(s) @@ -59,6 +88,9 @@ SUBCOMMANDS: **> sirene serve --help** ``` +sirene-serve +Serve data from database to /unites_legales/ and /etablissements/ + USAGE: sirene serve [OPTIONS] @@ -67,25 +99,32 @@ FLAGS: -V, --version Prints version information OPTIONS: - -k, --api-key API key needed to allow maintenance operation from HTTP, you can set in environment - variable as API_KEY - --env Production, Staging or Development, will change log level, you can set in environment - variable as SIRENE_ENV - -h, --host Listen this host, you can set in environment variable as HOST - -p, --port Listen this port, you can set in environment variable as PORT + -k, --api-key API key needed to allow maintenance operation from HTTP, you can + set in environment variable as API_KEY + -b, --base-url Base URL needed to configure asynchronous polling for updates, you + can set in environment variable as BASE_URL + --env Configure log level, you can set in environment variable as + SIRENE_ENV [possible values: development, staging, production] + -h, --host Listen this host, you can set in environment variable as HOST + -p, --port Listen this port, you can set in environment variable as PORT ``` **> sirene update --help** ``` +sirene-update +Update data from CSV source files + USAGE: sirene update [FLAGS] [SUBCOMMAND] ARGS: - Configure which part will be updated, UnitesLegales, Etablissements or All + Configure which part will be updated [possible values: unites-legales, + etablissements, all] FLAGS: - --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete it + --data-only Use an existing CSV file already present in FILE_FOLDER and does not delete + it --force Force update even if the source data where not updated -h, --help Prints help information -V, --version Prints version information @@ -93,9 +132,12 @@ FLAGS: SUBCOMMANDS: clean-file Clean files from FILE_FOLDER download-file Download file in TEMP_FOLDER + finish-error Set a staled update process to error, use only if the process is really + stopped help Prints this message or the help of the given subcommand(s) insert-data Load CSV file in database in loader-table from DB_FOLDER swap-data Swap loader-table to production + sync-insee Synchronise daily data from INSEE since the last modification unzip-file Unzip file from TEMP_FOLDER, and move it to the FILE_FOLDER ``` @@ -117,7 +159,29 @@ POST /admin/update api_key: string, group_type: "UnitesLegales" | "Etablissements" | "All", force: bool, - data_only: bool, + asynchronous: bool, +} +``` + +If `asynchronous` is set to `true`, the update endpoint will immediately return the following: +``` +Status: 202 Accepted +Location: /admin/update/status?api_key=string +Retry-After: 10 + +[Initial status for the started update] +``` + +``` +GET /admin/update/status?api_key=string +``` +If an update is in progress, the status code will be 202, otherwise 200. + +``` +POST /admin/update/status/error + +{ + api_key: string, } ``` diff --git a/build-musl.sh b/build-musl.sh index 42ebdfc..8b9b35a 100755 --- a/build-musl.sh +++ b/build-musl.sh @@ -1,6 +1,8 @@ #! /bin/bash +set -e rustc --version && cargo --version cargo build --release --target x86_64-unknown-linux-musl strip target/x86_64-unknown-linux-musl/release/sirene -cp target/x86_64-unknown-linux-musl/release/sirene . +mkdir -p ./dist +cp target/x86_64-unknown-linux-musl/release/sirene ./dist/ diff --git a/no-builder.Dockerfile b/dist/Dockerfile similarity index 85% rename from no-builder.Dockerfile rename to dist/Dockerfile index 5a67490..7fe6800 100644 --- a/no-builder.Dockerfile +++ b/dist/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine AS app +FROM alpine WORKDIR /app COPY ./sirene /app/ ENV HOST 0.0.0.0 diff --git a/migrations/2020-06-05-141443_add_insee_synced_date/down.sql b/migrations/2020-06-05-141443_add_insee_synced_date/down.sql new file mode 100644 index 0000000..ddcef17 --- /dev/null +++ b/migrations/2020-06-05-141443_add_insee_synced_date/down.sql @@ -0,0 +1,9 @@ +ALTER TABLE "public"."group_metadata" DROP COLUMN "last_insee_synced_timestamp"; + +DROP INDEX "public"."unite_legale_staging_date_dernier_traitement_idx"; + +DROP INDEX "public"."etablissement_staging_date_dernier_traitement_idx"; + +DROP INDEX "public"."unite_legale_date_dernier_traitement_idx"; + +DROP INDEX "public"."etablissement_date_dernier_traitement_idx"; diff --git a/migrations/2020-06-05-141443_add_insee_synced_date/up.sql b/migrations/2020-06-05-141443_add_insee_synced_date/up.sql new file mode 100644 index 0000000..e867fc2 --- /dev/null +++ b/migrations/2020-06-05-141443_add_insee_synced_date/up.sql @@ -0,0 +1,7 @@ +ALTER TABLE "public"."group_metadata" ADD COLUMN "last_insee_synced_timestamp" timestamptz DEFAULT NULL; + +CREATE INDEX "unite_legale_staging_date_dernier_traitement_idx" ON "public"."unite_legale_staging" USING BTREE ("date_dernier_traitement"); +CREATE INDEX "unite_legale_date_dernier_traitement_idx" ON "public"."unite_legale" USING BTREE ("date_dernier_traitement"); + +CREATE INDEX "etablissement_staging_date_dernier_traitement_idx" ON "public"."etablissement_staging" USING BTREE ("date_dernier_traitement"); +CREATE INDEX "etablissement_date_dernier_traitement_idx" ON "public"."etablissement" USING BTREE ("date_dernier_traitement"); diff --git a/migrations/2020-06-26-132333_switch_to_https/down.sql b/migrations/2020-06-26-132333_switch_to_https/down.sql new file mode 100644 index 0000000..9630783 --- /dev/null +++ b/migrations/2020-06-26-132333_switch_to_https/down.sql @@ -0,0 +1,3 @@ +UPDATE "public"."group_metadata" SET "url" = 'http://files.data.gouv.fr/insee-sirene/StockEtablissement_utf8.zip' WHERE "group_type" = 'etablissements'; + +UPDATE "public"."group_metadata" SET "url" = 'http://files.data.gouv.fr/insee-sirene/StockUniteLegale_utf8.zip' WHERE "group_type" = 'unites_legales'; diff --git a/migrations/2020-06-26-132333_switch_to_https/up.sql b/migrations/2020-06-26-132333_switch_to_https/up.sql new file mode 100644 index 0000000..a971bf2 --- /dev/null +++ b/migrations/2020-06-26-132333_switch_to_https/up.sql @@ -0,0 +1,3 @@ +UPDATE "public"."group_metadata" SET "url" = 'https://files.data.gouv.fr/insee-sirene/StockEtablissement_utf8.zip' WHERE "group_type" = 'etablissements'; + +UPDATE "public"."group_metadata" SET "url" = 'https://files.data.gouv.fr/insee-sirene/StockUniteLegale_utf8.zip' WHERE "group_type" = 'unites_legales'; diff --git a/src/commands/common.rs b/src/commands/common.rs index cfcef5f..f79e02b 100644 --- a/src/commands/common.rs +++ b/src/commands/common.rs @@ -1,19 +1,18 @@ use crate::models::update_metadata::common::SyntheticGroupType; use serde::Deserialize; +#[derive(Clone)] pub struct FolderOptions { pub temp: String, pub file: String, pub db: String, } -arg_enum! { - #[derive(Debug, Deserialize, Clone, Copy)] - pub enum CmdGroupType { - UnitesLegales, - Etablissements, - All - } +#[derive(Clap, Debug, Deserialize, Clone, Copy)] +pub enum CmdGroupType { + UnitesLegales, + Etablissements, + All, } impl From for SyntheticGroupType { diff --git a/src/commands/mod.rs b/src/commands/mod.rs index f472294..f7a73e9 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -12,7 +12,7 @@ use update::UpdateFlags; /// Sirene service used to update data in database /// and serve it through a HTTP REST API #[derive(Clap, Debug)] -#[clap(version = "1.0.0", author = "Julien Blatecky")] +#[clap(version = "2.0.0", author = "Julien Blatecky")] struct Opts { /// Path to the temp folder, you can set in environment variable as TEMP_FOLDER #[clap(long = "temp-folder")] @@ -22,8 +22,8 @@ struct Opts { #[clap(long = "file-folder")] file_folder: Option, - /// Path to the file storage folder for the database, you can set in environment variable as DB_FOLDER - /// Could be the same as FILE_FOLDER if this app and the database are on the same file system + /// Path to the file storage folder for the database, you can set in environment variable as DB_FOLDER. + /// Could be the same as FILE_FOLDER if this app and the database are on the same file system. /// Files copied by this app inside FILE_FOLDER must be visible by the database in DB_FOLDER #[clap(long = "db-folder")] db_folder: Option, @@ -43,7 +43,7 @@ enum MainCommand { Serve(ServeFlags), } -pub fn run(builders: ConnectorsBuilders) { +pub async fn run(builders: ConnectorsBuilders) { let opts = Opts::parse(); let temp_folder = opts @@ -65,7 +65,9 @@ pub fn run(builders: ConnectorsBuilders) { }; match opts.main_command { - MainCommand::Update(update_flags) => update::run(update_flags, folder_options, builders), - MainCommand::Serve(serve_flags) => serve::run(serve_flags, folder_options, builders), + MainCommand::Update(update_flags) => { + update::run(update_flags, folder_options, builders).await + } + MainCommand::Serve(serve_flags) => serve::run(serve_flags, folder_options, builders).await, } } diff --git a/src/commands/serve/mod.rs b/src/commands/serve/mod.rs index 8bf1034..f33170b 100644 --- a/src/commands/serve/mod.rs +++ b/src/commands/serve/mod.rs @@ -2,45 +2,38 @@ mod runner; use super::common::FolderOptions; use crate::connectors::ConnectorsBuilders; -use rocket::config::{Config, Environment}; +use runner::common::Context; use std::env; +use std::net::ToSocketAddrs; #[derive(Clap, Debug)] pub struct ServeFlags { - /// Production, Staging or Development, will change log level, you can set in environment variable as SIRENE_ENV - #[clap(long = "env")] + /// Configure log level, you can set in environment variable as SIRENE_ENV + #[clap(arg_enum, long = "env")] environment: Option, /// Listen this port, you can set in environment variable as PORT - #[clap(short = "p", long = "port")] + #[clap(short = 'p', long = "port")] port: Option, /// Listen this host, you can set in environment variable as HOST - #[clap(short = "h", long = "host")] + #[clap(short = 'h', long = "host")] host: Option, /// API key needed to allow maintenance operation from HTTP, you can set in environment variable as API_KEY - #[clap(short = "k", long = "api-key")] + #[clap(short = 'k', long = "api-key")] api_key: Option, -} -arg_enum! { - #[derive(Debug)] - enum CmdEnvironment { - Development, - Staging, - Production - } + /// Base URL needed to configure asynchronous polling for updates, you can set in environment variable as BASE_URL + #[clap(short = 'b', long = "base-url")] + base_url: Option, } -impl From for Environment { - fn from(env: CmdEnvironment) -> Self { - match env { - CmdEnvironment::Development => Environment::Development, - CmdEnvironment::Staging => Environment::Staging, - CmdEnvironment::Production => Environment::Production, - } - } +#[derive(Clap, Debug)] +enum CmdEnvironment { + Development, + Staging, + Production, } impl CmdEnvironment { @@ -54,7 +47,7 @@ impl CmdEnvironment { } } -pub fn run(flags: ServeFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { +pub async fn run(flags: ServeFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { let env = flags.environment.unwrap_or_else(|| { CmdEnvironment::from_str(env::var("SIRENE_ENV").expect("Missing SIRENE_ENV")) .expect("Invalid SIRENE_ENV") @@ -71,15 +64,32 @@ pub fn run(flags: ServeFlags, folder_options: FolderOptions, builders: Connector .host .unwrap_or_else(|| env::var("HOST").expect("Missing HOST")); + let addr = format!("{}:{}", host, port) + .to_socket_addrs() + .expect("Unable to resolve domain") + .next() + .expect("No address available"); + let api_key = match flags.api_key { Some(key) => Some(key), None => env::var("API_KEY").ok(), }; - let config = Config::build(env.into()) - .address(host) - .port(port) - .finalize(); + let base_url = match flags.base_url { + Some(key) => Some(key), + None => env::var("BASE_URL").ok(), + }; + + log::info!("[Warp] Configuring for {:#?}", env); - runner::run(config.unwrap(), api_key, folder_options, builders) + runner::run( + addr, + Context { + builders, + api_key, + folder_options, + base_url, + }, + ) + .await; } diff --git a/src/commands/serve/runner/common.rs b/src/commands/serve/runner/common.rs index c0eace1..225c2f5 100644 --- a/src/commands/serve/runner/common.rs +++ b/src/commands/serve/runner/common.rs @@ -1,14 +1,16 @@ use super::super::super::common::FolderOptions; -use crate::connectors::Connectors; +use crate::connectors::ConnectorsBuilders; use crate::models::etablissement::common::Etablissement; use crate::models::unite_legale::common::UniteLegale; -use crate::models::update_metadata::common::{SyntheticGroupType, UpdateSummary}; +use crate::models::update_metadata::common::SyntheticGroupType; use serde::{Deserialize, Serialize}; +#[derive(Clone)] pub struct Context { - pub connectors: Connectors, + pub builders: ConnectorsBuilders, pub api_key: Option, pub folder_options: FolderOptions, + pub base_url: Option, } #[derive(Deserialize)] @@ -16,12 +18,12 @@ pub struct UpdateOptions { pub api_key: String, pub group_type: SyntheticGroupType, pub force: bool, - pub data_only: bool, + pub asynchronous: bool, } -#[derive(Serialize)] -pub struct UpdateResponse { - pub summary: UpdateSummary, +#[derive(Deserialize)] +pub struct StatusQueryString { + pub api_key: String, } #[derive(Serialize)] diff --git a/src/commands/serve/runner/error.rs b/src/commands/serve/runner/error.rs index bc63169..62586b5 100644 --- a/src/commands/serve/runner/error.rs +++ b/src/commands/serve/runner/error.rs @@ -1,59 +1,129 @@ -use crate::models::{etablissement, unite_legale}; +use crate::connectors::Error as ConnectorError; +use crate::models::{etablissement, unite_legale, update_metadata}; use crate::update::error::Error as InternalUpdateError; use custom_error::custom_error; -use rocket::http::Status; -use rocket::request::Request; -use rocket::response::{self, content, Responder, Response}; use serde::Serialize; +use std::convert::Infallible; +use warp::{http::StatusCode, Rejection, Reply}; custom_error! { pub Error InvalidData = "Invalid data", MissingApiKeyError = "[Admin] Missing API key in configuration", ApiKeyError = "[Admin] Wrong API key", + MissingBaseUrlForAsyncError = "[Admin] No BASE_URL configured, needed for asynchronous updates", + LocalConnectionFailed{source: r2d2::Error} = "Unable to connect to local database ({source}).", + UpdateConnectorError {source: ConnectorError} = "[Update] Error while creating connector: {source}", UpdateError {source: InternalUpdateError} = "[Update] {source}", UniteLegaleError {source: unite_legale::error::Error} = "[UniteLegale] {source}", EtablissementError {source: etablissement::error::Error} = "[Etablissement] {source}", + StatusError {source: update_metadata::error::Error} = "[Status] {source}", +} + +impl warp::reject::Reject for Error {} + +impl From for Rejection { + fn from(e: Error) -> Self { + warp::reject::custom(e) + } +} + +impl From for Rejection { + fn from(e: ConnectorError) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: InternalUpdateError) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: unite_legale::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: etablissement::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } +} + +impl From for Rejection { + fn from(e: update_metadata::error::Error) -> Self { + let error: Error = e.into(); + error.into() + } } #[derive(Debug, Serialize)] struct ErrorResponse { + code: u16, message: String, } -impl<'r> Responder<'r> for Error { - fn respond_to(self, req: &Request) -> response::Result<'r> { - // Log error - println!("{}", self); +pub async fn handle_rejection(err: Rejection) -> Result { + let (code, message) = if err.is_not_found() { + (StatusCode::NOT_FOUND, String::from("Not found")) + } else if let Some(e) = err.find::() { + log::debug!("[Warp][Error] {:?}", e); - let status = match &self { - Error::InvalidData => Status::BadRequest, - Error::MissingApiKeyError => return Err(Status::Unauthorized), - Error::ApiKeyError => return Err(Status::Unauthorized), - Error::UpdateError { source: _ } => Status::InternalServerError, - Error::UniteLegaleError { source } => match source { - unite_legale::error::Error::UniteLegaleNotFound => Status::NotFound, - _ => Status::InternalServerError, + ( + match e { + Error::InvalidData => StatusCode::BAD_REQUEST, + Error::MissingApiKeyError => StatusCode::UNAUTHORIZED, + Error::ApiKeyError => StatusCode::UNAUTHORIZED, + Error::MissingBaseUrlForAsyncError => StatusCode::BAD_REQUEST, + Error::LocalConnectionFailed { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, + Error::UpdateConnectorError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, + Error::UpdateError { source: _ } => StatusCode::INTERNAL_SERVER_ERROR, + Error::UniteLegaleError { source } => match source { + unite_legale::error::Error::UniteLegaleNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, + Error::EtablissementError { source } => match source { + etablissement::error::Error::EtablissementNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, + Error::StatusError { source } => match source { + update_metadata::error::Error::MetadataNotFound => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }, }, - Error::EtablissementError { source } => match source { - etablissement::error::Error::EtablissementNotFound => Status::NotFound, - _ => Status::InternalServerError, - }, - }; - - let error_response = ErrorResponse { - message: self.to_string(), - }; - - let json_result = serde_json::to_string(&error_response) - .map(|string| content::Json(string).respond_to(req).unwrap()) - .map_err(|e| { - eprintln!("JSON failed to serialize: {:?}", e); - Status::InternalServerError - }); - - match json_result { - Ok(json) => Response::build_from(json).status(status).ok(), - Err(status) => Err(status), - } + e.to_string(), + ) + } else if let Some(body_error) = err.find::() { + log::debug!("[Warp][Json] {}", body_error); + + (StatusCode::BAD_REQUEST, body_error.to_string()) + } else if let Some(e) = err.find::() { + log::debug!("[Warp][Method] {}", e); + + (StatusCode::NOT_FOUND, String::from("Not found")) + } else { + log::debug!("[Warp][Rejection] Unhandled error {:?}", err); + + ( + StatusCode::INTERNAL_SERVER_ERROR, + String::from("Internal server error"), + ) + }; + + if code == StatusCode::INTERNAL_SERVER_ERROR { + log::error!("[Warp][InternalServerError] {}", message); } + + Ok(warp::reply::with_status( + warp::reply::json(&ErrorResponse { + code: code.as_u16(), + message, + }), + code, + )) } diff --git a/src/commands/serve/runner/mod.rs b/src/commands/serve/runner/mod.rs index ba34358..49d2de1 100644 --- a/src/commands/serve/runner/mod.rs +++ b/src/commands/serve/runner/mod.rs @@ -1,69 +1,149 @@ -mod common; mod error; -use super::super::common::FolderOptions; -use crate::connectors::ConnectorsBuilders; +pub mod common; + use crate::models; +use crate::models::update_metadata::common::UpdateMetadata; use crate::update::{common::Config as DataConfig, update as update_data}; +use chrono::Utc; use common::{ - Context, EtablissementInnerResponse, EtablissementResponse, + Context, EtablissementInnerResponse, EtablissementResponse, StatusQueryString, UniteLegaleEtablissementInnerResponse, UniteLegaleInnerResponse, UniteLegaleResponse, - UpdateOptions, UpdateResponse, + UpdateOptions, }; use error::Error; -use rocket::config::Config; -use rocket::State; -use rocket_contrib::json::Json; +use std::convert::Infallible; +use std::net::SocketAddr; +use warp::{ + http::{header, Method, StatusCode}, + Filter, Rejection, Reply, +}; -#[get("/")] fn index() -> &'static str { "SIRENE API v3" } -#[post("/update", format = "application/json", data = "")] -fn update( - state: State, - options: Json, -) -> Result, Error> { - let api_key = match &state.api_key { +async fn update(options: UpdateOptions, context: Context) -> Result { + let api_key = match &context.api_key { Some(key) => key, - None => return Err(Error::MissingApiKeyError), + None => return Err(Error::MissingApiKeyError.into()), }; if &options.api_key != api_key { - return Err(Error::ApiKeyError); + return Err(Error::ApiKeyError.into()); + } + + if options.asynchronous && context.base_url == None { + return Err(Error::MissingBaseUrlForAsyncError.into()); } - let summary = update_data( + let mut connectors = context.builders.create_with_insee().await?; + + let update_metadata = update_data( options.group_type, DataConfig { force: options.force, - data_only: options.data_only, - temp_folder: state.folder_options.temp.clone(), - file_folder: state.folder_options.file.clone(), - db_folder: state.folder_options.db.clone(), + data_only: false, + temp_folder: context.folder_options.temp.clone(), + file_folder: context.folder_options.file.clone(), + db_folder: context.folder_options.db.clone(), + asynchronous: options.asynchronous, }, - &state.connectors, + &mut connectors, + ) + .await?; + + reply_with_update_metadata(&update_metadata, context.base_url, api_key) +} + +async fn status(query: StatusQueryString, context: Context) -> Result { + let api_key = match &context.api_key { + Some(key) => key, + None => return Err(Error::MissingApiKeyError.into()), + }; + + if &query.api_key != api_key { + return Err(Error::ApiKeyError.into()); + } + + let connectors = context.builders.create(); + + let update_metadata = models::update_metadata::current_update(&connectors)?; + + reply_with_update_metadata(&update_metadata, context.base_url, api_key) +} + +fn reply_with_update_metadata( + update_metadata: &UpdateMetadata, + base_url: Option, + api_key: &String, +) -> Result { + let status_code = match update_metadata.status.as_str() { + "launched" => StatusCode::ACCEPTED, + "error" => StatusCode::INTERNAL_SERVER_ERROR, + _ => StatusCode::OK, + }; + + Ok(warp::reply::with_status( + warp::reply::with_header( + warp::reply::with_header( + warp::reply::json(&update_metadata), + "Location", + format!( + "{}/admin/update/status?api_key={}", + base_url.unwrap_or_default(), + api_key + ), + ), + "Retry-After", + "10", + ), + status_code, + )) +} + +async fn set_status_to_error( + query: StatusQueryString, + context: Context, +) -> Result { + let api_key = match &context.api_key { + Some(key) => key, + None => return Err(Error::MissingApiKeyError.into()), + }; + + if &query.api_key != api_key { + return Err(Error::ApiKeyError.into()); + } + + let connectors = context.builders.create(); + + models::update_metadata::error_update( + &connectors, + String::from("Process stopped manually."), + Utc::now(), )?; - Ok(Json(UpdateResponse { summary })) + Ok(warp::reply()) } -#[get("/unites_legales/")] -fn unites_legales( - state: State, - siren: String, -) -> Result, Error> { +async fn unites_legales(siren: String, context: Context) -> Result { if siren.len() != 9 { - return Err(Error::InvalidData); + return Err(Error::InvalidData.into()); } - let unite_legale = models::unite_legale::get(&state.connectors, &siren)?; - let etablissements = models::etablissement::get_with_siren(&state.connectors, &siren)?; + let connectors = context.builders.create(); + let connection = connectors + .local + .pool + .get() + .map_err(|e| Error::LocalConnectionFailed { source: e })?; + + let unite_legale = models::unite_legale::get(&connection, &siren)?; + let etablissements = models::etablissement::get_with_siren(&connection, &siren)?; let etablissement_siege = - models::etablissement::get_siege_with_siren(&state.connectors, &unite_legale.siren)?; + models::etablissement::get_siege_with_siren(&connection, &unite_legale.siren)?; - Ok(Json(UniteLegaleResponse { + Ok(warp::reply::json(&UniteLegaleResponse { unite_legale: UniteLegaleInnerResponse { unite_legale, etablissements, @@ -72,21 +152,24 @@ fn unites_legales( })) } -#[get("/etablissements/")] -fn etablissements( - state: State, - siret: String, -) -> Result, Error> { +async fn etablissements(siret: String, context: Context) -> Result { if siret.len() != 14 { - return Err(Error::InvalidData); + return Err(Error::InvalidData.into()); } - let etablissement = models::etablissement::get(&state.connectors, &siret)?; - let unite_legale = models::unite_legale::get(&state.connectors, &etablissement.siren)?; + let connectors = context.builders.create(); + let connection = connectors + .local + .pool + .get() + .map_err(|e| Error::LocalConnectionFailed { source: e })?; + + let etablissement = models::etablissement::get(&connection, &siret)?; + let unite_legale = models::unite_legale::get(&connection, &etablissement.siren)?; let etablissement_siege = - models::etablissement::get_siege_with_siren(&state.connectors, &etablissement.siren)?; + models::etablissement::get_siege_with_siren(&connection, &etablissement.siren)?; - Ok(Json(EtablissementResponse { + Ok(warp::reply::json(&EtablissementResponse { etablissement: EtablissementInnerResponse { etablissement, unite_legale: UniteLegaleEtablissementInnerResponse { @@ -97,19 +180,80 @@ fn etablissements( })) } -pub fn run( - config: Config, - api_key: Option, - folder_options: FolderOptions, - builders: ConnectorsBuilders, -) { - rocket::custom(config) - .mount("/v3", routes![index, unites_legales, etablissements]) - .mount("/admin", routes![update]) - .manage(Context { - connectors: builders.create(), - api_key, - folder_options, - }) - .launch(); +pub async fn run(addr: SocketAddr, context: Context) { + // GET / -> OK + let health_route = warp::get() + .and(warp::path::end()) + .map(|| warp::reply::with_status("OK", warp::http::StatusCode::OK)); + log::info!("[Warp] Mount GET /"); + + let v3_route = warp::path!("v3" / ..); + + // GET /v3 -> "SIRENE API v3" + let v3_index = warp::path::end().map(index); + log::info!("[Warp] Mount GET /v3"); + + // GET /unites_legales/ + let v3_unites_legales_route = warp::get() + .and(warp::path!("unites_legales" / String)) + .and(with_context(context.clone())) + .and_then(unites_legales); + log::info!("[Warp] Mount GET /v3/unites_legales/"); + + // GET /etablissements/ + let v3_etablissement_route = warp::get() + .and(warp::path!("etablissements" / String)) + .and(with_context(context.clone())) + .and_then(etablissements); + log::info!("[Warp] Mount GET /v3/etablissements/"); + + let admin_update_route = warp::path!("admin" / "update" / ..); + + // POST /admin/update {json} + let update_route = warp::post() + .and(warp::path::end()) + .and(warp::body::content_length_limit(1024 * 32)) + .and(warp::body::json::()) + .and(with_context(context.clone())) + .and_then(update); + log::info!("[Warp] Mount POST /admin/update {{json}}"); + + // GET /admin/update/status?api_key="" + let status_route = warp::get() + .and(warp::path!("status")) + .and(warp::query::()) + .and(with_context(context.clone())) + .and_then(status); + log::info!("[Warp] Mount GET /admin/update/status?api_key="); + + // POST /admin/update/status/error { api_key } + let status_error_route = warp::post() + .and(warp::path!("status" / "error")) + .and(warp::body::content_length_limit(1024 * 32)) + .and(warp::body::json::()) + .and(with_context(context)) + .and_then(set_status_to_error); + log::info!("[Warp] Mount POST /admin/update/status/error {{api_key}}"); + + // Cors + let cors = warp::cors() + .allow_methods(&[Method::GET, Method::POST]) + .allow_headers(vec![header::CONTENT_TYPE]) + .allow_any_origin(); + + let routes = health_route + .or(v3_route.and( + v3_unites_legales_route + .or(v3_etablissement_route) + .or(v3_index), + )) + .or(admin_update_route.and(status_route.or(update_route).or(status_error_route))) + .recover(error::handle_rejection) + .with(cors); + + warp::serve(routes).run(addr).await; +} + +fn with_context(context: Context) -> impl Filter + Clone { + warp::any().map(move || context.clone()) } diff --git a/src/commands/update.rs b/src/commands/update.rs index 71c6bca..bcda428 100644 --- a/src/commands/update.rs +++ b/src/commands/update.rs @@ -1,11 +1,14 @@ use super::common::{CmdGroupType, FolderOptions}; use crate::connectors::ConnectorsBuilders; use crate::models::update_metadata::common::{Step, SyntheticGroupType}; -use crate::update::{common::Config, update, update_step}; +use crate::models::update_metadata::error_update; +use crate::update::{common::Config, error::Error, update, update_step}; +use chrono::Utc; #[derive(Clap, Debug)] pub struct UpdateFlags { - /// Configure which part will be updated, UnitesLegales, Etablissements or All + /// Configure which part will be updated + #[clap(arg_enum)] group_type: CmdGroupType, /// Force update even if the source data where not updated @@ -41,10 +44,21 @@ enum UpdateSubCommand { /// Clean files from FILE_FOLDER #[clap(name = "clean-file")] CleanFile, + + /// Synchronise daily data from INSEE since the last modification + #[clap(name = "sync-insee")] + SyncInsee, + + /// Set a staled update process to error, use only if the process is really stopped + #[clap(name = "finish-error")] + FinishError, } -pub fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { - let connectors = builders.create(); +pub async fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: ConnectorsBuilders) { + let mut connectors = builders + .create_with_insee() + .await + .expect("Unable to create INSEE connector"); let synthetic_group_type: SyntheticGroupType = flags.group_type.into(); // Prepare config @@ -54,6 +68,7 @@ pub fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: Connecto temp_folder: folder_options.temp, file_folder: folder_options.file, db_folder: folder_options.db, + asynchronous: false, }; let summary_result = match flags.subcmd { @@ -64,15 +79,31 @@ pub fn run(flags: UpdateFlags, folder_options: FolderOptions, builders: Connecto UpdateSubCommand::InsertData => Step::InsertData, UpdateSubCommand::SwapData => Step::SwapData, UpdateSubCommand::CleanFile => Step::CleanFile, + UpdateSubCommand::SyncInsee => Step::SyncInsee, + UpdateSubCommand::FinishError => { + if let Err(err) = error_update( + &connectors, + "Process stopped manually.".to_string(), + Utc::now(), + ) { + let error: Error = err.into(); + error.exit() + } + + std::process::exit(0); + } }; - update_step(step, synthetic_group_type, config, &connectors) + update_step(step, synthetic_group_type, config, &mut connectors).await } - None => update(synthetic_group_type, config, &connectors), + None => update(synthetic_group_type, config, &mut connectors).await, }; match summary_result { - Ok(summary) => println!("{}", serde_json::to_string(&summary).unwrap()), + Ok(summary) => println!( + "{}", + serde_json::to_string_pretty(&summary).expect("Unable to stringify summary") + ), Err(error) => error.exit(), } } diff --git a/src/connectors/insee/error.rs b/src/connectors/insee/error.rs new file mode 100644 index 0000000..628c016 --- /dev/null +++ b/src/connectors/insee/error.rs @@ -0,0 +1,12 @@ +use custom_error::custom_error; + +custom_error! { pub InseeTokenError + NetworkError { source: reqwest::Error } = "Unable to retrieve INSEE token (network error: {source})", + ApiError = "Unable to retrieve INSEE token", + MalformedError {source: serde_json::Error} = "Unable to read INSEE token ({source})", + InvalidError {source: reqwest::header::InvalidHeaderValue} = "Unable to use INSEE token in header ({source})", +} + +custom_error! { pub InseeUpdateError + NetworkError {source: reqwest::Error} = "Unable to retrieve INSEE data (network error: {source})", +} diff --git a/src/connectors/insee/implementation.rs b/src/connectors/insee/implementation.rs new file mode 100644 index 0000000..43f3c87 --- /dev/null +++ b/src/connectors/insee/implementation.rs @@ -0,0 +1,194 @@ +use super::error::InseeUpdateError; +use super::types::{ + etablissement::InseeEtablissementResponse, unite_legale::InseeUniteLegaleResponse, + InseeCountQueryParams, InseeCountResponse, InseeQueryParams, InseeResponse, +}; +use super::Connector; +use crate::models::etablissement::common::Etablissement; +use crate::models::unite_legale::common::UniteLegale; +use chrono::NaiveDateTime; + +const MAX_CALL: u8 = 20; +const MAX_DURATION: std::time::Duration = std::time::Duration::from_secs(60); +const BASE_URL: &str = "https://api.insee.fr/entreprises/sirene/V3"; +pub const INITIAL_CURSOR: &str = "*"; + +struct EndpointConfig { + route: &'static str, + query_field: &'static str, +} + +const UNITES_LEGALES_ENDPOINT: EndpointConfig = EndpointConfig { + route: "siren", + query_field: "dateDernierTraitementUniteLegale", +}; + +const ETABLISSEMENTS_ENDPOINT: EndpointConfig = EndpointConfig { + route: "siret", + query_field: "dateDernierTraitementEtablissement", +}; + +impl Connector { + async fn wait_for_insee_limitation(&mut self) { + if self.calls == 0 { + self.started_at = std::time::Instant::now(); + } + + self.calls += 1; + + let elapsed = self.started_at.elapsed(); + if self.calls > MAX_CALL && elapsed < MAX_DURATION { + tokio::time::delay_for(MAX_DURATION).await; + self.calls = 0; + } else if elapsed >= MAX_DURATION { + self.calls = 0; + } + } + + pub async fn get_total_unites_legales( + &mut self, + start_timestamp: NaiveDateTime, + ) -> Result { + self.wait_for_insee_limitation().await; + + get_total(&self.client, &UNITES_LEGALES_ENDPOINT, start_timestamp).await + } + + pub async fn get_total_etablissements( + &mut self, + start_timestamp: NaiveDateTime, + ) -> Result { + self.wait_for_insee_limitation().await; + + get_total(&self.client, &ETABLISSEMENTS_ENDPOINT, start_timestamp).await + } + + pub async fn get_daily_unites_legales( + &mut self, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, Vec), InseeUpdateError> { + self.wait_for_insee_limitation().await; + + let (next_cursor, response) = get_daily_data::( + &self.client, + &UNITES_LEGALES_ENDPOINT, + start_timestamp, + cursor, + ) + .await?; + + Ok(( + next_cursor, + match response { + Some(resp) => resp + .unites_legales + .iter() + .filter_map(|u| u.into()) + .collect(), + None => vec![], + }, + )) + } + + pub async fn get_daily_etablissements( + &mut self, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, Vec), InseeUpdateError> { + self.wait_for_insee_limitation().await; + + let (next_cursor, response) = get_daily_data::( + &self.client, + &ETABLISSEMENTS_ENDPOINT, + start_timestamp, + cursor, + ) + .await?; + + Ok(( + next_cursor, + match response { + Some(resp) => resp + .etablissements + .iter() + .filter_map(|u| u.into()) + .collect(), + None => vec![], + }, + )) + } +} + +async fn get_daily_data( + client: &reqwest::Client, + config: &EndpointConfig, + start_timestamp: NaiveDateTime, + cursor: String, +) -> Result<(Option, Option), InseeUpdateError> { + let url = format!("{}/{}", BASE_URL, config.route); + + let response = match client + .get(&url) + .query(&InseeQueryParams { + q: format!( + "{}:[{} TO *]", + config.query_field, + start_timestamp.format("%Y-%m-%dT%H:%M:%S") + ), + nombre: 1000, + curseur: cursor, + tri: format!("{} asc", config.query_field), + }) + .send() + .await? + .error_for_status() + { + Ok(response) => response.json::().await.map_err(|error| error.into()), + Err(error) => { + // Insee returns 404 for empty data + if let Some(status) = error.status() { + if status == reqwest::StatusCode::NOT_FOUND { + return Ok((None, None)); + } + } + + Err(error) + } + }?; + + let header = response.header(); + let next_cursor = if header.curseur == header.curseur_suivant { + None + } else { + Some(header.curseur_suivant) + }; + + Ok((next_cursor, Some(response))) +} + +async fn get_total( + client: &reqwest::Client, + config: &EndpointConfig, + start_timestamp: NaiveDateTime, +) -> Result { + let url = format!("{}/{}", BASE_URL, config.route); + + let response = client + .get(&url) + .query(&InseeCountQueryParams { + q: format!( + "{}:[{} TO *]", + config.query_field, + start_timestamp.format("%Y-%m-%dT%H:%M:%S") + ), + nombre: 1, + champs: config.route, + }) + .send() + .await? + .json::() + .await?; + + Ok(response.header.total) +} diff --git a/src/connectors/insee/mod.rs b/src/connectors/insee/mod.rs new file mode 100644 index 0000000..945c2b3 --- /dev/null +++ b/src/connectors/insee/mod.rs @@ -0,0 +1,95 @@ +mod implementation; +mod types; + +pub mod error; + +use error::InseeTokenError; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION}; +use serde::{Deserialize, Serialize}; +use std::env; +use std::time::{Duration, Instant}; + +pub use implementation::INITIAL_CURSOR; + +#[derive(Clone)] +pub struct Connector { + client: reqwest::Client, + calls: u8, + started_at: Instant, +} + +#[derive(Clone)] +pub struct ConnectorBuilder { + pub credentials: String, +} + +#[derive(Serialize)] +struct InseeTokenParameters { + pub grant_type: String, + pub validity_period: u32, +} + +#[derive(Deserialize)] +struct InseeTokenResponse { + pub access_token: String, + pub scope: String, + pub token_type: String, + pub expires_in: i64, +} + +impl ConnectorBuilder { + pub fn new() -> Option { + env::var("INSEE_CREDENTIALS") + .ok() + .and_then(|credentials| match credentials.len() { + 0 => None, + _ => Some(ConnectorBuilder { credentials }), + }) + } + + pub async fn create(&self) -> Result { + self.generate_token().await.and_then(|token| { + // Build headers + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(format!("Bearer {}", token).as_str())?, + ); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + + // Build client + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .default_headers(headers) + .build()?; + + Ok(Connector { + client, + calls: 0, + started_at: Instant::now(), + }) + }) + } + + async fn generate_token(&self) -> Result { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(20)) + .build()?; + + let response = client + .post("https://api.insee.fr/token") + .header(AUTHORIZATION, format!("Basic {}", self.credentials)) + .form(&InseeTokenParameters { + grant_type: String::from("client_credentials"), + validity_period: 86400, + }) + .send() + .await? + .json::() + .await?; + + Ok(response.access_token) + } +} diff --git a/src/connectors/insee/types/etablissement.rs b/src/connectors/insee/types/etablissement.rs new file mode 100644 index 0000000..8633e70 --- /dev/null +++ b/src/connectors/insee/types/etablissement.rs @@ -0,0 +1,201 @@ +use super::{Header, InseeResponse}; +use crate::models::etablissement::common::Etablissement; +use chrono::{NaiveDate, NaiveDateTime}; +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissementResponse { + pub header: Header, + pub etablissements: Vec, +} + +impl InseeResponse for InseeEtablissementResponse { + fn header(&self) -> Header { + self.header.clone() + } +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissementInner { + pub siren: String, + pub nic: String, + pub siret: String, + pub statut_diffusion_etablissement: String, + pub date_creation_etablissement: Option, + pub tranche_effectifs_etablissement: Option, + #[serde(deserialize_with = "super::from_str_optional")] + pub annee_effectifs_etablissement: Option, + pub activite_principale_registre_metiers_etablissement: Option, + pub date_dernier_traitement_etablissement: Option, + pub etablissement_siege: bool, + pub nombre_periodes_etablissement: Option, + pub adresse_etablissement: InseeAdresseEtablissement, + pub adresse2_etablissement: InseeAdresse2Etablissement, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeEtablissement { + #[serde(flatten)] + pub content: InseeEtablissementInner, + pub periodes_etablissement: Vec, +} + +#[derive(Debug)] +pub struct InseeEtablissementWithPeriode { + pub content: InseeEtablissementInner, + pub periode: InseePeriodeEtablissement, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeAdresseEtablissement { + pub complement_adresse_etablissement: Option, + pub numero_voie_etablissement: Option, + pub indice_repetition_etablissement: Option, + pub type_voie_etablissement: Option, + pub libelle_voie_etablissement: Option, + pub code_postal_etablissement: Option, + pub libelle_commune_etablissement: Option, + pub libelle_commune_etranger_etablissement: Option, + pub distribution_speciale_etablissement: Option, + pub code_commune_etablissement: Option, + pub code_cedex_etablissement: Option, + pub libelle_cedex_etablissement: Option, + pub code_pays_etranger_etablissement: Option, + pub libelle_pays_etranger_etablissement: Option, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeAdresse2Etablissement { + complement_adresse2_etablissement: Option, + numero_voie2_etablissement: Option, + indice_repetition2_etablissement: Option, + type_voie2_etablissement: Option, + libelle_voie2_etablissement: Option, + code_postal2_etablissement: Option, + libelle_commune2_etablissement: Option, + libelle_commune_etranger2_etablissement: Option, + distribution_speciale2_etablissement: Option, + code_commune2_etablissement: Option, + code_cedex2_etablissement: Option, + libelle_cedex2_etablissement: Option, + code_pays_etranger2_etablissement: Option, + libelle_pays_etranger2_etablissement: Option, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseePeriodeEtablissement { + date_fin: Option, + date_debut: Option, + #[serde(deserialize_with = "deserialize_etat_administratif")] + etat_administratif_etablissement: String, + changement_etat_administratif_etablissement: bool, + enseigne1_etablissement: Option, + enseigne2_etablissement: Option, + enseigne3_etablissement: Option, + changement_enseigne_etablissement: bool, + denomination_usuelle_etablissement: Option, + changement_denomination_usuelle_etablissement: bool, + activite_principale_etablissement: Option, + nomenclature_activite_principale_etablissement: Option, + changement_activite_principale_etablissement: bool, + caractere_employeur_etablissement: Option, + changement_caractere_employeur_etablissement: bool, +} + +impl From<&InseeEtablissement> for Option { + fn from(u: &InseeEtablissement) -> Self { + match u + .periodes_etablissement + .iter() + .find(|p| p.date_fin.is_none()) + { + Some(periode) => { + // Convert + Some( + InseeEtablissementWithPeriode { + content: u.content.clone(), + periode: periode.clone(), + } + .into(), + ) + } + None => None, + } + } +} + +impl From for Etablissement { + fn from(e: InseeEtablissementWithPeriode) -> Self { + let adresse = e.content.adresse_etablissement; + let adresse2 = e.content.adresse2_etablissement; + + Etablissement { + siret: e.content.siret, + siren: e.content.siren, + nic: e.content.nic, + statut_diffusion: e.content.statut_diffusion_etablissement, + date_creation: e.content.date_creation_etablissement, + tranche_effectifs: e.content.tranche_effectifs_etablissement, + annee_effectifs: e.content.annee_effectifs_etablissement, + activite_principale_registre_metiers: e + .content + .activite_principale_registre_metiers_etablissement, + date_dernier_traitement: e.content.date_dernier_traitement_etablissement, + etablissement_siege: e.content.etablissement_siege, + nombre_periodes: e.content.nombre_periodes_etablissement, + complement_adresse: adresse.complement_adresse_etablissement, + numero_voie: adresse.numero_voie_etablissement, + indice_repetition: adresse.indice_repetition_etablissement, + type_voie: adresse.type_voie_etablissement, + libelle_voie: adresse.libelle_voie_etablissement, + code_postal: adresse.code_postal_etablissement, + libelle_commune: adresse.libelle_commune_etablissement, + libelle_commune_etranger: adresse.libelle_commune_etranger_etablissement, + distribution_speciale: adresse.distribution_speciale_etablissement, + code_commune: adresse.code_commune_etablissement, + code_cedex: adresse.code_cedex_etablissement, + libelle_cedex: adresse.libelle_cedex_etablissement, + code_pays_etranger: adresse.code_pays_etranger_etablissement, + libelle_pays_etranger: adresse.libelle_pays_etranger_etablissement, + complement_adresse2: adresse2.complement_adresse2_etablissement, + numero_voie_2: adresse2.numero_voie2_etablissement, + indice_repetition_2: adresse2.indice_repetition2_etablissement, + type_voie_2: adresse2.type_voie2_etablissement, + libelle_voie_2: adresse2.libelle_voie2_etablissement, + code_postal_2: adresse2.code_postal2_etablissement, + libelle_commune_2: adresse2.libelle_commune2_etablissement, + libelle_commune_etranger_2: adresse2.libelle_commune_etranger2_etablissement, + distribution_speciale_2: adresse2.distribution_speciale2_etablissement, + code_commune_2: adresse2.code_commune2_etablissement, + code_cedex_2: adresse2.code_cedex2_etablissement, + libelle_cedex_2: adresse2.libelle_cedex2_etablissement, + code_pays_etranger_2: adresse2.code_pays_etranger2_etablissement, + libelle_pays_etranger_2: adresse2.libelle_pays_etranger2_etablissement, + date_debut: e.periode.date_debut, + etat_administratif: e.periode.etat_administratif_etablissement, + enseigne_1: e.periode.enseigne1_etablissement, + enseigne_2: e.periode.enseigne2_etablissement, + enseigne_3: e.periode.enseigne3_etablissement, + denomination_usuelle: e.periode.denomination_usuelle_etablissement, + activite_principale: e.periode.activite_principale_etablissement, + nomenclature_activite_principale: e + .periode + .nomenclature_activite_principale_etablissement, + caractere_employeur: e.periode.caractere_employeur_etablissement, + } + } +} + +fn deserialize_etat_administratif<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let opt = Option::deserialize(deserializer)?; + Ok(opt.unwrap_or(String::from("F"))) +} diff --git a/src/connectors/insee/types/mod.rs b/src/connectors/insee/types/mod.rs new file mode 100644 index 0000000..b91ef2a --- /dev/null +++ b/src/connectors/insee/types/mod.rs @@ -0,0 +1,74 @@ +pub mod etablissement; +pub mod unite_legale; + +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::fmt::Display; +use std::str::FromStr; + +#[derive(Serialize)] +pub struct InseeQueryParams { + pub q: String, + pub nombre: u16, + pub curseur: String, + pub tri: String, +} + +#[derive(Serialize)] +pub struct InseeCountQueryParams { + pub q: String, + pub nombre: u16, + pub champs: &'static str, +} + +pub trait InseeResponse: DeserializeOwned { + fn header(&self) -> Header; +} + +#[derive(Clone, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Header { + total: u32, + debut: u32, + nombre: u32, + pub curseur: String, + pub curseur_suivant: String, +} + +#[derive(Deserialize, Debug)] +pub struct CountHeader { + #[serde(default = "default_as_zero")] + pub total: u32, +} + +#[derive(Deserialize, Debug)] +pub struct InseeCountResponse { + pub header: CountHeader, +} + +fn from_str_optional<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: FromStr, + T::Err: Display, + D: serde::Deserializer<'de>, +{ + let deser_res: Result = serde::Deserialize::deserialize(deserializer); + match deser_res { + Ok(serde_json::Value::String(s)) => T::from_str(&s) + .map_err(serde::de::Error::custom) + .map(Option::from), + Ok(serde_json::Value::Null) => Ok(None), + Ok(v) => { + log::error!("string expected but found something else: {}", v); + Ok(None) + } + Err(_) => Ok(None), + } +} + +fn default_as_false() -> bool { + false +} + +fn default_as_zero() -> u32 { + 0 +} diff --git a/src/connectors/insee/types/unite_legale.rs b/src/connectors/insee/types/unite_legale.rs new file mode 100644 index 0000000..dea7da0 --- /dev/null +++ b/src/connectors/insee/types/unite_legale.rs @@ -0,0 +1,168 @@ +use super::{Header, InseeResponse}; +use crate::models::unite_legale::common::UniteLegale; +use chrono::{NaiveDate, NaiveDateTime}; +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegaleResponse { + pub header: Header, + pub unites_legales: Vec, +} + +impl InseeResponse for InseeUniteLegaleResponse { + fn header(&self) -> Header { + self.header.clone() + } +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegaleInner { + pub siren: String, + pub statut_diffusion_unite_legale: String, + + #[serde(default = "super::default_as_false")] + pub unite_purgee_unite_legale: bool, + + pub date_creation_unite_legale: Option, + pub sigle_unite_legale: Option, + pub sexe_unite_legale: Option, + pub prenom1_unite_legale: Option, + pub prenom2_unite_legale: Option, + pub prenom3_unite_legale: Option, + pub prenom4_unite_legale: Option, + pub prenom_usuel_unite_legale: Option, + pub pseudonyme_unite_legale: Option, + pub identifiant_association_unite_legale: Option, + pub tranche_effectifs_unite_legale: Option, + + #[serde(deserialize_with = "super::from_str_optional")] + pub annee_effectifs_unite_legale: Option, + + pub date_dernier_traitement_unite_legale: Option, + pub nombre_periodes_unite_legale: Option, + pub categorie_entreprise: Option, + + #[serde(deserialize_with = "super::from_str_optional")] + pub annee_categorie_entreprise: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct InseeUniteLegale { + #[serde(flatten)] + pub content: InseeUniteLegaleInner, + pub periodes_unite_legale: Vec, +} + +#[derive(Debug)] +pub struct InseeUniteLegaleWithPeriode { + pub content: InseeUniteLegaleInner, + pub periode: PeriodeInseeUniteLegale, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PeriodeInseeUniteLegale { + pub date_fin: Option, + pub date_debut: Option, + + #[serde(deserialize_with = "deserialize_etat_administratif")] + pub etat_administratif_unite_legale: String, + + pub changement_etat_administratif_unite_legale: bool, + pub nom_unite_legale: Option, + pub changement_nom_unite_legale: bool, + pub nom_usage_unite_legale: Option, + pub changement_nom_usage_unite_legale: bool, + pub denomination_unite_legale: Option, + pub changement_denomination_unite_legale: bool, + pub denomination_usuelle1_unite_legale: Option, + pub denomination_usuelle2_unite_legale: Option, + pub denomination_usuelle3_unite_legale: Option, + pub changement_denomination_usuelle_unite_legale: bool, + pub categorie_juridique_unite_legale: Option, + pub changement_categorie_juridique_unite_legale: bool, + pub activite_principale_unite_legale: Option, + pub nomenclature_activite_principale_unite_legale: Option, + pub changement_activite_principale_unite_legale: bool, + pub nic_siege_unite_legale: Option, + pub changement_nic_siege_unite_legale: bool, + pub economie_sociale_solidaire_unite_legale: Option, + pub changement_economie_sociale_solidaire_unite_legale: bool, + pub caractere_employeur_unite_legale: Option, + pub changement_caractere_employeur_unite_legale: bool, +} + +impl From<&InseeUniteLegale> for Option { + fn from(u: &InseeUniteLegale) -> Self { + match u + .periodes_unite_legale + .iter() + .find(|p| p.date_fin.is_none()) + { + Some(periode) => { + // Convert + Some( + InseeUniteLegaleWithPeriode { + content: u.content.clone(), + periode: periode.clone(), + } + .into(), + ) + } + None => None, + } + } +} + +impl From for UniteLegale { + fn from(u: InseeUniteLegaleWithPeriode) -> Self { + UniteLegale { + siren: u.content.siren, + statut_diffusion: u.content.statut_diffusion_unite_legale, + unite_purgee: Some(u.content.unite_purgee_unite_legale.to_string()), + date_creation: u.content.date_creation_unite_legale, + sigle: u.content.sigle_unite_legale, + sexe: u.content.sexe_unite_legale, + prenom_1: u.content.prenom1_unite_legale, + prenom_2: u.content.prenom2_unite_legale, + prenom_3: u.content.prenom3_unite_legale, + prenom_4: u.content.prenom4_unite_legale, + prenom_usuel: u.content.prenom_usuel_unite_legale, + pseudonyme: u.content.pseudonyme_unite_legale, + identifiant_association: u.content.identifiant_association_unite_legale, + tranche_effectifs: u.content.tranche_effectifs_unite_legale, + annee_effectifs: u.content.annee_effectifs_unite_legale, + date_dernier_traitement: u.content.date_dernier_traitement_unite_legale, + nombre_periodes: u.content.nombre_periodes_unite_legale, + categorie_entreprise: u.content.categorie_entreprise, + annee_categorie_entreprise: u.content.annee_categorie_entreprise, + date_debut: u.periode.date_debut, + etat_administratif: u.periode.etat_administratif_unite_legale, + nom: u.periode.nom_unite_legale, + nom_usage: u.periode.nom_usage_unite_legale, + denomination: u.periode.denomination_unite_legale, + denomination_usuelle_1: u.periode.denomination_usuelle1_unite_legale, + denomination_usuelle_2: u.periode.denomination_usuelle2_unite_legale, + denomination_usuelle_3: u.periode.denomination_usuelle3_unite_legale, + categorie_juridique: u.periode.categorie_juridique_unite_legale, + activite_principale: u.periode.activite_principale_unite_legale, + nomenclature_activite_principale: u + .periode + .nomenclature_activite_principale_unite_legale, + nic_siege: u.periode.nic_siege_unite_legale, + economie_sociale_solidaire: u.periode.economie_sociale_solidaire_unite_legale, + caractere_employeur: u.periode.caractere_employeur_unite_legale, + } + } +} + +fn deserialize_etat_administratif<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let opt = Option::deserialize(deserializer)?; + Ok(opt.unwrap_or(String::from("C"))) +} diff --git a/src/connectors/local.rs b/src/connectors/local.rs index 2ac8b80..202a75a 100644 --- a/src/connectors/local.rs +++ b/src/connectors/local.rs @@ -5,10 +5,14 @@ use std::env; embed_migrations!("./migrations"); +pub type Connection = r2d2::PooledConnection>; + +#[derive(Clone)] pub struct Connector { pub pool: Pool>, } +#[derive(Clone)] pub struct ConnectorBuilder { pool: Pool>, } @@ -16,17 +20,25 @@ pub struct ConnectorBuilder { impl ConnectorBuilder { pub fn new() -> ConnectorBuilder { let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let pool_size = env::var("DATABASE_POOL_SIZE") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(15); let manager = ConnectionManager::::new(database_url.clone()); let builder = ConnectorBuilder { pool: Pool::builder() - .max_size(15) + .max_size(pool_size) .build(manager) .expect(&format!("Error connecting to {}", database_url)), }; - let connection = builder.pool.get().expect("Unable to connect for migrations"); - embedded_migrations::run_with_output(&connection, &mut std::io::stdout()).expect("Unable to run migrations"); + let connection = builder + .pool + .get() + .expect("Unable to connect for migrations"); + embedded_migrations::run_with_output(&connection, &mut std::io::stdout()) + .expect("Unable to run migrations"); builder } diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs index 33320ef..7dbee10 100644 --- a/src/connectors/mod.rs +++ b/src/connectors/mod.rs @@ -1,23 +1,46 @@ +use custom_error::custom_error; + +pub mod insee; pub mod local; +custom_error! { pub Error + InseeError { source: insee::error::InseeTokenError } = "{source}", +} + +#[derive(Clone)] pub struct ConnectorsBuilders { local: local::ConnectorBuilder, + insee: Option, } +#[derive(Clone)] pub struct Connectors { pub local: local::Connector, + pub insee: Option, } impl ConnectorsBuilders { pub fn new() -> Self { ConnectorsBuilders { local: local::ConnectorBuilder::new(), + insee: insee::ConnectorBuilder::new(), } } pub fn create(&self) -> Connectors { Connectors { local: self.local.create(), + insee: None, } } + + pub async fn create_with_insee(&self) -> Result { + Ok(Connectors { + local: self.local.create(), + insee: match self.insee.as_ref() { + Some(insee_builder) => Some(insee_builder.create().await?), + None => None, + }, + }) + } } diff --git a/src/main.rs b/src/main.rs index ce67aff..9878e56 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,8 @@ -#![feature(proc_macro_hygiene, decl_macro)] - extern crate chrono; #[macro_use] extern crate clap; extern crate custom_error; +#[cfg(any(target_os = "unix", target_os = "linux"))] extern crate openssl; // Should be before diesel #[macro_use] extern crate diesel; @@ -12,11 +11,10 @@ extern crate diesel_migrations; extern crate dotenv; extern crate r2d2; extern crate reqwest; -#[macro_use] -extern crate rocket; -extern crate rocket_contrib; extern crate serde; extern crate serde_json; +extern crate tokio; +extern crate warp; extern crate zip; mod commands; @@ -24,26 +22,20 @@ mod connectors; mod models; mod update; -use chrono::Utc; use connectors::ConnectorsBuilders; use dotenv::dotenv; -fn main() { +#[tokio::main] +async fn main() { // Load configuration dotenv().ok(); + // Load Logger + pretty_env_logger::init(); + // Load database let connectors_builders = ConnectorsBuilders::new(); - // Close running updates - let connectors = connectors_builders.create(); - models::update_metadata::error_update( - &connectors, - String::from("Program unexpectedly closed"), - Utc::now(), - ) - .unwrap(); // Fail launch in case of error - // Run command - commands::run(connectors_builders); + commands::run(connectors_builders).await; } diff --git a/src/models/common.rs b/src/models/common.rs index 9abbf42..a619b63 100644 --- a/src/models/common.rs +++ b/src/models/common.rs @@ -1,14 +1,35 @@ +use crate::connectors::insee::error::InseeUpdateError; use crate::connectors::Connectors; +use async_trait::async_trait; +use chrono::NaiveDateTime; use custom_error::custom_error; -pub trait UpdatableModel { +#[async_trait] +pub trait UpdatableModel: Sync + Send { fn count(&self, connectors: &Connectors) -> Result; fn count_staging(&self, connectors: &Connectors) -> Result; fn insert_in_staging(&self, connectors: &Connectors, file_path: String) -> Result; fn swap(&self, connectors: &Connectors) -> Result<(), Error>; + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result; + fn get_last_insee_synced_timestamp( + &self, + connectors: &Connectors, + ) -> Result, Error>; + async fn update_daily_data( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, usize), Error>; } custom_error! { pub Error LocalConnectionFailed{source: r2d2::Error} = "Unable to connect to local database ({source}).", DatabaseError{source: diesel::result::Error} = "Unable to run some operations on updatable model ({source}).", + UpdateError {source: InseeUpdateError} = "{source}", + MissingInseeConnector = "Missing required Insee connector", } diff --git a/src/models/etablissement/common.rs b/src/models/etablissement/common.rs index ce77aff..9033126 100644 --- a/src/models/etablissement/common.rs +++ b/src/models/etablissement/common.rs @@ -1,7 +1,9 @@ +use super::super::schema::etablissement; use chrono::{NaiveDate, NaiveDateTime}; use serde::Serialize; -#[derive(Queryable, Serialize, Clone, Debug)] +#[derive(Insertable, Queryable, Serialize, Clone, Debug)] +#[table_name = "etablissement"] pub struct Etablissement { pub siret: String, pub siren: String, diff --git a/src/models/etablissement/mod.rs b/src/models/etablissement/mod.rs index efa7671..2e3eabb 100644 --- a/src/models/etablissement/mod.rs +++ b/src/models/etablissement/mod.rs @@ -4,44 +4,45 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::etablissement::dsl; -use crate::connectors::Connectors; +use crate::connectors::{local::Connection, Connectors}; +use async_trait::async_trait; +use chrono::NaiveDateTime; use common::Etablissement; +use diesel::pg::upsert::excluded; use diesel::prelude::*; use diesel::sql_query; use error::Error; -pub fn get(connectors: &Connectors, siret: &String) -> Result { - let connection = connectors.local.pool.get()?; +pub fn get(connection: &Connection, siret: &String) -> Result { dsl::etablissement .find(siret) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } pub fn get_with_siren( - connectors: &Connectors, + connection: &Connection, siren: &String, ) -> Result, Error> { - let connection = connectors.local.pool.get()?; dsl::etablissement .filter(dsl::siren.eq(siren)) - .load::(&connection) + .load::(connection) .map_err(|error| error.into()) } pub fn get_siege_with_siren( - connectors: &Connectors, + connection: &Connection, siren: &String, ) -> Result { - let connection = connectors.local.pool.get()?; dsl::etablissement .filter(dsl::siren.eq(siren).and(dsl::etablissement_siege.eq(true))) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } pub struct EtablissementModel {} +#[async_trait] impl UpdatableModel for EtablissementModel { fn count(&self, connectors: &Connectors) -> Result { let connection = connectors.local.pool.get()?; @@ -109,4 +110,108 @@ impl UpdatableModel for EtablissementModel { Ok(()) }) } + + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + Ok(insee.get_total_etablissements(start_timestamp).await?) + } + + // SELECT date_dernier_traitement FROM etablissement WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; + fn get_last_insee_synced_timestamp( + &self, + connectors: &Connectors, + ) -> Result, UpdatableError> { + let connection = connectors.local.pool.get()?; + dsl::etablissement + .select(dsl::date_dernier_traitement) + .order(dsl::date_dernier_traitement.desc()) + .filter(dsl::date_dernier_traitement.is_not_null()) + .first::>(&connection) + .map_err(|error| error.into()) + } + + async fn update_daily_data( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, usize), UpdatableError> { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + let (next_cursor, etablissements) = insee + .get_daily_etablissements(start_timestamp, cursor) + .await?; + + let connection = connectors.local.pool.get()?; + + let updated_count = diesel::insert_into(dsl::etablissement) + .values(&etablissements) + .on_conflict(dsl::siret) + .do_update() + .set(( + dsl::nic.eq(excluded(dsl::nic)), + dsl::siren.eq(excluded(dsl::siren)), + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::activite_principale_registre_metiers + .eq(excluded(dsl::activite_principale_registre_metiers)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::etablissement_siege.eq(excluded(dsl::etablissement_siege)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::complement_adresse.eq(excluded(dsl::complement_adresse)), + dsl::numero_voie.eq(excluded(dsl::numero_voie)), + dsl::indice_repetition.eq(excluded(dsl::indice_repetition)), + dsl::type_voie.eq(excluded(dsl::type_voie)), + dsl::libelle_voie.eq(excluded(dsl::libelle_voie)), + dsl::code_postal.eq(excluded(dsl::code_postal)), + dsl::libelle_commune.eq(excluded(dsl::libelle_commune)), + dsl::libelle_commune_etranger.eq(excluded(dsl::libelle_commune_etranger)), + dsl::distribution_speciale.eq(excluded(dsl::distribution_speciale)), + dsl::code_commune.eq(excluded(dsl::code_commune)), + dsl::code_cedex.eq(excluded(dsl::code_cedex)), + dsl::libelle_cedex.eq(excluded(dsl::libelle_cedex)), + dsl::code_pays_etranger.eq(excluded(dsl::code_pays_etranger)), + dsl::libelle_pays_etranger.eq(excluded(dsl::libelle_pays_etranger)), + dsl::complement_adresse2.eq(excluded(dsl::complement_adresse2)), + dsl::numero_voie_2.eq(excluded(dsl::numero_voie_2)), + dsl::indice_repetition_2.eq(excluded(dsl::indice_repetition_2)), + dsl::type_voie_2.eq(excluded(dsl::type_voie_2)), + dsl::libelle_voie_2.eq(excluded(dsl::libelle_voie_2)), + dsl::code_postal_2.eq(excluded(dsl::code_postal_2)), + dsl::libelle_commune_2.eq(excluded(dsl::libelle_commune_2)), + dsl::libelle_commune_etranger_2.eq(excluded(dsl::libelle_commune_etranger_2)), + dsl::distribution_speciale_2.eq(excluded(dsl::distribution_speciale_2)), + dsl::code_commune_2.eq(excluded(dsl::code_commune_2)), + dsl::code_cedex_2.eq(excluded(dsl::code_cedex_2)), + dsl::libelle_cedex_2.eq(excluded(dsl::libelle_cedex_2)), + dsl::code_pays_etranger_2.eq(excluded(dsl::code_pays_etranger_2)), + dsl::libelle_pays_etranger_2.eq(excluded(dsl::libelle_pays_etranger_2)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::enseigne_1.eq(excluded(dsl::enseigne_1)), + dsl::enseigne_2.eq(excluded(dsl::enseigne_2)), + dsl::enseigne_3.eq(excluded(dsl::enseigne_3)), + dsl::denomination_usuelle.eq(excluded(dsl::denomination_usuelle)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; + + Ok((next_cursor, updated_count)) + } } diff --git a/src/models/group_metadata/common.rs b/src/models/group_metadata/common.rs index 59ce012..c2156a3 100644 --- a/src/models/group_metadata/common.rs +++ b/src/models/group_metadata/common.rs @@ -23,6 +23,7 @@ pub struct Metadata { pub url: String, pub created_at: DateTime, pub updated_at: DateTime, + pub last_insee_synced_timestamp: Option>, } #[derive(AsChangeset)] diff --git a/src/models/group_metadata/mod.rs b/src/models/group_metadata/mod.rs index f604968..09b2fa6 100644 --- a/src/models/group_metadata/mod.rs +++ b/src/models/group_metadata/mod.rs @@ -68,6 +68,19 @@ pub fn set_last_imported_timestamp( .map_err(|error| error.into()) } +pub fn set_last_insee_synced_timestamp( + connectors: &Connectors, + group_type: GroupType, + timestamp: DateTime, +) -> Result { + let connection = connectors.local.pool.get()?; + diesel::update(dsl::group_metadata.filter(dsl::group_type.eq(group_type))) + .set(dsl::last_insee_synced_timestamp.eq(timestamp)) + .execute(&connection) + .map(|count| count > 0) + .map_err(|error| error.into()) +} + pub fn reset_staging_timestamps( connectors: &Connectors, group_type: GroupType, diff --git a/src/models/schema.rs b/src/models/schema.rs index 8077f85..dff139d 100644 --- a/src/models/schema.rs +++ b/src/models/schema.rs @@ -117,6 +117,7 @@ table! { url -> Text, created_at -> Timestamptz, updated_at -> Timestamptz, + last_insee_synced_timestamp -> Nullable, } } diff --git a/src/models/unite_legale/common.rs b/src/models/unite_legale/common.rs index f2aff92..1d97939 100644 --- a/src/models/unite_legale/common.rs +++ b/src/models/unite_legale/common.rs @@ -1,7 +1,9 @@ +use super::super::schema::unite_legale; use chrono::{NaiveDate, NaiveDateTime}; use serde::Serialize; -#[derive(Queryable, Serialize, Clone, Debug)] +#[derive(Insertable, Queryable, Serialize, Clone, Debug)] +#[table_name = "unite_legale"] pub struct UniteLegale { pub siren: String, pub statut_diffusion: String, diff --git a/src/models/unite_legale/mod.rs b/src/models/unite_legale/mod.rs index f2e80ab..9b972f1 100644 --- a/src/models/unite_legale/mod.rs +++ b/src/models/unite_legale/mod.rs @@ -4,22 +4,25 @@ pub mod error; use super::common::{Error as UpdatableError, UpdatableModel}; use super::schema::unite_legale::dsl; -use crate::connectors::Connectors; +use crate::connectors::{local::Connection, Connectors}; +use async_trait::async_trait; +use chrono::NaiveDateTime; use common::UniteLegale; +use diesel::pg::upsert::excluded; use diesel::prelude::*; use diesel::sql_query; use error::Error; -pub fn get(connectors: &Connectors, siren: &String) -> Result { - let connection = connectors.local.pool.get()?; +pub fn get(connection: &Connection, siren: &String) -> Result { dsl::unite_legale .find(siren) - .first::(&connection) + .first::(connection) .map_err(|error| error.into()) } pub struct UniteLegaleModel {} +#[async_trait] impl UpdatableModel for UniteLegaleModel { fn count(&self, connectors: &Connectors) -> Result { let connection = connectors.local.pool.get()?; @@ -87,4 +90,92 @@ impl UpdatableModel for UniteLegaleModel { Ok(()) }) } + + async fn get_total_count( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + ) -> Result { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + Ok(insee.get_total_unites_legales(start_timestamp).await?) + } + + // SELECT date_dernier_traitement FROM unite_legale WHERE date_dernier_traitement IS NOT NULL ORDER BY date_dernier_traitement DESC LIMIT 1; + fn get_last_insee_synced_timestamp( + &self, + connectors: &Connectors, + ) -> Result, UpdatableError> { + let connection = connectors.local.pool.get()?; + dsl::unite_legale + .select(dsl::date_dernier_traitement) + .order(dsl::date_dernier_traitement.desc()) + .filter(dsl::date_dernier_traitement.is_not_null()) + .first::>(&connection) + .map_err(|error| error.into()) + } + + async fn update_daily_data( + &self, + connectors: &mut Connectors, + start_timestamp: NaiveDateTime, + cursor: String, + ) -> Result<(Option, usize), UpdatableError> { + let insee = connectors + .insee + .as_mut() + .ok_or(UpdatableError::MissingInseeConnector)?; + + let (next_cursor, unites_legales) = insee + .get_daily_unites_legales(start_timestamp, cursor) + .await?; + + let connection = connectors.local.pool.get()?; + + let updated_count = diesel::insert_into(dsl::unite_legale) + .values(&unites_legales) + .on_conflict(dsl::siren) + .do_update() + .set(( + dsl::statut_diffusion.eq(excluded(dsl::statut_diffusion)), + dsl::unite_purgee.eq(excluded(dsl::unite_purgee)), + dsl::date_creation.eq(excluded(dsl::date_creation)), + dsl::sigle.eq(excluded(dsl::sigle)), + dsl::sexe.eq(excluded(dsl::sexe)), + dsl::prenom_1.eq(excluded(dsl::prenom_1)), + dsl::prenom_2.eq(excluded(dsl::prenom_2)), + dsl::prenom_3.eq(excluded(dsl::prenom_3)), + dsl::prenom_4.eq(excluded(dsl::prenom_4)), + dsl::prenom_usuel.eq(excluded(dsl::prenom_usuel)), + dsl::pseudonyme.eq(excluded(dsl::pseudonyme)), + dsl::identifiant_association.eq(excluded(dsl::identifiant_association)), + dsl::tranche_effectifs.eq(excluded(dsl::tranche_effectifs)), + dsl::annee_effectifs.eq(excluded(dsl::annee_effectifs)), + dsl::date_dernier_traitement.eq(excluded(dsl::date_dernier_traitement)), + dsl::nombre_periodes.eq(excluded(dsl::nombre_periodes)), + dsl::categorie_entreprise.eq(excluded(dsl::categorie_entreprise)), + dsl::annee_categorie_entreprise.eq(excluded(dsl::annee_categorie_entreprise)), + dsl::date_debut.eq(excluded(dsl::date_debut)), + dsl::etat_administratif.eq(excluded(dsl::etat_administratif)), + dsl::nom.eq(excluded(dsl::nom)), + dsl::nom_usage.eq(excluded(dsl::nom_usage)), + dsl::denomination.eq(excluded(dsl::denomination)), + dsl::denomination_usuelle_1.eq(excluded(dsl::denomination_usuelle_1)), + dsl::denomination_usuelle_2.eq(excluded(dsl::denomination_usuelle_2)), + dsl::denomination_usuelle_3.eq(excluded(dsl::denomination_usuelle_3)), + dsl::categorie_juridique.eq(excluded(dsl::categorie_juridique)), + dsl::activite_principale.eq(excluded(dsl::activite_principale)), + dsl::nomenclature_activite_principale + .eq(excluded(dsl::nomenclature_activite_principale)), + dsl::nic_siege.eq(excluded(dsl::nic_siege)), + dsl::economie_sociale_solidaire.eq(excluded(dsl::economie_sociale_solidaire)), + dsl::caractere_employeur.eq(excluded(dsl::caractere_employeur)), + )) + .execute(&connection)?; + + Ok((next_cursor, updated_count)) + } } diff --git a/src/models/update_metadata/common.rs b/src/models/update_metadata/common.rs index 540d348..75f1906 100644 --- a/src/models/update_metadata/common.rs +++ b/src/models/update_metadata/common.rs @@ -8,6 +8,21 @@ use diesel::sql_types::{Jsonb, Text}; use serde::{Deserialize, Serialize}; use std::io::Write; +#[derive(Queryable, Serialize)] +pub struct UpdateMetadata { + pub id: i32, + pub synthetic_group_type: SyntheticGroupType, + pub force: bool, + pub data_only: bool, + pub status: String, + pub summary: Option, + pub error: Option, + pub launched_timestamp: DateTime, + pub finished_timestamp: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + #[derive(Insertable)] #[table_name = "update_metadata"] pub struct LaunchUpdateMetadata { @@ -23,7 +38,7 @@ pub struct LaunchUpdateMetadata { pub struct FinishedUpdateMetadata { pub status: UpdateStatus, pub summary: UpdateSummary, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, } #[derive(AsChangeset)] @@ -57,7 +72,10 @@ pub struct UpdateGroupSummary { pub updated: bool, pub status_label: String, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, + pub planned_count: u32, + pub done_count: u32, + pub reference_timestamp: Option>, } #[derive(Deserialize, Serialize, Clone, Copy, Debug)] @@ -67,6 +85,7 @@ pub enum Step { InsertData, SwapData, CleanFile, + SyncInsee, } #[derive(Deserialize, Serialize, Clone, Debug)] @@ -74,7 +93,7 @@ pub struct UpdateStepSummary { pub step: Step, pub updated: bool, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, pub groups: Vec, } @@ -83,7 +102,7 @@ pub struct UpdateStepSummary { pub struct UpdateSummary { pub updated: bool, pub started_timestamp: DateTime, - pub finished_timestamp: DateTime, + pub finished_timestamp: Option>, pub steps: Vec, } diff --git a/src/models/update_metadata/mod.rs b/src/models/update_metadata/mod.rs index d1120cd..755665a 100644 --- a/src/models/update_metadata/mod.rs +++ b/src/models/update_metadata/mod.rs @@ -6,7 +6,7 @@ use crate::connectors::Connectors; use chrono::{DateTime, Utc}; use common::{ ErrorUpdateMetadata, FinishedUpdateMetadata, LaunchUpdateMetadata, SyntheticGroupType, - UpdateStatus, UpdateSummary, + UpdateMetadata, UpdateStatus, UpdateSummary, }; use diesel::prelude::*; use error::Error; @@ -51,6 +51,16 @@ pub fn launch_update( } } +pub fn progress_update(connectors: &Connectors, summary: UpdateSummary) -> Result { + let connection = connectors.local.pool.get()?; + + diesel::update(dsl::update_metadata.filter(dsl::status.eq(UpdateStatus::Launched))) + .set(dsl::summary.eq(summary)) + .execute(&connection) + .map(|count| count > 0) + .map_err(|error| error.into()) +} + pub fn finished_update(connectors: &Connectors, summary: UpdateSummary) -> Result { let connection = connectors.local.pool.get()?; let finished_timestamp = summary.finished_timestamp; @@ -58,7 +68,7 @@ pub fn finished_update(connectors: &Connectors, summary: UpdateSummary) -> Resul diesel::update(dsl::update_metadata.filter(dsl::status.eq(UpdateStatus::Launched))) .set(&FinishedUpdateMetadata { status: UpdateStatus::Finished, - summary: summary, + summary, finished_timestamp, }) .execute(&connection) @@ -83,3 +93,12 @@ pub fn error_update( .map(|count| count > 0) .map_err(|error| error.into()) } + +pub fn current_update(connectors: &Connectors) -> Result { + let connection = connectors.local.pool.get()?; + + dsl::update_metadata + .order(dsl::launched_timestamp.desc()) + .first::(&connection) + .map_err(|error| error.into()) +} diff --git a/src/update/action/clean.rs b/src/update/action/clean.rs index ee3a8df..4ae56ca 100644 --- a/src/update/action/clean.rs +++ b/src/update/action/clean.rs @@ -1,10 +1,10 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; -use chrono::Utc; +use async_trait::async_trait; use std::fs::remove_file; use std::path::PathBuf; @@ -13,19 +13,20 @@ pub struct CleanAction { pub file_folder: String, } +#[async_trait] impl Action for CleanAction { - fn step(&self) -> Step { - Step::CleanFile - } - - fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, - connectors: &Connectors, - ) -> Result { - println!("[Clean] Cleaning {:#?}", group_type); - let started_timestamp = Utc::now(); + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[Clean] Cleaning {:#?}", group_type); + + summary_delegate.start(connectors, None, 2)?; + let mut updated = true; + let mut done_count = 2; let mut status_label = String::from("cleaned"); let metadata = group_metadata::get(connectors, group_type)?; @@ -41,27 +42,25 @@ impl Action for CleanAction { csv_path.set_extension("csv"); if let Err(error) = remove_file(zip_path) { - println!("[Clean] Zip not deleted ({})", error); + log::debug!("[Clean] Zip not deleted ({})", error); updated = false; + done_count -= 1; status_label = String::from("zip not deleted"); } if let Err(error) = remove_file(csv_path) { - println!("[Clean] CSV not deleted ({})", error); + log::debug!("[Clean] CSV not deleted ({})", error); updated = false; + done_count -= 1; status_label = String::from("csv not deleted"); } group_metadata::reset_staging_timestamps(connectors, group_type)?; - println!("[Clean] Finished cleaning of {:#?}", group_type); + summary_delegate.finish(connectors, status_label, done_count, updated)?; + + log::debug!("[Clean] Finished cleaning of {:#?}", group_type); - Ok(UpdateGroupSummary { - group_type, - updated, - status_label, - started_timestamp, - finished_timestamp: Utc::now(), - }) + Ok(()) } } diff --git a/src/update/action/common.rs b/src/update/action/common.rs index d3bc3c1..44da398 100644 --- a/src/update/action/common.rs +++ b/src/update/action/common.rs @@ -1,13 +1,15 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; +use async_trait::async_trait; -pub trait Action { - fn step(&self) -> Step; - fn execute( +#[async_trait] +pub trait Action: Sync + Send { + async fn execute<'a, 'b>( &self, group_type: GroupType, - connectors: &Connectors, - ) -> Result; + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error>; } diff --git a/src/update/action/download.rs b/src/update/action/download.rs deleted file mode 100644 index 8f2572b..0000000 --- a/src/update/action/download.rs +++ /dev/null @@ -1,103 +0,0 @@ -use super::super::error::Error; -use super::common::Action; -use crate::connectors::Connectors; -use crate::models::group_metadata; -use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; -use chrono::{DateTime, Utc}; -use reqwest::header::LAST_MODIFIED; -use std::fs::{create_dir_all, File}; -use std::io; -use std::path::PathBuf; - -pub struct DownloadAction { - pub temp_folder: String, - pub force: bool, -} - -impl Action for DownloadAction { - fn step(&self) -> Step { - Step::DownloadFile - } - - fn execute( - &self, - group_type: GroupType, - connectors: &Connectors, - ) -> Result { - println!("[Download] Downloading {:#?}", group_type); - let started_timestamp = Utc::now(); - - let metadata = group_metadata::get(connectors, group_type)?; - - // Create temp path - create_dir_all(self.temp_folder.clone()) - .map_err(|io_error| Error::TempFolderCreationError { io_error })?; - - // Get Zip path - let mut zip_path = PathBuf::from(self.temp_folder.clone()); - zip_path.push(metadata.file_name); - zip_path.set_extension("zip"); - - // Prepare file download - let mut resp = reqwest::blocking::get(metadata.url.as_str()) - .map_err(|req_error| Error::DownloadError { req_error })?; - - // Decode Last-Modified header - let last_modified_str = resp - .headers() - .get(LAST_MODIFIED) - .ok_or(Error::MissingLastModifiedHeader)? - .to_str() - .map_err(|head_error| Error::InvalidLastModifiedHeader { head_error })?; - let last_modified = DateTime::parse_from_rfc2822(last_modified_str) - .map_err(|date_error| Error::InvalidLastModifiedDate { date_error })?; - let last_modified = last_modified.with_timezone(&Utc); - - // Test if not already imported or downloaded - if !self.force { - if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { - if last_modified.le(&last_imported_timestamp) { - println!("[Download] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); - } - } - - if let Some(staging_file_timestamp) = metadata.staging_file_timestamp { - if last_modified.le(&staging_file_timestamp) { - println!("[Download] {:#?} already downloaded", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already downloaded"), - started_timestamp, - finished_timestamp: Utc::now(), - }); - } - } - } - - // Download data and store it on filesystem - let mut out = - File::create(zip_path).map_err(|io_error| Error::FileCreationError { io_error })?; - io::copy(&mut resp, &mut out).map_err(|io_error| Error::FileCopyError { io_error })?; - println!("[Download] Download of {:#?} finished", group_type); - - // Update staging file timestamp - group_metadata::set_staging_file_timestamp(connectors, group_type, last_modified)?; - - return Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("downloaded"), - started_timestamp, - finished_timestamp: Utc::now(), - }); - } -} diff --git a/src/update/action/download_stock.rs b/src/update/action/download_stock.rs new file mode 100644 index 0000000..c1e4eab --- /dev/null +++ b/src/update/action/download_stock.rs @@ -0,0 +1,128 @@ +use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; +use super::common::Action; +use crate::connectors::Connectors; +use crate::models::group_metadata; +use crate::models::group_metadata::common::GroupType; +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::stream::TryStreamExt; +use reqwest::header::LAST_MODIFIED; +use std::fs::create_dir_all; +use std::path::PathBuf; +use std::time::Duration; +use tokio::fs::File; +use tokio_util::compat::FuturesAsyncReadCompatExt; + +pub struct DownloadAction { + pub temp_folder: String, + pub force: bool, +} + +#[async_trait] +impl Action for DownloadAction { + async fn execute<'a, 'b>( + &self, + group_type: GroupType, + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[Download] Downloading {:#?}", group_type); + summary_delegate.start(connectors, None, 1)?; + + let metadata = group_metadata::get(connectors, group_type)?; + + // Prepare file download + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(3600)) + .build() + .map_err(|req_error| Error::DownloadError { req_error })?; + + let resp = client + .get(&metadata.url) + .send() + .await + .map_err(|req_error| Error::DownloadError { req_error })?; + + // Decode Last-Modified header + let last_modified_str = resp + .headers() + .get(LAST_MODIFIED) + .ok_or(Error::MissingLastModifiedHeader)? + .to_str() + .map_err(|head_error| Error::InvalidLastModifiedHeader { head_error })?; + + let last_modified = DateTime::parse_from_rfc2822(last_modified_str) + .map_err(|date_error| Error::InvalidLastModifiedDate { date_error })?; + + let last_modified = last_modified.with_timezone(&Utc); + + // Test if not already imported or downloaded + if !self.force { + if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { + if last_modified.le(&last_imported_timestamp) { + log::debug!("[Download] {:#?} already imported", group_type); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); + } + } + + if let Some(staging_file_timestamp) = metadata.staging_file_timestamp { + if last_modified.le(&staging_file_timestamp) { + log::debug!("[Download] {:#?} already downloaded", group_type); + + summary_delegate.finish( + connectors, + String::from("already downloaded"), + 0, + false, + )?; + + return Ok(()); + } + } + } + + // Create temp path + create_dir_all(self.temp_folder.clone()) + .map_err(|io_error| Error::TempFolderCreationError { io_error })?; + + // Get Zip path + let mut zip_path = PathBuf::from(self.temp_folder.clone()); + zip_path.push(metadata.file_name); + zip_path.set_extension("zip"); + + // Create an output file into which we will save current stock. + let mut outfile = File::create(zip_path) + .await + .map_err(|io_error| Error::FileCreationError { io_error })?; + + let mut stream = resp + .bytes_stream() // Convert the body of the response into a futures::io::Stream. + .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) // We must first convert the reqwest::Error into an futures::io::Error. + .into_async_read() // Convert the stream into an futures::io::AsyncRead. + .compat(); // Convert the futures::io::AsyncRead into a tokio::io::AsyncRead. + + // Invoke tokio::io::copy to actually perform the download. + tokio::io::copy(&mut stream, &mut outfile) + .await + .map_err(|io_error| Error::FileCopyError { io_error })?; + + log::debug!("[Download] Download of {:#?} finished", group_type); + + // Update staging file timestamp + group_metadata::set_staging_file_timestamp(connectors, group_type, last_modified)?; + + summary_delegate.finish(connectors, String::from("downloaded"), 1, true)?; + + Ok(()) + } +} diff --git a/src/update/action/insert.rs b/src/update/action/insert_stock.rs similarity index 51% rename from src/update/action/insert.rs rename to src/update/action/insert_stock.rs index d19ed38..6e18cf6 100644 --- a/src/update/action/insert.rs +++ b/src/update/action/insert_stock.rs @@ -1,10 +1,10 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; -use chrono::Utc; +use async_trait::async_trait; use std::fs::canonicalize; use std::path::PathBuf; @@ -13,18 +13,16 @@ pub struct InsertAction { pub force: bool, } +#[async_trait] impl Action for InsertAction { - fn step(&self) -> Step { - Step::InsertData - } - - fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, - connectors: &Connectors, - ) -> Result { - println!("[Insert] Insert {:#?}", group_type); - let started_timestamp = Utc::now(); + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[Insert] Insert {:#?}", group_type); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -32,14 +30,11 @@ impl Action for InsertAction { let staging_csv_file_timestamp = match metadata.staging_csv_file_timestamp { Some(staging_csv_file_timestamp) => staging_csv_file_timestamp, None => { - println!("[Insert] Nothing to insert for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to insert"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Insert] Nothing to insert for {:#?}", group_type); + + summary_delegate.finish(connectors, String::from("nothing to insert"), 0, false)?; + + return Ok(()); } }; @@ -48,26 +43,30 @@ impl Action for InsertAction { if let Some(staging_imported_timestamp) = metadata.staging_imported_timestamp { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { - println!("[Insert] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Insert] {:#?} already imported", group_type); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } if staging_csv_file_timestamp.le(&staging_imported_timestamp) { - println!("[Insert] {:#?} already inserted", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already inserted"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Insert] {:#?} already inserted", group_type); + + summary_delegate.finish( + connectors, + String::from("already inserted"), + 0, + false, + )?; + + return Ok(()); } } } @@ -76,8 +75,10 @@ impl Action for InsertAction { let mut csv_path = PathBuf::from(self.db_folder.clone()); csv_path.push(metadata.file_name); csv_path.set_extension("csv"); + let absolute_csv_path = canonicalize(csv_path) .map_err(|io_error| Error::InvalidComponentInCSVPath { io_error })?; + let csv_path_str = absolute_csv_path .into_os_string() .into_string() @@ -93,14 +94,10 @@ impl Action for InsertAction { staging_csv_file_timestamp, )?; - println!("[Insert] Finished insert of {:#?}", group_type); + log::debug!("[Insert] Finished insert of {:#?}", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("inserted"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("inserted"), 1, true)?; + + Ok(()) } } diff --git a/src/update/action/mod.rs b/src/update/action/mod.rs index a582d83..cff54bb 100644 --- a/src/update/action/mod.rs +++ b/src/update/action/mod.rs @@ -1,53 +1,57 @@ use super::common::Config; use super::error::Error; +use super::summary::SummaryStepDelegate; use crate::connectors::Connectors; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary, UpdateStepSummary}; -use chrono::Utc; +use crate::models::update_metadata::common::Step; use common::Action; pub mod clean; pub mod common; -pub mod download; -pub mod insert; +pub mod download_stock; +pub mod insert_stock; pub mod swap; -pub mod unzip; +pub mod sync_insee; +pub mod unzip_stock; -pub fn execute_step( +pub async fn execute_step<'a>( step: Step, config: &Config, groups: &Vec, - connectors: &Connectors, -) -> Result { - let started_timestamp = Utc::now(); - let mut groups_summary: Vec = vec![]; + connectors: &mut Connectors, + summary_delegate: &'a mut SummaryStepDelegate<'a>, +) -> Result<(), Error> { let action = build_action(config, step); + summary_delegate.start(connectors)?; + for group in groups { - groups_summary.push(action.execute(*group, connectors)?); + action + .execute( + *group, + connectors, + &mut summary_delegate.group_delegate(*group), + ) + .await?; } - Ok(UpdateStepSummary { - step: Step::DownloadFile, - updated: groups_summary.iter().find(|&g| g.updated).is_some(), - started_timestamp, - finished_timestamp: Utc::now(), - groups: groups_summary, - }) + summary_delegate.finish(connectors)?; + + Ok(()) } fn build_action(config: &Config, step: Step) -> Box { match step { - Step::DownloadFile => Box::new(download::DownloadAction { + Step::DownloadFile => Box::new(download_stock::DownloadAction { temp_folder: config.temp_folder.clone(), force: config.force, }), - Step::UnzipFile => Box::new(unzip::UnzipAction { + Step::UnzipFile => Box::new(unzip_stock::UnzipAction { temp_folder: config.temp_folder.clone(), file_folder: config.file_folder.clone(), force: config.force, }), - Step::InsertData => Box::new(insert::InsertAction { + Step::InsertData => Box::new(insert_stock::InsertAction { db_folder: config.file_folder.clone(), force: config.force, }), @@ -58,5 +62,6 @@ fn build_action(config: &Config, step: Step) -> Box { temp_folder: config.temp_folder.clone(), file_folder: config.file_folder.clone(), }), + Step::SyncInsee => Box::new(sync_insee::SyncInseeAction {}), } } diff --git a/src/update/action/swap.rs b/src/update/action/swap.rs index 699cedd..2e87fa8 100644 --- a/src/update/action/swap.rs +++ b/src/update/action/swap.rs @@ -1,27 +1,25 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; -use chrono::Utc; +use async_trait::async_trait; pub struct SwapAction { pub force: bool, } +#[async_trait] impl Action for SwapAction { - fn step(&self) -> Step { - Step::SwapData - } - - fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, - connectors: &Connectors, - ) -> Result { - println!("[Insert] Swapping {:#?}", group_type); - let started_timestamp = Utc::now(); + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[Swap] Swapping {:#?}", group_type); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -29,14 +27,11 @@ impl Action for SwapAction { let staging_imported_timestamp = match metadata.staging_imported_timestamp { Some(staging_imported_timestamp) => staging_imported_timestamp, None => { - println!("[Swap] Nothing to swap for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to swap"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Swap] Nothing to swap for {:#?}", group_type); + + summary_delegate.finish(connectors, String::from("nothing to swap"), 0, false)?; + + return Ok(()); } }; @@ -44,14 +39,16 @@ impl Action for SwapAction { if !self.force { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_imported_timestamp.le(&last_imported_timestamp) { - println!("[Swap] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Swap] {:#?} already imported", group_type); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } } @@ -80,14 +77,10 @@ impl Action for SwapAction { staging_imported_timestamp, )?; - println!("[Insert] Swap of {:#?} finished", group_type); + log::debug!("[Swap] Swap of {:#?} finished", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("swapped"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("swapped"), 1, true)?; + + Ok(()) } } diff --git a/src/update/action/sync_insee.rs b/src/update/action/sync_insee.rs new file mode 100644 index 0000000..ffe299e --- /dev/null +++ b/src/update/action/sync_insee.rs @@ -0,0 +1,89 @@ +use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; +use super::common::Action; +use crate::connectors::{insee::INITIAL_CURSOR, Connectors}; +use crate::models::group_metadata; +use crate::models::group_metadata::common::GroupType; +use async_trait::async_trait; +use chrono::{DateTime, Duration, NaiveDateTime, Utc}; + +pub struct SyncInseeAction {} + +#[async_trait] +impl Action for SyncInseeAction { + async fn execute<'a, 'b>( + &self, + group_type: GroupType, + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[SyncInsee] Syncing {:#?}", group_type); + + // Use Insee connector only if present + if connectors.insee.is_some() { + let model = group_type.get_updatable_model(); + + if let Some(last_timestamp) = model.get_last_insee_synced_timestamp(connectors)? { + let mut current_cursor: Option = Some(INITIAL_CURSOR.to_string()); + let mut updated_count = 0; + let timestamp = get_minimum_timestamp_for_request(last_timestamp); + + let planned_count = model.get_total_count(connectors, timestamp).await?; + + summary_delegate.start( + connectors, + Some(DateTime::::from_utc(timestamp, Utc)), + planned_count, + )?; + + while let Some(cursor) = current_cursor { + let (next_cursor, inserted_count) = model + .update_daily_data(connectors, timestamp, cursor) + .await?; + + current_cursor = next_cursor; + updated_count += inserted_count; + + summary_delegate.progress(connectors, updated_count as u32)?; + } + + log::debug!("[SyncInsee] {} {:#?} synced", updated_count, group_type); + + group_metadata::set_last_insee_synced_timestamp( + connectors, + group_type, + Utc::now(), + )?; + + summary_delegate.finish( + connectors, + String::from("synced"), + updated_count as u32, + updated_count > 0, + )?; + } else { + summary_delegate.finish( + connectors, + String::from("missing last treatment date"), + 0, + false, + )?; + } + } else { + summary_delegate.finish( + connectors, + String::from("no insee connector configured"), + 0, + false, + )?; + } + + log::debug!("[SyncInsee] Syncing of {:#?} done", group_type); + + Ok(()) + } +} + +fn get_minimum_timestamp_for_request(timestamp: NaiveDateTime) -> NaiveDateTime { + timestamp.max(Utc::now().naive_local() - Duration::days(31)) +} diff --git a/src/update/action/unzip.rs b/src/update/action/unzip_stock.rs similarity index 64% rename from src/update/action/unzip.rs rename to src/update/action/unzip_stock.rs index 25d41fe..016697c 100644 --- a/src/update/action/unzip.rs +++ b/src/update/action/unzip_stock.rs @@ -1,10 +1,10 @@ use super::super::error::Error; +use super::super::summary::SummaryGroupDelegate; use super::common::Action; use crate::connectors::Connectors; use crate::models::group_metadata; use crate::models::group_metadata::common::GroupType; -use crate::models::update_metadata::common::{Step, UpdateGroupSummary}; -use chrono::Utc; +use async_trait::async_trait; use std::fs::{create_dir_all, set_permissions, File, Permissions}; use std::io; use std::path::PathBuf; @@ -15,18 +15,16 @@ pub struct UnzipAction { pub force: bool, } +#[async_trait] impl Action for UnzipAction { - fn step(&self) -> Step { - Step::UnzipFile - } - - fn execute( + async fn execute<'a, 'b>( &self, group_type: GroupType, - connectors: &Connectors, - ) -> Result { - println!("[Unzip] Unzipping {:#?}", group_type); - let started_timestamp = Utc::now(); + connectors: &mut Connectors, + summary_delegate: &'b mut SummaryGroupDelegate<'a, 'b>, + ) -> Result<(), Error> { + log::debug!("[Unzip] Unzipping {:#?}", group_type); + summary_delegate.start(connectors, None, 1)?; let metadata = group_metadata::get(connectors, group_type)?; @@ -34,14 +32,11 @@ impl Action for UnzipAction { let staging_file_timestamp = match metadata.staging_file_timestamp { Some(staging_file_timestamp) => staging_file_timestamp, None => { - println!("[Unzip] Nothing to unzip for {:#?}", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("nothing to unzip"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Unzip] Nothing to unzip for {:#?}", group_type); + + summary_delegate.finish(connectors, String::from("nothing to unzip"), 0, false)?; + + return Ok(()); } }; @@ -50,26 +45,30 @@ impl Action for UnzipAction { if let Some(staging_csv_file_timestamp) = metadata.staging_csv_file_timestamp { if let Some(last_imported_timestamp) = metadata.last_imported_timestamp { if staging_csv_file_timestamp.le(&last_imported_timestamp) { - println!("[Unzip] {:#?} already imported", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already imported"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Unzip] {:#?} already imported", group_type); + + summary_delegate.finish( + connectors, + String::from("already imported"), + 0, + false, + )?; + + return Ok(()); } } if staging_file_timestamp.le(&staging_csv_file_timestamp) { - println!("[Unzip] {:#?} already unzipped", group_type); - return Ok(UpdateGroupSummary { - group_type, - updated: false, - status_label: String::from("already unzipped"), - started_timestamp, - finished_timestamp: Utc::now(), - }); + log::debug!("[Unzip] {:#?} already unzipped", group_type); + + summary_delegate.finish( + connectors, + String::from("already unzipped"), + 0, + false, + )?; + + return Ok(()); } } } @@ -104,7 +103,7 @@ impl Action for UnzipAction { .by_index(0) .map_err(|zip_error| Error::ZipAccessFileError { zip_error })?; - println!( + log::debug!( "[Unzip] Unzipping file {:#?} extracted to \"{}\" ({} bytes)", group_type, csv_path.as_path().display(), @@ -132,14 +131,10 @@ impl Action for UnzipAction { staging_file_timestamp, )?; - println!("[Unzip] Unzip of {:#?} finished", group_type); + log::debug!("[Unzip] Unzip of {:#?} finished", group_type); - Ok(UpdateGroupSummary { - group_type, - updated: true, - status_label: String::from("unzipped"), - started_timestamp, - finished_timestamp: Utc::now(), - }) + summary_delegate.finish(connectors, String::from("unzipped"), 1, true)?; + + Ok(()) } } diff --git a/src/update/common.rs b/src/update/common.rs index bff66fb..5ac71bd 100644 --- a/src/update/common.rs +++ b/src/update/common.rs @@ -5,4 +5,5 @@ pub struct Config { pub temp_folder: String, pub file_folder: String, pub db_folder: String, + pub asynchronous: bool, } diff --git a/src/update/error.rs b/src/update/error.rs index 8928364..07b059c 100644 --- a/src/update/error.rs +++ b/src/update/error.rs @@ -1,3 +1,4 @@ +use crate::connectors::insee::error::InseeUpdateError; use crate::models; use crate::models::group_metadata::common::GroupType; use crate::models::{group_metadata, update_metadata}; @@ -5,32 +6,34 @@ use custom_error::custom_error; use std::process; custom_error! { pub Error - MetadataModelError {source: group_metadata::error::Error} = "Error on Metadata model: {source}.", - UpdateMetadataModelError {source: update_metadata::error::Error} = "Error on UpdateMetadata model: {source}.", - UpdatableModelError {source: models::common::Error} = "Error on UpdatableModel model: {source}.", - TempFolderCreationError {io_error: std::io::Error} = "Unable to create temporary folder ({io_error}).", - FileFolderCreationError {io_error: std::io::Error} = "Unable to create data folder ({io_error}).", - FileCreationError {io_error: std::io::Error} = "Unable to create file for download ({io_error}).", - FileCopyError {io_error: std::io::Error} = "Unable to copy file from download ({io_error}).", - FileCSVCreationError {io_error: std::io::Error} = "Unable to create CSV file for unzip ({io_error}).", - FileCSVCopyError {io_error: std::io::Error} = "Unable to copy CSV file from archive ({io_error}).", - FileCSVPermissionError {io_error: std::io::Error} = "Unable to set permission for CSV file ({io_error}).", - DownloadError {req_error: reqwest::Error} = "Unable to download data from remote server ({req_error}).", - ZipOpenError {io_error: std::io::Error} = "Unable to open data zip file ({io_error}).", - ZipDecodeError {zip_error: zip::result::ZipError} = "Unable to decode zip file ({zip_error}).", - ZipFormatError = "Archive has more than one file inside it, you should review it before running it again.", - ZipAccessFileError {zip_error: zip::result::ZipError} = "Unable to open file in archive ({zip_error}).", - MissingLastModifiedHeader = "Needed header 'Last-Modified' is missing while downloading.", - InvalidLastModifiedHeader {head_error: reqwest::header::ToStrError} = "Needed header 'Last-Modified' is invalid while downloading ({head_error}).", - InvalidLastModifiedDate {date_error: chrono::format::ParseError} = "Needed header 'Last-Modified' is invalid while downloading ({date_error}).", - InvalidCSVPath = "Invalid CSV path, not UTF8 compatible.", - InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error}).", - SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override.", + MetadataModelError {source: group_metadata::error::Error} = "Error on Metadata model: {source}", + UpdateMetadataModelError {source: update_metadata::error::Error} = "Error on UpdateMetadata model: {source}", + UpdatableModelError {source: models::common::Error} = "Error on UpdatableModel model: {source}", + TempFolderCreationError {io_error: std::io::Error} = "Unable to create temporary folder ({io_error})", + FileFolderCreationError {io_error: std::io::Error} = "Unable to create data folder ({io_error})", + FileCreationError {io_error: std::io::Error} = "Unable to create file for download ({io_error})", + FileCopyError {io_error: std::io::Error} = "Unable to copy file from download ({io_error})", + FileCSVCreationError {io_error: std::io::Error} = "Unable to create CSV file for unzip ({io_error})", + FileCSVCopyError {io_error: std::io::Error} = "Unable to copy CSV file from archive ({io_error})", + FileCSVPermissionError {io_error: std::io::Error} = "Unable to set permission for CSV file ({io_error})", + DownloadError {req_error: reqwest::Error} = "Unable to download data from remote server ({req_error})", + ZipOpenError {io_error: std::io::Error} = "Unable to open data zip file ({io_error})", + ZipDecodeError {zip_error: zip::result::ZipError} = "Unable to decode zip file ({zip_error})", + ZipFormatError = "Archive has more than one file inside it, you should review it before running it again", + ZipAccessFileError {zip_error: zip::result::ZipError} = "Unable to open file in archive ({zip_error})", + MissingLastModifiedHeader = "Needed header 'Last-Modified' is missing while downloading", + InvalidLastModifiedHeader {head_error: reqwest::header::ToStrError} = "Needed header 'Last-Modified' is invalid while downloading ({head_error})", + InvalidLastModifiedDate {date_error: chrono::format::ParseError} = "Needed header 'Last-Modified' is invalid while downloading ({date_error})", + InvalidCSVPath = "Invalid CSV path, not UTF8 compatible", + InvalidComponentInCSVPath {io_error: std::io::Error} = "Invalid component in CSV path ({io_error})", + SwapStoppedTooMuchDifference {group_type: GroupType} = "Swapping stopped on {group_type}, more than 1% difference between the old values and the new ones. Use --force to override", + SyncInseeError {source: InseeUpdateError} = "[SyncInsee] {source}", + WaitThreadError {source: tokio::task::JoinError} = "[Asynchronous] Error while waiting for thread: {source}", } impl Error { pub fn exit(&self) -> ! { - eprintln!("{}", self); + log::error!("{}", self); process::exit(1); } } diff --git a/src/update/mod.rs b/src/update/mod.rs index 1c2539a..a56a977 100644 --- a/src/update/mod.rs +++ b/src/update/mod.rs @@ -1,22 +1,24 @@ use crate::connectors::Connectors; use crate::models::update_metadata; use crate::models::update_metadata::common::{ - Step, SyntheticGroupType, UpdateStepSummary, UpdateSummary, + Step, SyntheticGroupType, UpdateMetadata, UpdateSummary, }; use action::execute_step; use chrono::Utc; use common::Config; use error::Error; +use tokio::task; pub mod action; pub mod common; pub mod error; +pub mod summary; -pub fn update( +pub async fn update( synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, -) -> Result { + connectors: &mut Connectors, +) -> Result { // Build and execute workflow execute_workflow( build_workflow(&config), @@ -24,61 +26,89 @@ pub fn update( config, connectors, ) + .await } -pub fn update_step( +pub async fn update_step( step: Step, synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, -) -> Result { + connectors: &mut Connectors, +) -> Result { // Execute step - execute_workflow(vec![step], synthetic_group_type, config, connectors) + execute_workflow(vec![step], synthetic_group_type, config, connectors).await } -fn execute_workflow( +async fn execute_workflow( workflow: Vec, synthetic_group_type: SyntheticGroupType, config: Config, - connectors: &Connectors, -) -> Result { - // Register start - update_metadata::launch_update( + connectors: &mut Connectors, +) -> Result { + // Execute workflow + let mut summary = UpdateSummary::new(); + + summary.start( connectors, synthetic_group_type, config.force, config.data_only, )?; - // Start - println!("[Update] Starting"); - let started_timestamp = Utc::now(); + let asynchronous = config.asynchronous; + let mut thread_connectors = connectors.clone(); - // Execute workflow - let result_steps: Result, Error> = workflow - .into_iter() - .map(|step| execute_step(step, &config, &synthetic_group_type.into(), connectors)) - .collect(); - - let steps = match result_steps { - Ok(s) => s, - Err(error) => { - update_metadata::error_update(connectors, error.to_string(), Utc::now())?; - return Err(error); - } - }; - - // End - println!("[Update] Finished"); - let summary = UpdateSummary { - updated: steps.iter().find(|&s| s.updated).is_some(), - started_timestamp, - finished_timestamp: Utc::now(), - steps, - }; - update_metadata::finished_update(connectors, summary.clone())?; - - Ok(summary) + let handle = task::spawn(async move { + task::yield_now().await; + + execute_workflow_thread( + workflow, + synthetic_group_type, + config, + &mut thread_connectors, + summary, + ) + .await + }); + + if !asynchronous { + handle.await??; + } + + Ok(update_metadata::current_update(&connectors)?) +} + +async fn execute_workflow_thread( + workflow: Vec, + synthetic_group_type: SyntheticGroupType, + config: Config, + mut connectors: &mut Connectors, + mut summary: UpdateSummary, +) -> Result<(), Error> { + log::debug!("[Update] Starting"); + + for step in workflow.into_iter() { + execute_step( + step, + &config, + &synthetic_group_type.into(), + &mut connectors, + &mut summary.step_delegate(step), + ) + .await + .or_else(|error| { + log::error!("[Update] Errored: {}", error.to_string()); + + update_metadata::error_update(&mut connectors, error.to_string(), Utc::now())?; + Err(error) + })?; + } + + summary.finish(&mut connectors)?; + + log::debug!("[Update] Finished"); + + Ok(()) } fn build_workflow(config: &Config) -> Vec { @@ -86,6 +116,7 @@ fn build_workflow(config: &Config) -> Vec { if !config.data_only { workflow.push(Step::DownloadFile); + // If INSEE && newly downloaded file, get update date from INSEE and update workflow.push(Step::UnzipFile); } @@ -96,5 +127,8 @@ fn build_workflow(config: &Config) -> Vec { workflow.push(Step::CleanFile); } + // If INSEE, download and insert daily modifications + workflow.push(Step::SyncInsee); + workflow } diff --git a/src/update/summary.rs b/src/update/summary.rs new file mode 100644 index 0000000..8454198 --- /dev/null +++ b/src/update/summary.rs @@ -0,0 +1,157 @@ +use crate::connectors::Connectors; +use crate::models::group_metadata::common::GroupType; +use crate::models::update_metadata::{ + self, + common::{Step, SyntheticGroupType, UpdateGroupSummary, UpdateStepSummary, UpdateSummary}, + error::Error, +}; +use chrono::{DateTime, Utc}; + +pub struct SummaryStepDelegate<'a> { + _step: Step, + summary: &'a mut UpdateSummary, +} + +pub struct SummaryGroupDelegate<'a, 'b> { + _group: GroupType, + step_delegate: &'b mut SummaryStepDelegate<'a>, +} + +impl UpdateSummary { + pub fn new() -> Self { + UpdateSummary { + steps: vec![], + updated: false, + started_timestamp: Utc::now(), + finished_timestamp: None, + } + } + + pub fn step_delegate<'a>(&'a mut self, step: Step) -> SummaryStepDelegate<'a> { + let step_summary = UpdateStepSummary { + step, + updated: false, + started_timestamp: Utc::now(), + finished_timestamp: None, + groups: vec![], + }; + + self.steps.insert(0, step_summary); + + SummaryStepDelegate { + _step: step, + summary: self, + } + } + + pub fn start( + &mut self, + connectors: &Connectors, + synthetic_group: SyntheticGroupType, + force: bool, + data_only: bool, + ) -> Result<(), Error> { + update_metadata::launch_update(connectors, synthetic_group, force, data_only).map( + |date| { + self.started_timestamp = date; + Ok(()) + }, + )? + } + + pub fn finish(&mut self, connectors: &Connectors) -> Result<(), Error> { + self.finished_timestamp = Some(Utc::now()); + self.updated = self.steps.iter().find(|s| s.updated).is_some(); + + update_metadata::finished_update(connectors, self.clone()).map(|_| Ok(()))? + } +} + +impl<'a> SummaryStepDelegate<'a> { + pub fn group_delegate<'b>(&'b mut self, group: GroupType) -> SummaryGroupDelegate<'a, 'b> { + if let Some(step) = self.summary.steps.first_mut() { + let group_summary = UpdateGroupSummary { + group_type: group, + updated: false, + status_label: String::from("initialized"), + started_timestamp: Utc::now(), + finished_timestamp: None, + planned_count: 0, + done_count: 0, + reference_timestamp: None, + }; + + step.groups.insert(0, group_summary); + } + + SummaryGroupDelegate { + _group: group, + step_delegate: self, + } + } + + pub fn start(&self, connectors: &Connectors) -> Result<(), Error> { + update_metadata::progress_update(connectors, self.summary.clone()).map(|_| Ok(()))? + } + + pub fn finish(&mut self, connectors: &Connectors) -> Result<(), Error> { + if let Some(step_summary) = self.summary.steps.first_mut() { + step_summary.finished_timestamp = Some(Utc::now()); + step_summary.updated = step_summary.groups.iter().find(|g| g.updated).is_some(); + } + + update_metadata::progress_update(connectors, self.summary.clone()).map(|_| Ok(()))? + } +} + +impl<'a, 'b> SummaryGroupDelegate<'a, 'b> { + fn get_current_mut(&mut self) -> Option<&mut UpdateGroupSummary> { + match self.step_delegate.summary.steps.first_mut() { + Some(step_summary) => step_summary.groups.first_mut(), + None => None, + } + } + + pub fn start( + &mut self, + connectors: &Connectors, + reference_timestamp: Option>, + planned_count: u32, + ) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.reference_timestamp = reference_timestamp; + group_summary.planned_count = planned_count; + group_summary.status_label = String::from("in progress") + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } + + pub fn progress(&mut self, connectors: &Connectors, done_count: u32) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.done_count = done_count; + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } + + pub fn finish( + &mut self, + connectors: &Connectors, + status_label: String, + done_count: u32, + updated: bool, + ) -> Result<(), Error> { + if let Some(group_summary) = self.get_current_mut() { + group_summary.status_label = status_label; + group_summary.done_count = done_count; + group_summary.updated = updated; + group_summary.finished_timestamp = Some(Utc::now()); + } + + update_metadata::progress_update(connectors, self.step_delegate.summary.clone()) + .map(|_| Ok(()))? + } +}