diff --git a/.editorconfig b/.editorconfig index eadd72e..a1bd918 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,7 +20,7 @@ max_line_length = 100 # denotes a line break in Markdown trim_trailing_whitespace = false -[*.yml] +[*.{yml,yaml}] indent_size = 2 [Makefile] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..6377c96 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,119 @@ +name: test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: stable + override: true + + - name: Format + run: cargo fmt -- --check + + build: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-11 + - windows-latest + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: stable + override: true + + - uses: Swatinem/rust-cache@v2 + + - name: build and lint with clippy + run: cargo clippy --tests + + - name: Check docs + run: cargo doc --no-deps + + unit: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-11 + - windows-latest + runs-on: ${{ matrix.os }} + + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + RUSTFLAGS: -C debuginfo=line-tables-only + # Disable incremental builds by cargo for CI which should save disk space + # and hopefully avoid final link "No space left on device" + CARGO_INCREMENTAL: 0 + + steps: + - uses: actions/checkout@v3 + + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: "stable" + override: true + + - uses: Swatinem/rust-cache@v2 + + - name: Run unit tests + run: cargo test --lib + + integration: + strategy: + fail-fast: false + runs-on: ubuntu-latest + services: + postgres: + image: postgres:alpine + env: + POSTGRES_DB: sharing + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + ports: + - 5432:5432 + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + RUSTFLAGS: -C debuginfo=line-tables-only + # Disable incremental builds by cargo for CI which should save disk space + # and hopefully avoid final link "No space left on device" + CARGO_INCREMENTAL: 0 + DATABASE_URL: postgres://postgres:secret@127.0.0.1:5432/sharing + + steps: + - uses: actions/checkout@v3 + + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: "stable" + override: true + + - uses: Swatinem/rust-cache@v2 + + - name: Run integration tests + run: cargo test --tests diff --git a/Cargo.lock b/Cargo.lock index 5901946..f4e9f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,9 +108,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrow" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3724c874f1517cf898cd1c3ad18ab5071edf893c48e73139ab1e16cf0f2affe" +checksum = "b7104b9e9761613ae92fe770c741d6bbf1dbc791a0fe204400aebdd429875741" dependencies = [ "ahash 0.8.3", "arrow-arith", @@ -130,9 +130,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958823b8383ca14d0a2e973de478dd7674cd9f72837f8c41c132a0fda6a4e5e" +checksum = "38e597a8e8efb8ff52c50eaf8f4d85124ce3c1bf20fab82f476d73739d9ab1c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -145,9 +145,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db670eab50e76654065b5aed930f4367101fcddcb2223802007d1e0b4d5a2579" +checksum = "2a86d9c1473db72896bd2345ebb6b8ad75b8553ba390875c76708e8dc5c5492d" dependencies = [ "ahash 0.8.3", "arrow-buffer", @@ -155,15 +155,15 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "num", ] [[package]] name = "arrow-buffer" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f0e01c931882448c0407bd32311a624b9f099739e94e786af68adc97016b5f2" +checksum = "234b3b1c8ed00c874bf95972030ac4def6f58e02ea5a7884314388307fb3669b" dependencies = [ "half", "num", @@ -171,9 +171,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf35d78836c93f80d9362f3ccb47ff5e2c5ecfc270ff42cdf1ef80334961d44" +checksum = "22f61168b853c7faea8cea23a2169fdff9c82fb10ae5e2c07ad1cab8f6884931" dependencies = [ "arrow-array", "arrow-buffer", @@ -181,15 +181,16 @@ dependencies = [ "arrow-schema", "arrow-select", "chrono", + "half", "lexical-core", "num", ] [[package]] name = "arrow-csv" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6aa7c2531d89d01fed8c469a9b1bf97132a0bdf70b4724fe4bbb4537a50880" +checksum = "10b545c114d9bf8569c84d2fbe2020ac4eea8db462c0a37d0b65f41a90d066fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -206,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea50db4d1e1e4c2da2bfdea7b6d2722eef64267d5ab680d815f7ae42428057f5" +checksum = "c6b6852635e7c43e5b242841c7470606ff0ee70eef323004cacc3ecedd33dd8f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -218,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4042fe6585155d1ec28a8e4937ec901a3ca7a19a22b9f6cd3f551b935cd84f5" +checksum = "a66da9e16aecd9250af0ae9717ae8dd7ea0d8ca5a3e788fe3de9f4ee508da751" dependencies = [ "arrow-array", "arrow-buffer", @@ -232,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c907c4ab4f26970a3719dc06e78e8054a01d0c96da3664d23b941e201b33d2b" +checksum = "60ee0f9d8997f4be44a60ee5807443e396e025c23cf14d2b74ce56135cb04474" dependencies = [ "arrow-array", "arrow-buffer", @@ -243,31 +244,33 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 1.9.2", + "indexmap 2.0.0", "lexical-core", "num", + "serde", "serde_json", ] [[package]] name = "arrow-ord" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e131b447242a32129efc7932f58ed8931b42f35d8701c1a08f9f524da13b1d3c" +checksum = "7fcab05410e6b241442abdab6e1035177dc082bdb6f17049a4db49faed986d63" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "half", "num", ] [[package]] name = "arrow-row" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b591ef70d76f4ac28dd7666093295fece0e5f9298f49af51ea49c001e1635bb6" +checksum = "91a847dd9eb0bacd7836ac63b3475c68b2210c2c96d0ec1b808237b973bd5d73" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -275,20 +278,20 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.13.2", + "hashbrown 0.14.0", ] [[package]] name = "arrow-schema" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb327717d87eb94be5eff3b0cb8987f54059d343ee5235abf7f143c85f54cfc8" +checksum = "54df8c47918eb634c20e29286e69494fdc20cafa5173eb6dad49c7f6acece733" [[package]] name = "arrow-select" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d3c389d1cea86793934f31594f914c8547d82e91e3411d4833ad0aac3266a7" +checksum = "941dbe481da043c4bd40c805a19ec2fc008846080c4953171b62bcad5ee5f7fb" dependencies = [ "arrow-array", "arrow-buffer", @@ -299,17 +302,18 @@ dependencies = [ [[package]] name = "arrow-string" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee67790496dd310ddbf5096870324431e89aa76453e010020ac29b1184d356" +checksum = "359b2cd9e071d5a3bcf44679f9d85830afebc5b9c98a08019a570a65ae933e0f" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "num", "regex", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -488,292 +492,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "aws-config" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "hex", - "http", - "hyper", - "ring", - "time 0.3.20", - "tokio", - "tower", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-endpoint" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d" -dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "http-body", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sso" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "regex", - "tokio-stream", - "tower", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-http", - "aws-types", - "http", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297" -dependencies = [ - "aws-smithy-http", - "form_urlencoded", - "hex", - "hmac 0.12.1", - "http", - "once_cell", - "percent-encoding", - "regex", - "sha2 0.10.6", - "time 0.3.20", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63c712a28a4f2f2139759235c08bf98aca99d4fdf1b13c78c5f95613df0a5db9" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-client" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "pin-project-lite", - "tokio", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-http" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844" -dependencies = [ - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", - "hyper", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-types" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8161232eda10290f5136610a1eb9de56aceaccd70c963a26a260af20ac24794f" -dependencies = [ - "base64-simd", - "itoa", - "num-integer", - "ryu", - "time 0.3.20", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.54.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343ffe9a9bb3f542675f4df0e0d5933513d6ad038ca3907ad1767ba690a99684" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "0.54.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", - "http", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.6.20" @@ -875,16 +593,6 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.6.0" @@ -1021,16 +729,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" -[[package]] -name = "bytes-utils" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "cc" version = "1.0.79" @@ -1475,11 +1173,18 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.8.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbde004c8489393679b22f316a1d09f14f1f08ccbbdc26aa2420318bf7bad36" +checksum = "39fbcd162d595e3b7e7af762b05abbdb14218615d24e8c40d23b1cfe1a408589" dependencies = [ "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-trait", "bytes", "cfg-if 1.0.0", @@ -1487,13 +1192,13 @@ dependencies = [ "dynamodb_lock", "errno 0.3.0", "futures", - "glibc_version", - "itertools", + "itertools 0.11.0", "lazy_static", "libc", "log", "num-bigint", "num-traits", + "num_cpus", "object_store", "once_cell", "parking_lot", @@ -1945,15 +1650,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "glibc_version" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803ff7635f1ab4e2c064b68a0c60da917d3d18dc8d086130f689d62ce4f1c33e" -dependencies = [ - "regex", -] - [[package]] name = "glob" version = "0.3.1" @@ -2010,12 +1706,6 @@ dependencies = [ "ahash 0.7.6", ] -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" - [[package]] name = "hashbrown" version = "0.14.0" @@ -2172,6 +1862,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.25" @@ -2207,7 +1903,7 @@ dependencies = [ "hyper", "log", "rustls 0.19.1", - "rustls-native-certs 0.5.0", + "rustls-native-certs", "tokio", "tokio-rustls 0.22.0", "webpki 0.21.4", @@ -2221,9 +1917,7 @@ checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", - "log", "rustls 0.20.8", - "rustls-native-certs 0.6.2", "tokio", "tokio-rustls 0.23.4", ] @@ -2371,6 +2065,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -2728,9 +2431,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" dependencies = [ "num-bigint", "num-complex", @@ -2841,19 +2544,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ea8f683b4f89a64181393742c041520a1a87e9775e6b4c0dd5a3281af05fc6" +checksum = "27c776db4f332b571958444982ff641d2531417a326ca368995073b639205d58" dependencies = [ "async-trait", - "aws-config", - "aws-credential-types", - "aws-types", "base64 0.21.0", "bytes", "chrono", "futures", - "itertools", + "humantime", + "hyper", + "itertools 0.10.5", "parking_lot", "percent-encoding", "quick-xml", @@ -2952,12 +2654,6 @@ version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" -[[package]] -name = "outref" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" - [[package]] name = "overload" version = "0.1.1" @@ -2995,9 +2691,9 @@ dependencies = [ [[package]] name = "parquet" -version = "33.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b076829801167d889795cd1957989055543430fa1469cb1f6e32b789bfc764" +checksum = "49f9739b984380582bdb7749ae5b5d28839bce899212cf16465c1ac1f8b65d79" dependencies = [ "ahash 0.8.3", "arrow-array", @@ -3013,7 +2709,7 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "lz4", "num", "num-bigint", @@ -3240,9 +2936,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.27.1" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" dependencies = [ "memchr", "serde", @@ -3315,7 +3011,7 @@ checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.6.28", ] [[package]] @@ -3324,7 +3020,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.28", ] [[package]] @@ -3333,6 +3029,12 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "reqwest" version = "0.11.16" @@ -3724,18 +3426,6 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -4147,7 +3837,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ - "itertools", + "itertools 0.10.5", "nom", "unicode_categories", ] @@ -4928,12 +4618,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" - [[package]] name = "utoipa" version = "3.1.2" @@ -5054,12 +4738,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - [[package]] name = "waker-fn" version = "1.1.0" @@ -5424,12 +5102,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" -[[package]] -name = "xmlparser" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" - [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index bc41127..b082e13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ async-trait = "0.1.64" axum = { version = "0.6.20", features = ["headers"] } axum-extra = { version = "0.8", features = ["json-lines"] } clap = "4.1.4" -deltalake = { version = "0.8.0", features = ["s3", "azure", "gcs"] } +deltalake = { version = "0.15.0", features = ["s3", "azure", "gcs"] } futures = "0.3.28" futures-util = "0.3.28" tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] } diff --git a/Justfile b/Justfile index 9193cce..40ea6a7 100644 --- a/Justfile +++ b/Justfile @@ -19,13 +19,15 @@ build: # Conduct unit tests test: - @cargo test -- --nocapture + @cargo test --lib -- --nocapture # Conduct DB-related unit tests -testdb: +test-integration: @# Be sure run the following command before conducting this: @# $ docker compose -f ./devops/local/docker-compose.yaml up - @cargo test -- --nocapture --ignored + @cargo test --tests -- --nocapture + +alias testdb := test-integration # Run local docker emvironment docker: @@ -37,4 +39,4 @@ server: # Build delta-sharing-rs into a docker image for local use package: - DOCKER_BUILDKIT=0 docker build . -t delta-sharing:local -f devops/docker/Dockerfile \ No newline at end of file + DOCKER_BUILDKIT=0 docker build . -t delta-sharing:local -f devops/docker/Dockerfile diff --git a/src/server.rs b/src/server.rs index 9a53929..d9d8335 100644 --- a/src/server.rs +++ b/src/server.rs @@ -15,6 +15,22 @@ use tame_gcs::signing::ServiceAccount; use crate::bootstrap; +pub use crate::server::middlewares::jwt::Role; +pub use entities::account::{Entity as AccountEntity, Id as AccountId}; +pub use entities::schema::{Entity as SchemaEntity, Id as SchemaId}; +pub use entities::share::{Entity as ShareEntity, Id as ShareId}; +pub use entities::table::{Entity as TableEntity, Id as TableId}; +pub use entities::token::{Entity as TokenEntity, Id as TokenId}; +pub use repositories::account::Repository as AccountRepository; +pub use repositories::schema::Repository as SchemaRepository; +pub use repositories::share::Repository as ShareRepository; +pub use repositories::table::Repository as TableRepository; +pub use repositories::token::Repository as TokenRepository; +pub use services::account::Service as AccountService; +pub use services::schema::Service as SchemaService; +pub use services::share::Service as ShareService; +pub use services::table::Service as TableService; + pub struct Server { pg_pool: PgPool, gcp_service_account: Option, diff --git a/src/server/entities/table.rs b/src/server/entities/table.rs index 42861e9..3c7d1d7 100644 --- a/src/server/entities/table.rs +++ b/src/server/entities/table.rs @@ -64,8 +64,8 @@ impl Entity { }) } - pub async fn load(name: &Name, pg_pool: &PgPool) -> Result> { - match Repository::select_by_name(name, pg_pool).await? { + pub async fn load(schema_id: &SchemaId, name: &Name, pg_pool: &PgPool) -> Result> { + match Repository::select_by_name(schema_id, name, pg_pool).await? { Some(row) => Ok(Self { id: Id::new(row.id), name: Name::new(row.name)?, diff --git a/src/server/repositories/account.rs b/src/server/repositories/account.rs index c49d317..cf0fbe7 100644 --- a/src/server/repositories/account.rs +++ b/src/server/repositories/account.rs @@ -88,57 +88,3 @@ impl Repository { Ok(row) } } - -#[cfg(test)] -mod tests { - use super::*; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - - async fn create(tx: &mut PgConnection) -> Result { - let account = Entity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - Repository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_select_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create(&mut tx) - .await - .expect("new account should be created"); - let fetched = Repository::select_by_name(account.name(), &mut tx) - .await - .expect("created account should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, account.id().as_uuid()); - assert_eq!(&fetched.name, account.name().as_str()); - assert_eq!(&fetched.email, account.email().as_str()); - assert_eq!(&fetched.password, account.password().as_str()); - assert_eq!(&fetched.namespace, account.namespace().as_str()); - assert_eq!(&fetched.ttl, account.ttl().as_i64()); - } else { - panic!("created account should be matched"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/repositories/schema.rs b/src/server/repositories/schema.rs index 3e23831..3aa33f4 100644 --- a/src/server/repositories/schema.rs +++ b/src/server/repositories/schema.rs @@ -85,90 +85,3 @@ impl Repository { Ok(row) } } - -#[cfg(test)] -mod tests { - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - - use super::*; - use crate::server::entities::account::Entity as Account; - use crate::server::entities::account::Id as AccountId; - use crate::server::entities::share::Entity as Share; - use crate::server::repositories::account::Repository as AccountRepository; - use crate::server::repositories::share::Repository as ShareRepository; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = Account::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = Share::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - ShareRepository::upsert(&share, tx) - .await - .context("failed to create share")?; - Ok(share) - } - - async fn create_schema( - account_id: &AccountId, - share_id: &ShareId, - tx: &mut PgConnection, - ) -> Result { - let schema = Entity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - share_id.to_uuid().to_string(), - account_id.to_uuid().to_string(), - ) - .context("failed to validate schema")?; - Repository::upsert(&schema, tx) - .await - .context("failed to create schema")?; - Ok(schema) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_schema(account.id(), share.id(), &mut tx) - .await - .expect("new schema should be created"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/repositories/share.rs b/src/server/repositories/share.rs index 1e9e741..4bd881c 100644 --- a/src/server/repositories/share.rs +++ b/src/server/repositories/share.rs @@ -73,73 +73,3 @@ impl Repository { Ok(row) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::server::entities::account::Entity as Account; - use crate::server::entities::account::Id as AccountId; - use crate::server::repositories::account::Repository as AccountRepository; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = Account::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = Entity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - Repository::upsert(&share, tx) - .await - .context("failed to create share")?; - Ok(share) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_select_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let fetched = Repository::select_by_name(share.name(), &mut tx) - .await - .expect("created share should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, share.id().as_uuid()); - assert_eq!(&fetched.name, share.name().as_str()); - assert_eq!(&fetched.created_by, share.created_by().as_uuid()); - } else { - panic!("created share should be matched"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/repositories/table.rs b/src/server/repositories/table.rs index ff06257..4f7dac2 100644 --- a/src/server/repositories/table.rs +++ b/src/server/repositories/table.rs @@ -5,6 +5,7 @@ use chrono::Utc; use sqlx::postgres::PgQueryResult; use uuid::Uuid; +use crate::server::entities::schema::Id as SchemaId; use crate::server::entities::table::Entity; use crate::server::entities::table::Name; use crate::server::utilities::postgres::PgAcquire; @@ -45,17 +46,23 @@ impl Repository { ) .bind(table.id()) .bind(table.name()) + .bind(table.schema_id()) .bind(table.location()) .bind(table.created_by()) .execute(&mut *conn) .await .context(format!( - r#"failed to upsert "{}" into [table]"#, + r#"failed to upsert "{} ({})" into [table]"#, + table.name().to_string(), table.id().as_uuid() )) } - pub async fn select_by_name(name: &Name, executor: impl PgAcquire<'_>) -> Result> { + pub async fn select_by_name( + schema_id: &SchemaId, + name: &Name, + executor: impl PgAcquire<'_>, + ) -> Result> { let mut conn = executor .acquire() .await @@ -70,8 +77,9 @@ impl Repository { created_at, updated_at FROM "table" - WHERE name = $1"#, + WHERE schema_id = $1 AND name = $2"#, ) + .bind(schema_id) .bind(name) .fetch_optional(&mut *conn) .await @@ -82,124 +90,3 @@ impl Repository { Ok(row) } } - -#[cfg(test)] -mod tests { - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - - use super::*; - use crate::server::entities::account::Entity as Account; - use crate::server::entities::account::Id as AccountId; - use crate::server::entities::schema::Entity as Schema; - use crate::server::entities::schema::Id as SchemaId; - use crate::server::entities::share::Entity as Share; - use crate::server::entities::share::Id as ShareId; - use crate::server::repositories::account::Repository as AccountRepository; - use crate::server::repositories::schema::Repository as SchemaRepository; - use crate::server::repositories::share::Repository as ShareRepository; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = Account::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = Share::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - ShareRepository::upsert(&share, tx) - .await - .context("failed to create share")?; - Ok(share) - } - - async fn create_schema( - account_id: &AccountId, - share_id: &ShareId, - tx: &mut PgConnection, - ) -> Result { - let schema = Schema::new( - testutils::rand::uuid(), - testutils::rand::string(10), - share_id.to_uuid().to_string(), - account_id.to_uuid().to_string(), - ) - .context("failed to validate schema")?; - SchemaRepository::upsert(&schema, tx) - .await - .context("failed to create schema")?; - Ok(schema) - } - - async fn create_table( - account_id: &AccountId, - schema_id: &SchemaId, - tx: &mut PgConnection, - ) -> Result { - let table = Entity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - schema_id.to_uuid().to_string(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate table")?; - Repository::upsert(&table, tx) - .await - .context("failed to create table")?; - Ok(table) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_select_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema = create_schema(account.id(), share.id(), &mut tx) - .await - .expect("new share should be created"); - let table = create_table(account.id(), schema.id(), &mut tx) - .await - .expect("new table should be created"); - let fetched = Repository::select_by_name(table.name(), &mut tx) - .await - .expect("created table should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, table.id().as_uuid()); - assert_eq!(&fetched.name, table.name().as_str()); - assert_eq!(&fetched.location, table.location().as_str()); - assert_eq!(&fetched.created_by, table.created_by().as_uuid()); - } else { - panic!("created table should be matched"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/repositories/token.rs b/src/server/repositories/token.rs index dbab834..0d73a07 100644 --- a/src/server/repositories/token.rs +++ b/src/server/repositories/token.rs @@ -56,69 +56,3 @@ impl Repository { )) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::server::entities::account::Entity as Account; - use crate::server::entities::account::Id as AccountId; - use crate::server::repositories::account::Repository as AccountRepository; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - use std::str::FromStr; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = Account::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_token(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let roles = vec!["Admin", "Guest"]; - let role = testutils::rand::choose(&roles); - let role = Role::from_str(role).context("failed to choose role")?; - let token = Entity::new( - testutils::rand::uuid(), - testutils::rand::email(), - role, - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate token")?; - Repository::upsert(&token, tx) - .await - .context("failed to create token")?; - Ok(token) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - create_token(account.id(), &mut tx) - .await - .expect("new token should be created"); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/routers/shares/schemas/tables/query.rs b/src/server/routers/shares/schemas/tables/query.rs index 3e6e1da..38d959d 100644 --- a/src/server/routers/shares/schemas/tables/query.rs +++ b/src/server/routers/shares/schemas/tables/query.rs @@ -77,7 +77,7 @@ pub async fn post( .into_iter() .map(|p| SQLUtility::parse(p.to_owned())) .collect(); - if let Err(_) = predicate_hints { + if predicate_hints.is_err() { tracing::warn!("requested predicate hints are malformed"); } predicate_hints.ok() @@ -86,7 +86,7 @@ pub async fn post( }; let json_predicate_hints = if let Some(json_predicate_hints) = payload.json_predicate_hints { let predicate = JSONUtility::parse(json_predicate_hints); - if let Err(_) = predicate { + if predicate.is_err() { tracing::warn!("requested predicate hints are malformed"); } predicate.ok() @@ -162,7 +162,7 @@ pub async fn post( metadata.to_owned() }; let url_signer = |name: String| match &platform { - Platform::AWS { url, bucket, path } => { + Platform::Aws { url, bucket, path } => { if let Some(aws_credentials) = &state.aws_credentials { let file: String = format!("{}/{}", path, name); let Ok(signed) = SignedUrlUtility::sign_aws( @@ -179,7 +179,7 @@ pub async fn post( tracing::warn!("AWS credentials were not set"); url.clone() } - Platform::GCP { url, bucket, path } => { + Platform::Gcp { url, bucket, path } => { if let Some(gcp_service_account) = &state.gcp_service_account { let file: String = format!("{}/{}", path, name); let Ok(signed) = SignedUrlUtility::sign_gcp( @@ -196,7 +196,7 @@ pub async fn post( tracing::warn!("GCP service account was not set"); url.clone() } - Platform::NONE { url } => { + Platform::None { url } => { tracing::warn!("no supported platforms"); url.clone() } diff --git a/src/server/services/account.rs b/src/server/services/account.rs index e482a45..53898ca 100644 --- a/src/server/services/account.rs +++ b/src/server/services/account.rs @@ -98,105 +98,3 @@ impl Service { Ok(row) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::server::entities::account::Entity as AccountEntity; - use crate::server::repositories::account::Repository as AccountRepository; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - use std::cmp::min; - - async fn create(tx: &mut PgConnection) -> Result { - let account = AccountEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create(&mut tx) - .await - .expect("new account should be created"); - } - let fetched = Service::query(None, None, &mut tx) - .await - .expect("created account should be listed"); - assert_eq!(records as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create(&mut tx) - .await - .expect("new account should be created"); - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query(Some(&limit), None, &mut tx) - .await - .expect("created account should be listed"); - assert_eq!(min(records, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create(&mut tx) - .await - .expect("new account should be created"); - let fetched = Service::query_by_name(account.name(), &mut tx) - .await - .expect("created account should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.name, account.name().as_str()); - assert_eq!(&fetched.email, account.email().as_str()); - assert_eq!(&fetched.namespace, account.namespace().as_str()); - assert_eq!(&fetched.ttl, account.ttl().as_i64()); - } else { - panic!("created account should be found"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/services/schema.rs b/src/server/services/schema.rs index cb3e167..a3281c5 100644 --- a/src/server/services/schema.rs +++ b/src/server/services/schema.rs @@ -86,137 +86,3 @@ impl Service { Ok(rows) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::server::entities::account::Entity as AccountEntity; - use crate::server::entities::account::Id as AccountId; - use crate::server::entities::schema::Entity as SchemaEntity; - use crate::server::entities::schema::Name as SchemaName; - use crate::server::entities::share::Entity as ShareEntity; - use crate::server::entities::share::Id as ShareId; - use crate::server::repositories::account::Repository as AccountRepository; - use crate::server::repositories::schema::Repository as SchemaRepository; - use crate::server::repositories::share::Repository as ShareRepository; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - use std::cmp::min; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = AccountEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = ShareEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - ShareRepository::upsert(&share, tx) - .await - .context("failed to crate share")?; - Ok(share) - } - - async fn create_schema( - schema_name: &SchemaName, - share_id: &ShareId, - account_id: &AccountId, - tx: &mut PgConnection, - ) -> Result { - let schema = SchemaEntity::new( - testutils::rand::uuid(), - schema_name.to_string(), - share_id.to_uuid().to_string(), - account_id.to_uuid().to_string(), - ) - .context("failed to validate schema")?; - SchemaRepository::upsert(&schema, tx) - .await - .context("failed to crate schema")?; - Ok(schema) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..testutils::rand::i64(1, 20) { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let fetched = Service::query_by_share_name(share.name(), None, None, &mut tx) - .await - .expect("created schema should be listed"); - assert_eq!(records as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..testutils::rand::i64(1, 20) { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query_by_share_name(share.name(), Some(&limit), None, &mut tx) - .await - .expect("created schema should be listed"); - assert_eq!(min(records, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/services/share.rs b/src/server/services/share.rs index d9cb1b4..f142cd3 100644 --- a/src/server/services/share.rs +++ b/src/server/services/share.rs @@ -90,128 +90,3 @@ impl Service { Ok(row) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::server::entities::account::Entity as AccountEntity; - use crate::server::entities::account::Id as AccountId; - use crate::server::entities::share::Entity as ShareEntity; - use crate::server::repositories::account::Repository as AccountRepository; - use crate::server::repositories::share::Repository as ShareRepository; - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - use std::cmp::min; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = AccountEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = ShareEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - ShareRepository::upsert(&share, tx) - .await - .context("failed to crate share")?; - Ok(share) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - } - let fetched = Service::query(None, None, &mut tx) - .await - .expect("created share should be listed"); - assert_eq!(records as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query(Some(&limit), None, &mut tx) - .await - .expect("created share should be listed"); - assert_eq!(min(records, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let fetched = Service::query_by_name(share.name(), &mut tx) - .await - .expect("created share should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, share.id().as_uuid().to_string().as_str()); - assert_eq!(&fetched.name, share.name().as_str()); - } else { - panic!("created account should be found"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/services/table.rs b/src/server/services/table.rs index 8fbaa69..1180e23 100644 --- a/src/server/services/table.rs +++ b/src/server/services/table.rs @@ -31,8 +31,6 @@ impl Table { #[derive(Debug, Clone, serde::Serialize, sqlx::FromRow, ToSchema)] #[serde(rename_all = "camelCase")] pub struct TableDetail { - pub id: String, - pub share_id: String, pub name: String, pub schema: String, pub share: String, @@ -122,7 +120,7 @@ impl Service { "table".name AS name, "table".location AS location FROM "table" - LEFT JOIN "schema" ON "schema".table_id = "table".id + LEFT JOIN "schema" ON "schema".id = "table".schema_id LEFT JOIN share ON share.id = "schema".share_id WHERE share.name = $1 AND "schema".name = $2 AND "table".name = $3"#, ) @@ -159,7 +157,7 @@ impl Service { "schema".name AS schema, share.name AS share FROM "table" - LEFT JOIN "schema" ON "schema".table_id = "table".id + LEFT JOIN "schema" ON "schema".id = "table".schema_id LEFT JOIN share ON share.id = "schema".share_id WHERE share.name = "#, ); @@ -168,8 +166,6 @@ impl Service { " ) SELECT - id::text, - share_id::text, name, schema, share @@ -219,7 +215,7 @@ impl Service { "schema".name AS schema, share.name AS share FROM "table" - LEFT JOIN "schema" ON "schema".table_id = "table".id + LEFT JOIN "schema" ON "schema".id = "table".schema_id LEFT JOIN share ON share.id = "schema".share_id WHERE share.name = "#, ); @@ -262,401 +258,3 @@ impl Service { Ok(rows) } } - -#[cfg(test)] -mod tests { - use anyhow::Context; - use anyhow::Result; - use sqlx::PgConnection; - use sqlx::PgPool; - use std::cmp::min; - - use super::*; - use crate::server::entities::account::Entity as AccountEntity; - use crate::server::entities::account::Id as AccountId; - use crate::server::entities::schema::Entity as SchemaEntity; - use crate::server::entities::schema::Id as SchemaId; - use crate::server::entities::schema::Name as SchemaName; - use crate::server::entities::share::Entity as ShareEntity; - use crate::server::entities::share::Id as ShareId; - use crate::server::entities::table::Entity as TableEntity; - use crate::server::repositories::account::Repository as AccountRepository; - use crate::server::repositories::schema::Repository as SchemaRepository; - use crate::server::repositories::share::Repository as ShareRepository; - use crate::server::repositories::table::Repository as TableRepository; - - async fn create_account(tx: &mut PgConnection) -> Result { - let account = AccountEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - testutils::rand::email(), - testutils::rand::string(10), - testutils::rand::string(10), - testutils::rand::i64(1, 100000), - ) - .context("failed to validate account")?; - AccountRepository::upsert(&account, tx) - .await - .context("failed to create account")?; - Ok(account) - } - - async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { - let share = ShareEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate share")?; - ShareRepository::upsert(&share, tx) - .await - .context("failed to crate share")?; - Ok(share) - } - - async fn create_schema( - schema_name: &SchemaName, - share_id: &ShareId, - account_id: &AccountId, - tx: &mut PgConnection, - ) -> Result { - let schema = SchemaEntity::new( - testutils::rand::uuid(), - schema_name.to_string(), - share_id.to_uuid().to_string(), - account_id.to_uuid().to_string(), - ) - .context("failed to validate schema")?; - SchemaRepository::upsert(&schema, tx) - .await - .context("failed to crate schema")?; - Ok(schema) - } - - async fn create_table( - account_id: &AccountId, - schema_id: &SchemaId, - tx: &mut PgConnection, - ) -> Result { - let table = TableEntity::new( - testutils::rand::uuid(), - testutils::rand::string(10), - schema_id.to_uuid().to_string(), - testutils::rand::string(10), - account_id.to_uuid().to_string(), - ) - .context("failed to validate table")?; - TableRepository::upsert(&table, tx) - .await - .context("failed to crate table")?; - Ok(table) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let schema = create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_table(account.id(), schema.id(), &mut tx) - .await - .expect("new table should be created"); - } - let fetched = Service::query(None, None, &mut tx) - .await - .expect("created table should be listed"); - assert_eq!(records as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let schema = create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_table(account.id(), schema.id(), &mut tx) - .await - .expect("new table should be created"); - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query(Some(&limit), None, &mut tx) - .await - .expect("created table should be listed"); - assert_eq!(min(records, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_name(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let schema = create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - let table = create_table(account.id(), schema.id(), &mut tx) - .await - .expect("new table should be created"); - let fetched = Service::query_by_name(table.name(), &mut tx) - .await - .expect("created table should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, table.id().as_uuid().to_string().as_str()); - assert_eq!(&fetched.name, table.name().as_str()); - } else { - panic!("created table should be found"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_fqn(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let schema = create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - let table = create_table(account.id(), schema.id(), &mut tx) - .await - .expect("new table should be created"); - - let fetched = Service::query_by_fqn(share.name(), &schema_name, table.name(), &mut tx) - .await - .expect("created table should be found"); - if let Some(fetched) = fetched { - assert_eq!(&fetched.id, table.id().as_uuid().to_string().as_str()); - assert_eq!(&fetched.name, table.name().as_str()); - } else { - panic!("created table should be found"); - } - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_share_name_with_default_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let num_schemas = testutils::rand::i64(0, 20); - let num_tables = testutils::rand::i64(0, 20); - for _ in 0..num_schemas { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..num_tables { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let fetched = Service::query_by_share_name(share.name(), None, None, &mut tx) - .await - .expect("created table should be listed"); - assert_eq!((num_schemas * num_tables) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_share_name_with_specified_limit(pool: PgPool) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let num_schemas = testutils::rand::i64(0, 20); - let num_tables = testutils::rand::i64(0, 20); - for _ in 0..num_schemas { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..num_tables { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query_by_share_name(share.name(), Some(&limit), None, &mut tx) - .await - .expect("created schema should be listed"); - assert_eq!(min(num_schemas * num_tables, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_share_and_schema_name_with_default_limit( - pool: PgPool, - ) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - for _ in 0..testutils::rand::i64(0, 20) { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..records { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let fetched = Service::query_by_share_and_schema_name( - share.name(), - &schema_name, - None, - None, - &mut tx, - ) - .await - .expect("created table should be listed"); - assert_eq!(records as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } - - #[sqlx::test] - #[ignore] // NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' before running this test - async fn test_create_and_query_by_share_and_schema_name_with_specified_limit( - pool: PgPool, - ) -> Result<()> { - let mut tx = pool - .begin() - .await - .expect("transaction should be started properly"); - let account = create_account(&mut tx) - .await - .expect("new account should be created"); - let share = create_share(account.id(), &mut tx) - .await - .expect("new share should be created"); - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - let records = testutils::rand::i64(0, 20); - for _ in 0..records { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - for _ in 0..testutils::rand::i64(0, 20) { - let schema_name = SchemaName::new(testutils::rand::string(10)) - .expect("new schema name should be created"); - for _ in 0..records { - create_schema(&schema_name, share.id(), account.id(), &mut tx) - .await - .expect("new schema should be created"); - } - } - let limit = testutils::rand::i64(0, 20); - let fetched = Service::query_by_share_and_schema_name( - share.name(), - &schema_name, - Some(&limit), - None, - &mut tx, - ) - .await - .expect("created schema should be listed"); - assert_eq!(min(records, limit) as usize, fetched.len()); - tx.rollback() - .await - .expect("rollback should be done properly"); - Ok(()) - } -} diff --git a/src/server/utilities/signed_url.rs b/src/server/utilities/signed_url.rs index f757362..622783f 100644 --- a/src/server/utilities/signed_url.rs +++ b/src/server/utilities/signed_url.rs @@ -17,17 +17,17 @@ use url::Url; #[derive(Debug, PartialEq, Eq)] pub enum Platform { - AWS { + Aws { url: String, bucket: String, path: String, }, - GCP { + Gcp { url: String, bucket: String, path: String, }, - NONE { + None { url: String, }, } @@ -38,22 +38,22 @@ impl FromStr for Platform { fn from_str(input: &str) -> std::result::Result { let url = Url::parse(input).context("failed to parse URL")?; match url.scheme() { - "s3" => Ok(Self::AWS { + "s3" => Ok(Self::Aws { url: String::from(url.as_str()), bucket: String::from(url.domain().unwrap_or("")), path: String::from(url.path().strip_prefix('/').unwrap_or("")), }), - "s3a" => Ok(Self::AWS { + "s3a" => Ok(Self::Aws { url: String::from(url.as_str()), bucket: String::from(url.domain().unwrap_or("")), path: String::from(url.path().strip_prefix('/').unwrap_or("")), }), - "gs" => Ok(Self::GCP { + "gs" => Ok(Self::Gcp { url: String::from(url.as_str()), bucket: String::from(url.domain().unwrap_or("")), path: String::from(url.path().strip_prefix('/').unwrap_or("")), }), - _ => Ok(Self::NONE { + _ => Ok(Self::None { url: String::from(url.as_str()), }), } @@ -106,7 +106,7 @@ mod tests { let path = testutils::rand::string(10); let url = format!("s3://{}/{}", bucket, path); let provider = Platform::from_str(&url).expect("should parse s3 url properly"); - if let Platform::AWS { + if let Platform::Aws { url: parsed_url, bucket: parsed_bucket, path: parsed_path, @@ -126,7 +126,7 @@ mod tests { let path = testutils::rand::string(10); let url = format!("gs://{}/{}", bucket, path); let provider = Platform::from_str(&url).expect("should parse gcs url properly"); - if let Platform::GCP { + if let Platform::Gcp { url: parsed_url, bucket: parsed_bucket, path: parsed_path, @@ -149,7 +149,7 @@ mod tests { .credentials() .await .expect("AWS credentials should be acquired properly"); - if let Ok(Platform::AWS { bucket, path, .. }) = + if let Ok(Platform::Aws { bucket, path, .. }) = Platform::from_str("s3://delta-sharing-test/covid") { if let Ok(url) = Utility::sign_aws(&creds, &bucket, &path, &300) { @@ -173,7 +173,7 @@ mod tests { ); let sa = bootstrap::gcp::new(&path).expect("GCP service account should be created properly"); - if let Ok(Platform::GCP { bucket, path, .. }) = + if let Ok(Platform::Gcp { bucket, path, .. }) = Platform::from_str("gs://delta-sharing-test/covid") { if let Ok(url) = Utility::sign_gcp(&sa, &bucket, &path, &300) { diff --git a/tests/common.rs b/tests/common.rs new file mode 100644 index 0000000..b6c54a8 --- /dev/null +++ b/tests/common.rs @@ -0,0 +1,96 @@ +use std::str::FromStr; + +use anyhow::Context; +use anyhow::Result; +use sqlx::PgConnection; + +use delta_sharing::server::Role; +use delta_sharing::server::{AccountEntity, AccountId, AccountRepository}; +use delta_sharing::server::{SchemaEntity, SchemaId, SchemaRepository}; +use delta_sharing::server::{ShareEntity, ShareId, ShareRepository}; +use delta_sharing::server::{TableEntity, TableRepository}; +use delta_sharing::server::{TokenEntity, TokenRepository}; + +pub async fn create_account(tx: &mut PgConnection) -> Result { + let account = AccountEntity::new( + testutils::rand::uuid(), + testutils::rand::string(10), + testutils::rand::email(), + testutils::rand::string(10), + testutils::rand::string(10), + testutils::rand::i64(1, 100000), + ) + .context("failed to validate account")?; + AccountRepository::upsert(&account, tx) + .await + .context("failed to create account")?; + Ok(account) +} + +pub async fn create_token(account_id: &AccountId, tx: &mut PgConnection) -> Result { + let roles = vec!["Admin", "Guest"]; + let role = testutils::rand::choose(&roles); + let role = Role::from_str(role).context("failed to choose role")?; + let token = TokenEntity::new( + testutils::rand::uuid(), + testutils::rand::email(), + role, + testutils::rand::string(10), + account_id.to_uuid().to_string(), + ) + .context("failed to validate token")?; + TokenRepository::upsert(&token, tx) + .await + .context("failed to create token")?; + Ok(token) +} + +pub async fn create_share(account_id: &AccountId, tx: &mut PgConnection) -> Result { + let share = ShareEntity::new( + testutils::rand::uuid(), + testutils::rand::string(10), + account_id.to_uuid().to_string(), + ) + .context("failed to validate share")?; + ShareRepository::upsert(&share, tx) + .await + .context("failed to create share")?; + Ok(share) +} + +pub async fn create_schema( + account_id: &AccountId, + share_id: &ShareId, + tx: &mut PgConnection, +) -> Result { + let schema = SchemaEntity::new( + testutils::rand::uuid(), + testutils::rand::string(10), + share_id.to_uuid().to_string(), + account_id.to_uuid().to_string(), + ) + .context("failed to validate schema")?; + SchemaRepository::upsert(&schema, tx) + .await + .context("failed to create schema")?; + Ok(schema) +} + +pub async fn create_table( + account_id: &AccountId, + schema_id: &SchemaId, + tx: &mut PgConnection, +) -> Result { + let table = TableEntity::new( + testutils::rand::uuid(), + testutils::rand::string(10), + schema_id.to_uuid().to_string(), + testutils::rand::string(10), + account_id.to_uuid().to_string(), + ) + .context("failed to validate table")?; + TableRepository::upsert(&table, tx) + .await + .context("failed to create table")?; + Ok(table) +} diff --git a/tests/repositories.rs b/tests/repositories.rs new file mode 100644 index 0000000..80d0913 --- /dev/null +++ b/tests/repositories.rs @@ -0,0 +1,154 @@ +// NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' +// before running these tests +mod common; + +use anyhow::Result; +use sqlx::PgPool; + +use delta_sharing::server::AccountRepository; +use delta_sharing::server::SchemaRepository; +use delta_sharing::server::ShareRepository; +use delta_sharing::server::TableRepository; + +use common::{create_account, create_schema, create_share, create_table, create_token}; + +#[sqlx::test] +async fn test_account_create_and_select_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let fetched = AccountRepository::select_by_name(account.name(), &mut tx) + .await + .expect("created account should be found"); + assert!(fetched.is_some()); + + let fetched = fetched.unwrap(); + assert_eq!(&fetched.id, account.id().as_uuid()); + assert_eq!(&fetched.name, account.name().as_str()); + assert_eq!(&fetched.email, account.email().as_str()); + assert_eq!(&fetched.password, account.password().as_str()); + assert_eq!(&fetched.namespace, account.namespace().as_str()); + assert_eq!(&fetched.ttl, account.ttl().as_i64()); + + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_token_create(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + create_token(account.id(), &mut tx) + .await + .expect("new token should be created"); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_share_create_and_select_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let fetched = ShareRepository::select_by_name(share.name(), &mut tx) + .await + .expect("created share should be found"); + assert!(fetched.is_some()); + + let fetched = fetched.unwrap(); + assert_eq!(&fetched.id, share.id().as_uuid()); + assert_eq!(&fetched.name, share.name().as_str()); + assert_eq!(&fetched.created_by, share.created_by().as_uuid()); + + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_schema_create_and_select_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + + let fetched = SchemaRepository::select_by_name(share.id(), schema.name(), &mut tx) + .await + .expect("created share should be found"); + assert!(fetched.is_some()); + + let fetched = fetched.unwrap(); + assert_eq!(&fetched.id, schema.id().as_uuid()); + assert_eq!(&fetched.name, schema.name().as_str()); + assert_eq!(&fetched.created_by, schema.created_by().as_uuid()); + + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_select_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new share should be created"); + let table = create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + let fetched = TableRepository::select_by_name(schema.id(), table.name(), &mut tx) + .await + .expect("created table should be found"); + assert!(fetched.is_some()); + + let fetched = fetched.unwrap(); + assert_eq!(&fetched.id, table.id().as_uuid()); + assert_eq!(&fetched.name, table.name().as_str()); + assert_eq!(&fetched.location, table.location().as_str()); + assert_eq!(&fetched.created_by, table.created_by().as_uuid()); + + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} diff --git a/tests/services.rs b/tests/services.rs new file mode 100644 index 0000000..30f18a7 --- /dev/null +++ b/tests/services.rs @@ -0,0 +1,500 @@ +// NOTE: Be sure '$ docker compose -f devops/local/docker-compose.yaml up' +// before running these tests +mod common; + +use std::cmp::min; + +use anyhow::Result; +use sqlx::PgPool; + +use delta_sharing::server::AccountService; +use delta_sharing::server::SchemaService; +use delta_sharing::server::ShareService; +use delta_sharing::server::TableService; + +use common::{create_account, create_schema, create_share, create_table}; + +#[sqlx::test] +async fn test_account_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_account(&mut tx) + .await + .expect("new account should be created"); + } + let fetched = AccountService::query(None, None, &mut tx) + .await + .expect("created account should be listed"); + assert_eq!(records as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_account_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_account(&mut tx) + .await + .expect("new account should be created"); + } + let limit = testutils::rand::i64(0, 20); + let fetched = AccountService::query(Some(&limit), None, &mut tx) + .await + .expect("created account should be listed"); + assert_eq!(min(records, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_account_create_and_query_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let fetched = AccountService::query_by_name(account.name(), &mut tx) + .await + .expect("created account should be found"); + if let Some(fetched) = fetched { + assert_eq!(&fetched.name, account.name().as_str()); + assert_eq!(&fetched.email, account.email().as_str()); + assert_eq!(&fetched.namespace, account.namespace().as_str()); + assert_eq!(&fetched.ttl, account.ttl().as_i64()); + } else { + panic!("created account should be found"); + } + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_share_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + } + let fetched = ShareService::query(None, None, &mut tx) + .await + .expect("created share should be listed"); + assert_eq!(records as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_share_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + } + let limit = testutils::rand::i64(0, 20); + let fetched = ShareService::query(Some(&limit), None, &mut tx) + .await + .expect("created share should be listed"); + assert_eq!(min(records, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_share_create_and_query_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let fetched = ShareService::query_by_name(share.name(), &mut tx) + .await + .expect("created share should be found"); + if let Some(fetched) = fetched { + assert_eq!(&fetched.id, share.id().as_uuid().to_string().as_str()); + assert_eq!(&fetched.name, share.name().as_str()); + } else { + panic!("created account should be found"); + } + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_schema_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + } + let fetched = SchemaService::query_by_share_name(share.name(), None, None, &mut tx) + .await + .expect("created schema should be listed"); + assert_eq!(records as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_schema_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + } + let limit = testutils::rand::i64(0, 20); + let fetched = SchemaService::query_by_share_name(share.name(), Some(&limit), None, &mut tx) + .await + .expect("created schema should be listed"); + assert_eq!(min(records, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_with_default_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + } + let fetched = TableService::query(None, None, &mut tx) + .await + .expect("created table should be listed"); + assert_eq!(records as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_with_specified_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + } + let limit = testutils::rand::i64(0, 20); + let fetched = TableService::query(Some(&limit), None, &mut tx) + .await + .expect("created table should be listed"); + assert_eq!(min(records, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_name(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let table = create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + let fetched = TableService::query_by_name(table.name(), &mut tx) + .await + .expect("created table should be found"); + if let Some(fetched) = fetched { + assert_eq!(&fetched.id, table.id().as_uuid().to_string().as_str()); + assert_eq!(&fetched.name, table.name().as_str()); + } else { + panic!("created table should be found"); + } + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_fqn(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let table = create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + let fetched = TableService::query_by_fqn(share.name(), schema.name(), table.name(), &mut tx) + .await + .expect("created table should be found"); + if let Some(fetched) = fetched { + assert_eq!(&fetched.id, table.id().as_uuid().to_string().as_str()); + assert_eq!(&fetched.name, table.name().as_str()); + } else { + panic!("created table should be found"); + } + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_share_name_with_default_limit(pool: PgPool) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let num_schemas = testutils::rand::i64(0, 20); + let num_tables = testutils::rand::i64(0, 20); + for _ in 0..num_schemas { + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + for _ in 0..num_tables { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new schema should be created"); + } + } + let fetched = TableService::query_by_share_name(share.name(), None, None, &mut tx) + .await + .expect("created table should be listed"); + assert_eq!((num_schemas * num_tables) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_share_name_with_specified_limit( + pool: PgPool, +) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let num_schemas = testutils::rand::i64(0, 20); + let num_tables = testutils::rand::i64(0, 20); + for _ in 0..num_schemas { + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + for _ in 0..num_tables { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new table should be created"); + } + } + let limit = testutils::rand::i64(0, 20); + let fetched = TableService::query_by_share_name(share.name(), Some(&limit), None, &mut tx) + .await + .expect("created schema should be listed"); + assert_eq!(min(num_schemas * num_tables, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_share_and_schema_name_with_default_limit( + pool: PgPool, +) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let records = testutils::rand::i64(0, 20); + for _ in 0..records { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new schema should be created"); + } + let fetched = TableService::query_by_share_and_schema_name( + share.name(), + schema.name(), + None, + None, + &mut tx, + ) + .await + .expect("created table should be listed"); + assert_eq!(records as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +} + +#[sqlx::test] +async fn test_table_create_and_query_by_share_and_schema_name_with_specified_limit( + pool: PgPool, +) -> Result<()> { + let mut tx = pool + .begin() + .await + .expect("transaction should be started properly"); + let account = create_account(&mut tx) + .await + .expect("new account should be created"); + let share = create_share(account.id(), &mut tx) + .await + .expect("new share should be created"); + let schema = create_schema(account.id(), share.id(), &mut tx) + .await + .expect("new schema should be created"); + let records = testutils::rand::i64(5, 20); + for _ in 0..records { + create_table(account.id(), schema.id(), &mut tx) + .await + .expect("new schema should be created"); + } + let limit = testutils::rand::i64(0, 20); + let fetched = TableService::query_by_share_and_schema_name( + share.name(), + schema.name(), + Some(&limit), + None, + &mut tx, + ) + .await + .expect("created schema should be listed"); + assert_eq!(min(records, limit) as usize, fetched.len()); + tx.rollback() + .await + .expect("rollback should be done properly"); + Ok(()) +}