From 49acc6e94da5507455839816501180c554693dbe Mon Sep 17 00:00:00 2001 From: Matthijs Brobbel Date: Mon, 15 Apr 2019 13:38:00 +0200 Subject: [PATCH] Add tests for cli (#50) * Adding tests for cli Setup cli testing via integration test Add all executables in target/debug to kcov Ignore more test binaries Adding tests Attempt to run cli tests via kcov Use single thread for kcov tests Make sure kcov only runs in linux build containers Fix cfg Fix cfg Attempt to fix kcov cli tests Fix kcov paths Use macro in tests Another attempt to collect coverage Use xenial * Enable branch builds * Add some tests * Add tests * Add test * Reproduction tests --- .gitignore | 2 +- .travis.yml | 9 +- Cargo.lock | 155 ++++++++ dqcsim-cli/Cargo.toml | 10 + dqcsim-cli/Makefile.toml | 13 + dqcsim-cli/examples/plugin.rs | 3 +- dqcsim-cli/src/arg_parse/parse.rs | 29 +- dqcsim-cli/src/arg_parse/plugins.rs | 121 ++++++ dqcsim-cli/src/main.rs | 11 +- dqcsim-cli/tests/cli.rs | 456 ++++++++++++++++++++++ dqcsim/src/host/reproduction/host_call.rs | 19 + 11 files changed, 808 insertions(+), 20 deletions(-) create mode 100644 dqcsim-cli/tests/cli.rs diff --git a/.gitignore b/.gitignore index 3cf9c9d37..b65ed800b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -/target +target **/*.rs.bk *.repro diff --git a/.travis.yml b/.travis.yml index 47d5b8150..855e89a9e 100755 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,6 @@ rust: # allow_failures: # - rust: nightly -branches: - only: - - master - cache: directories: - $HOME/.cargo @@ -53,6 +49,7 @@ addons: env: global: - DQCSIM_HOME=$HOME/.dqcsim + - PATH=$PATH:$HOME/kcov/bin/ install: # pip3 install --upgrade pip setuptools @@ -64,7 +61,6 @@ install: ( [ "$TRAVIS_OS_NAME" = "linux" ] && [ "$TRAVIS_RUST_VERSION" = "stable" ] && - export PATH=$PATH:$HOME/kcov/bin/ && ( kcov --version || ( wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && @@ -87,8 +83,7 @@ script: after_success: | [ "$TRAVIS_OS_NAME" = "linux" ] && [ "$TRAVIS_RUST_VERSION" = "stable" ] && - export PATH=$PATH:$HOME/kcov/bin/ && - for file in target/debug/{dqcsim,simulation,plugin,enum,quantum}*; do [ -x "${file}" ] || continue; mkdir -p "target/cov/$(basename $file)"; kcov --exclude-pattern=/.cargo,/usr/lib,/cpp/test_,/cpp/build/,/dqcsim/tests/ --exclude-region='#[cfg(test)]:#[cfg(testkcovstopmarker)]' --verify "target/cov/$(basename $file)" "$file"; done && + for file in target/debug/*; do [ -x "${file}" ] || continue; mkdir -p "target/cov/$(basename $file)"; kcov --exclude-pattern=/.cargo,/usr/lib,/cpp/test_,/cpp/build/,/dqcsim/tests/,/dqcsim-cli/tests,/enum-variants/tests --exclude-region='#[cfg(test)]:#[cfg(testkcovstopmarker)]' --verify "target/cov/$(basename $file)" "$file"; done && for file in `find dqcsim-api/cpp/build/test_* -executable -and -type f`; do [ -x "${file}" ] || continue; mkdir -p "target/cov/$(basename $file)"; kcov --exclude-pattern=/.cargo,/usr/lib,/cpp/test_,/cpp/build/,/dqcsim/tests/ --exclude-region='#[cfg(test)]:#[cfg(testkcovstopmarker)]' --verify "target/cov/$(basename $file)" "$file"; done && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`/target/debug && cd dqcsim-api/py && diff --git a/Cargo.lock b/Cargo.lock index f0a0ad29c..178015348 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,13 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -25,6 +33,17 @@ dependencies = [ "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "assert_cmd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "escargot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "predicates 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "predicates-tree 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "atty" version = "0.2.11" @@ -163,6 +182,11 @@ dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "dirs" version = "1.0.5" @@ -216,10 +240,15 @@ name = "dqcsim-cli" version = "0.1.0" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "dqcsim 0.1.0", "enum-variants 0.1.0", + "escargot 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "predicates 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -244,6 +273,28 @@ dependencies = [ "syn 0.15.31 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "escargot" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "escargot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "failure" version = "0.1.5" @@ -264,6 +315,14 @@ dependencies = [ "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "float-cmp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fnv" version = "1.0.6" @@ -377,6 +436,11 @@ dependencies = [ "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memchr" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "mio" version = "0.6.16" @@ -421,6 +485,11 @@ name = "nodrop" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "normalize-line-endings" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "num-complex" version = "0.2.1" @@ -439,6 +508,32 @@ name = "pathdiff" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "predicates" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "float-cmp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "normalize-line-endings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "predicates-core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "predicates-tree" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "treeline 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "proc-macro-hack" version = "0.5.4" @@ -625,6 +720,26 @@ name = "ref_thread_local" version = "0.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "regex" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "remove_dir_all" version = "0.5.1" @@ -802,6 +917,14 @@ dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "toml" version = "0.4.10" @@ -810,6 +933,16 @@ dependencies = [ "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ucd-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-segmentation" version = "1.2.1" @@ -825,6 +958,11 @@ name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "utf8-ranges" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "uuid" version = "0.7.2" @@ -893,9 +1031,11 @@ dependencies = [ ] [metadata] +"checksum aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e6f484ae0c99fec2e858eb6134949117399f222608d84cadb3f58c1f97c2364c" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" "checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" +"checksum assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2dc477793bd82ec39799b6f6b3df64938532fdf2ab0d49ef817eac65856a5a1e" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" @@ -912,10 +1052,14 @@ dependencies = [ "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum crossbeam-channel 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0f0ed1a4de2235cabda8558ff5840bffb97fcb64c97827f354a451307df5f72b" "checksum crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f8306fcef4a7b563b76b7dd949ca48f52bc1141aa067d2ea09565f3e2652aa5c" +"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" +"checksum escargot 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ceb9adbf9874d5d028b5e4c5739d22b71988252b25c9c98fe7cf9738bee84597" +"checksum escargot 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74cf96bec282dcdb07099f7e31d9fed323bca9435a09aba7b6d99b7617bca96d" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" +"checksum float-cmp 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "134a8fa843d80a51a5b77d36d42bc2def9edcb0262c914861d08129fd1926600" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" @@ -932,13 +1076,18 @@ dependencies = [ "checksum libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)" = "bedcc7a809076656486ffe045abeeac163da1b558e963a31e29fbfbeba916917" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" +"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" "checksum mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)" = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" +"checksum normalize-line-endings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2e0a1a39eab95caf4f5556da9289b9e68f0aafac901b2ce80daaf020d3b733a8" "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" "checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" "checksum pathdiff 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a3bf70094d203e07844da868b634207e71bfab254fe713171fae9a6e751ccf31" +"checksum predicates 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa984b7cd021a0bf5315bcce4c4ae61d2a535db2a8d288fc7578638690a7b7c3" +"checksum predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06075c3a3e92559ff8929e7a280684489ea27fe44805174c3ebd9328dcb37178" +"checksum predicates-tree 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e63c4859013b38a76eca2414c64911fba30def9e3202ac461a2d22831220124" "checksum proc-macro-hack 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3e90aa19cd73dedc2d0e1e8407473f073d735fef0ab521438de6da8ee449ab66" "checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" "checksum proc-quote 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fa612543f23fda013e1e6ce30b5285a9d313c6e582e57b4ceca74eb5b85685b5" @@ -960,6 +1109,8 @@ dependencies = [ "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum ref_thread_local 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d813022b2e00774a48eaf43caaa3c20b45f040ba8cbf398e2e8911a06668dbe6" +"checksum regex 1.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "559008764a17de49a3146b234641644ed37d118d1ef641a0bb573d146edc6ce0" +"checksum regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dcfd8681eebe297b81d98498869d4aae052137651ad7b96822f09ceb690d0a96" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "adacaae16d02b6ec37fdc7acfcddf365978de76d1983d3ee22afc260e1ca9619" "checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" @@ -981,10 +1132,14 @@ dependencies = [ "checksum term 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +"checksum treeline 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" +"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737" "checksum uuid 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0238db0c5b605dd1cf51de0f21766f97fba2645897024461d6a00c036819a768" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum whoami 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26a97763f20903e18047d20af1f1a667afe2fe6cbd22b80334f2f5fe4441db80" diff --git a/dqcsim-cli/Cargo.toml b/dqcsim-cli/Cargo.toml index 79d152eda..bb15536b2 100644 --- a/dqcsim-cli/Cargo.toml +++ b/dqcsim-cli/Cargo.toml @@ -13,3 +13,13 @@ structopt = "0.2" failure = "0.1" ansi_term = "0.11" clap = "~2.33.0" + +[dev-dependencies] +assert_cmd = "0.11" +predicates = "1.0" +escargot = "0.5.0" +lazy_static = "1.3.0" +rand = "0.6" + +[features] +kcov = [] diff --git a/dqcsim-cli/Makefile.toml b/dqcsim-cli/Makefile.toml index 8c78e2fcf..efc6db349 100644 --- a/dqcsim-cli/Makefile.toml +++ b/dqcsim-cli/Makefile.toml @@ -2,3 +2,16 @@ dependencies = ["release"] command = "make" args = ["install"] + +[tasks.build_examples] +command = "cargo" +args = ["build", "--examples"] + +[tasks.kcov-test] +env = { "RUSTFLAGS" = "-C link-dead-code" } +run_task = "test_kcov" + +[tasks.test_kcov] +command = "cargo" +dependencies = ["build_examples", "build"] +args = ["test", "--features", "kcov", "--", "--test-threads", "1"] diff --git a/dqcsim-cli/examples/plugin.rs b/dqcsim-cli/examples/plugin.rs index 9859c0ed9..dedb91012 100755 --- a/dqcsim-cli/examples/plugin.rs +++ b/dqcsim-cli/examples/plugin.rs @@ -37,8 +37,9 @@ fn main() -> Result<(), Error> { Ok(()) }); - definition.run = Box::new(|_state, _| { + definition.run = Box::new(|state, _| { info!("running run callback!"); + state.send(ArbData::default()).expect("send failed"); Ok(ArbData::default()) }); diff --git a/dqcsim-cli/src/arg_parse/parse.rs b/dqcsim-cli/src/arg_parse/parse.rs index 0352830b7..b433078fa 100644 --- a/dqcsim-cli/src/arg_parse/parse.rs +++ b/dqcsim-cli/src/arg_parse/parse.rs @@ -230,14 +230,6 @@ pub struct CommandLineConfiguration { } impl CommandLineConfiguration { - /// Produces a DQCsim configuration from `std::env::args()`. - /// - /// This is just a shorthand for `parse_from(std::env::args())`, refer - /// to its docs for more info. - pub fn parse() -> Result { - CommandLineConfiguration::parse_from(std::env::args()) - } - /// Produces a DQCsim configuration from the specified command line /// argument iterable. /// @@ -450,3 +442,24 @@ fn format_error_ctxt(ctxt: &str, e: impl Into) -> Result { ))) .into()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn debug() { + let c = CommandLineConfiguration { + host_calls: vec![], + host_stdout: true, + dqcsim: SimulatorConfiguration::default().with_seed("test"), + reproduction_file: None, + }; + + assert_eq!(format!("{:?}", c), "CommandLineConfiguration { host_calls: [], host_stdout: true, dqcsim: SimulatorConfiguration { seed: Seed { value: 14402189752926126668 }, stderr_level: Info, tee_files: [], log_callback: None, dqcsim_level: Trace, plugins: [], reproduction_path_style: Some(Keep) }, reproduction_file: None }"); + } + + #[test] + fn help() {} + +} diff --git a/dqcsim-cli/src/arg_parse/plugins.rs b/dqcsim-cli/src/arg_parse/plugins.rs index 8694fcf49..d4802c00b 100644 --- a/dqcsim-cli/src/arg_parse/plugins.rs +++ b/dqcsim-cli/src/arg_parse/plugins.rs @@ -110,3 +110,124 @@ impl PluginDefinition { } } } + +#[cfg(test)] +mod tests { + use super::*; + use dqcsim::{ + common::types::PluginType, host::configuration::PluginProcessNonfunctionalConfiguration, + }; + + #[test] + fn non_func_opt() { + let pnfo = PluginNonfunctionalOpts::default(); + assert_eq!( + pnfo, + PluginNonfunctionalOpts { + verbosity: None, + tee_files: vec![], + stdout_mode: None, + stderr_mode: None, + accept_timeout: None, + shutdown_timeout: None, + }, + ); + } + + #[test] + fn non_func_opt_debug() { + let pnfo = PluginNonfunctionalOpts::default(); + assert_eq!( + format!("{:?}", pnfo), + "PluginNonfunctionalOpts { verbosity: None, tee_files: [], stdout_mode: None, stderr_mode: None, accept_timeout: None, shutdown_timeout: None }", + ); + } + + #[test] + fn into_process_conf() { + let p = PluginNonfunctionalOpts::default(); + let c: PluginProcessNonfunctionalConfiguration = p.into_config(LoglevelFilter::Debug); + assert_eq!( + c, + PluginProcessNonfunctionalConfiguration { + verbosity: LoglevelFilter::Debug, + tee_files: vec![], + stdout_mode: StreamCaptureMode::Capture(Loglevel::Info), + stderr_mode: StreamCaptureMode::Capture(Loglevel::Info), + accept_timeout: Timeout::from_seconds(5), + shutdown_timeout: Timeout::from_seconds(5), + } + ); + + let p = PluginNonfunctionalOpts { + verbosity: Some(LoglevelFilter::Fatal), + tee_files: vec![TeeFileConfiguration::new( + LoglevelFilter::Error, + "/dev/null", + )], + stdout_mode: Some(StreamCaptureMode::Null), + stderr_mode: Some(StreamCaptureMode::Pass), + accept_timeout: Some(Timeout::Infinite), + shutdown_timeout: Some(Timeout::from_seconds(1)), + }; + let c: PluginProcessNonfunctionalConfiguration = p.into_config(LoglevelFilter::Debug); + assert_eq!( + c, + PluginProcessNonfunctionalConfiguration { + verbosity: LoglevelFilter::Fatal, + tee_files: vec![TeeFileConfiguration::new( + LoglevelFilter::Error, + "/dev/null", + )], + stdout_mode: StreamCaptureMode::Null, + stderr_mode: StreamCaptureMode::Pass, + accept_timeout: Timeout::Infinite, + shutdown_timeout: Timeout::from_seconds(1), + } + ); + } + + #[test] + fn into_def_conf() { + let p = PluginDefinition { + name: "name".to_string(), + specification: PluginProcessSpecification::from_sugar( + "/bin/echo", + PluginType::Operator, + ) + .unwrap(), + functional: PluginProcessFunctionalConfiguration::default(), + nonfunctional: PluginNonfunctionalOpts::default(), + }; + let c: PluginProcessConfiguration = p.into_config(LoglevelFilter::Trace); + assert_eq!( + c, + PluginProcessConfiguration { + name: "name".to_string(), + specification: PluginProcessSpecification::from_sugar( + "/bin/echo", + PluginType::Operator + ) + .unwrap(), + functional: PluginProcessFunctionalConfiguration::default(), + nonfunctional: PluginProcessNonfunctionalConfiguration::default(), + } + ); + } + + #[test] + fn debug() { + let p = PluginDefinition { + name: "name".to_string(), + specification: PluginProcessSpecification::from_sugar( + "/bin/echo", + PluginType::Operator, + ) + .unwrap(), + functional: PluginProcessFunctionalConfiguration::default(), + nonfunctional: PluginNonfunctionalOpts::default(), + }; + assert_eq!(format!("{:?}", p), "PluginDefinition { name: \"name\", specification: PluginProcessSpecification { executable: \"/bin/echo\", script: None, typ: Operator }, functional: PluginProcessFunctionalConfiguration { init: [], env: [], work: \".\" }, nonfunctional: PluginNonfunctionalOpts { verbosity: None, tee_files: [], stdout_mode: None, stderr_mode: None, accept_timeout: None, shutdown_timeout: None } }"); + } + +} diff --git a/dqcsim-cli/src/main.rs b/dqcsim-cli/src/main.rs index ebabbf403..87a663ab7 100755 --- a/dqcsim-cli/src/main.rs +++ b/dqcsim-cli/src/main.rs @@ -3,6 +3,7 @@ use dqcsim::{ host::{accelerator::Accelerator, reproduction::HostCall, simulator::Simulator}, info, note, }; +use std::ffi::OsString; use failure::Error; @@ -58,8 +59,12 @@ fn run( Ok(()) } -fn internal_main() -> Result<(), Error> { - let mut cfg = CommandLineConfiguration::parse().or_else(|e| { +fn internal_main(args: I) -> Result<(), Error> +where + I: IntoIterator, + T: Into + Clone, +{ + let mut cfg = CommandLineConfiguration::parse_from(args).or_else(|e| { println!("{}", e); Err(e) })?; @@ -87,7 +92,7 @@ fn internal_main() -> Result<(), Error> { } fn main() { - let result = internal_main(); + let result = internal_main(std::env::args()); std::process::exit(match result { Ok(_) => 0, Err(_) => 1, diff --git a/dqcsim-cli/tests/cli.rs b/dqcsim-cli/tests/cli.rs new file mode 100644 index 000000000..b97072dc6 --- /dev/null +++ b/dqcsim-cli/tests/cli.rs @@ -0,0 +1,456 @@ +use assert_cmd::prelude::*; +use lazy_static::lazy_static; +use predicates::prelude::*; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use std::{path::PathBuf, process::Command}; + +// https://github.com/glehmann/hld/blob/f40f3b51f84969cc13714f81451ad17f33fbf2dc/tests/common/mod.rs#L79 +macro_rules! cli { + ( $( $v:expr ),* ) => {{ + let cli_bin = escargot::CargoBuild::new() + .current_release() + .current_target() + .run() + .unwrap() + .path() + .to_path_buf(); + let rand_string: String = thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .collect(); + + if cfg!(all(feature = "kcov", target_os = "linux")) { + let target_dir = cli_bin.parent().unwrap(); + let cov_dir = target_dir.join("cov"); + std::fs::create_dir_all(&cov_dir).unwrap(); + Command::new("kcov").args(&[ + "--include-pattern=/src", + "--exclude-pattern=/.cargo", + "--exclude-region='#[cfg(test)]'", + "--verify", + &format!("{}/{}-{}", cov_dir.display().to_string(), env!("CARGO_PKG_NAME"), rand_string), + &cli_bin.display().to_string(), + $($v,)* + ]).assert() + } else { + let args: &[&str] = &[$($v,)*]; + Command::new(cli_bin.display().to_string()).args(args).assert() + } + }} +} + +lazy_static! { + static ref PLUGIN_PATH: PathBuf = assert_cmd::cargo::cargo_bin("examples/plugin"); + static ref PLUGIN: &'static str = PLUGIN_PATH.to_str().unwrap(); +} + +#[test] +fn with_macro() { + cli!(*PLUGIN, *PLUGIN).success(); +} + +#[test] +fn no_arguments() { + cli!() + .failure() + .code(1) + .stdout(predicate::str::contains(include_str!( + "../src/arg_parse/usage.txt" + ))); +} + +#[test] +fn help() { + cli!("--help") + .failure() + .code(1) + .stdout(predicate::str::contains("Used to specify the host API call sequence. Refer to the \"host call sequence\" section for more info.")); + + cli!("plugin", "--help") + // TODO: jeroen other helps exit with failure code 1 + .success() + .stdout(predicates::str::contains("PLUGIN OPTIONS")); +} + +#[test] +fn long_help() { + cli!("--long-help") + .failure() + .code(1) + .stdout(predicate::str::contains( + "Run the specified cQASM file using the cQASM frontend and (default) QX backend.", + )); +} + +#[test] +fn version() { + cli!("--version") + .failure() + .code(1) + .stdout(predicate::str::is_match("^DQCsim [0-9].[0-9].[0-9] (.*)").unwrap()); +} + +#[test] +fn host_call_no_value() { + cli!("--call") + .failure() + .code(1) + .stdout(predicate::str::contains( + "error: The argument '--call ...' requires a value but none was supplied", + )); + cli!("-C") + .failure() + .code(1) + .stdout(predicate::str::contains( + "error: The argument '--call ...' requires a value but none was supplied", + )); +} + +#[test] +fn host_call_ok() { + cli!("--call", "start", "--call", "wait", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains( + "Executing 'start(...)' host call...", + )) + .stderr(predicate::str::contains("Executing 'wait()' host call...")); +} + +#[test] +fn host_call_send_no_data() { + cli!("--call", "send", *PLUGIN, *PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "Invalid value for '--call ...': Invalid argument: the send API call requires an ArbData argument", + )); +} + +#[test] +fn host_call_arb_no_data() { + cli!("--call", "arb", *PLUGIN, *PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "the arb API call requires a plugin and an ArbCmd argument", + )); +} + +#[test] +fn host_call_arb_plugin_not_found() { + cli!( + "--call", + "arb:a:b.c:{\"answer\": 42},x,y,z", + *PLUGIN, + *PLUGIN + ) + .failure() + .stderr(predicate::str::contains( + "Invalid argument: plugin a not found", + )); +} + +#[test] +fn host_call_arb() { + cli!( + "--call", + "arb:front:b.c:{\"answer\": 42},x,y,z", + *PLUGIN, + *PLUGIN + ) + .success() + .stderr(predicate::str::contains( + "Executing 'arb(...)' host call...", + )); + + cli!( + "--host-stdout", + "--call", + "arb:front:b.c:{\"answer\": 42},x,y,z", + *PLUGIN, + *PLUGIN + ) + .success() + .stdout(predicate::str::contains("arb:")) + .stderr(predicate::str::contains( + "Executing 'arb(...)' host call...", + )); +} + +#[test] +fn host_call_wait_data() { + cli!("--call", "wait:{},a.b", *PLUGIN, *PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "the wait API call does not take an argument", + )); +} + +#[test] +fn host_call_recv_data() { + cli!("--call", "recv:{},a.b", *PLUGIN, *PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "the recv API call does not take an argument", + )); +} + +#[test] +fn host_call_yield_data() { + cli!("--call", "yield:{},a.b", *PLUGIN, *PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "the yield API call does not take an argument", + )); +} + +#[test] +fn host_call_yield() { + cli!("--call", "yield", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains("Executing 'yield()' host call...")); +} + +#[test] +fn host_call_recv() { + cli!("--call", "send:{},a.b", "--call", "recv", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains("Executing 'recv()' host call...")); +} + +#[test] +fn host_call_recv_deadlock() { + cli!("--call", "recv", "--call", "recv", *PLUGIN, *PLUGIN) + .failure() + .stderr(predicate::str::contains("Executing 'recv()' host call...")) + .stderr(predicate::str::contains( + "Deadlock: accelerator exited before sending data", + )); +} + +#[test] +fn host_call_send() { + cli!("--call", "send:{},a.b", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains( + "Executing 'send(...)' host call...", + )); +} + +#[test] +fn host_call_with_reproduce() { + cli!( + "--reproduce", + "/dev/zero", + "--call", + "start", + *PLUGIN, + *PLUGIN + ) + .failure() + .code(1) + .stdout(predicate::str::contains( + "The argument '--reproduce ' cannot be used with '--call ...", + )); +} + +#[test] +fn host_call_with_reproduce_exactly() { + cli!( + "--reproduce-exactly", + "/dev/zero", + "-C", + "start", + *PLUGIN, + *PLUGIN + ) + .failure() + .code(1) + .stdout(predicate::str::contains( + "The argument '--reproduce-exactly ' cannot be used with '--call ...", + )); +} + +#[test] +fn host_stdout() { + cli!("--host-stdout", *PLUGIN, *PLUGIN) + .success() + .stdout(predicate::str::contains("wait(): {}")); +} + +#[test] +fn with_operator() { + cli!(*PLUGIN, *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains("op1")); +} + +#[test] +fn plugin_config_name() { + cli!(*PLUGIN, "--name", "frontend-test", *PLUGIN) + .success() + .stderr(predicate::str::contains("frontend-test")); +} + +#[test] +fn plugin_env_mod() { + cli!(*PLUGIN, "--env", "key:value", *PLUGIN).success(); + cli!(*PLUGIN, "--env", "~key", *PLUGIN).success(); +} + +#[test] +fn double_start_insert_wait() { + cli!("-C", "start", "-C", "start", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains("Executing 'start(...)' host call...").count(2)) + .stderr(predicates::str::contains("Executing 'wait()' host call...").count(2)); +} + +#[test] +fn bad_path() { + let path = assert_cmd::cargo::cargo_bin("asdf"); + cli!(path.to_str().unwrap()) + .failure() + .code(1) + .stdout(predicate::str::contains( + "While interpreting plugin specification: Invalid argument: the plugin specification", + )) + .stdout(predicate::str::contains( + "/asdf' appears to be a path, but the referenced file does not exist", + )); +} + +#[test] +fn no_backend() { + cli!(*PLUGIN) + .failure() + .code(1) + .stdout(predicate::str::contains( + "While interpreting plugin specification: Invalid argument: could not find plugin executable 'dqcsbeqx', needed for plugin specification 'qx'", + )); +} + +#[test] +fn no_repro_out() { + cli!(*PLUGIN, "--no-repro-out") + .failure() + .code(1) + .stderr(predicate::str::contains( + "Found argument '--no-repro-out' which wasn't expected, or isn't valid in this context", + )); + + cli!("--no-repro-out", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains( + "Simulation completed successfully.", + )); +} + +#[test] +fn repro_out() { + cli!("--repro-out", "/not_allowed", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains( + "When trying to write reproduction file:", + )); + + cli!("--repro-out", "/tmp/repro-out.out", *PLUGIN, *PLUGIN) + .success() + .stderr(predicate::str::contains( + "Simulation completed successfully.", + )); +} + +#[test] +fn no_repro_out_repro_out() { + cli!( + "--no-repro-out", + "--repro-out", + "/tmp/repro.out", + *PLUGIN, + *PLUGIN + ) + .failure() + .code(1) + .stdout(predicate::str::contains( + "The argument '--no-repro-out' cannot be used with '--repro-out '", + )); +} + +#[test] +fn reproduce_bad_path() { + cli!("--reproduce", "./asdf") + .failure() + .stdout(predicates::str::contains("While reading reproduction file")); +} + +#[test] +fn reproduce() { + cli!( + "--repro-out", + "./dqcsim-cli.test.repro.out", + *PLUGIN, + *PLUGIN + ) + .success(); + cli!("--reproduce", "./dqcsim-cli.test.repro.out").success(); + + // illegal name override + cli!( + "--reproduce", + "./dqcsim-cli.test.repro.out", + "@front", + "-n", + "override-name" + ) + .failure() + .stdout(predicates::str::contains( + "cannot be used when referencing a previously defined plugin", + )); + + // illegal work override + cli!( + "--reproduce", + "./dqcsim-cli.test.repro.out", + "@front", + "--work", + "work" + ) + .failure() + .stdout(predicates::str::contains( + "cannot be used when referencing a previously defined plugin", + )); + + // override verbosity + cli!( + "--reproduce", + "./dqcsim-cli.test.repro.out", + "@front", + "-l", + "fatal" + ) + .success(); + + // exact reproduce + cli!( + "--reproduce-exactly", + "./dqcsim-cli.test.repro.out", + "@front", + "-l", + "fatal" + ) + .success(); + + // def with reproduce + cli!("--reproduce", "./dqcsim-cli.test.repro.out", *PLUGIN) + .failure() + .stdout(predicates::str::contains("Cannot define new plugins while")); + + // mod with def + cli!(*PLUGIN, *PLUGIN, "@front", "-l", "trace") + .failure() + .stdout(predicates::str::contains("Cannot modify plugins unless")); +} diff --git a/dqcsim/src/host/reproduction/host_call.rs b/dqcsim/src/host/reproduction/host_call.rs index b96e4448f..9e6969b3c 100644 --- a/dqcsim/src/host/reproduction/host_call.rs +++ b/dqcsim/src/host/reproduction/host_call.rs @@ -173,4 +173,23 @@ mod test { ); } + #[test] + fn display() { + assert_eq!( + format!("{}", HostCall::Start(ArbData::default())), + "start:{}" + ); + assert_eq!(format!("{}", HostCall::Send(ArbData::default())), "send:{}"); + assert_eq!( + format!( + "{}", + HostCall::Arb("a".to_string(), ArbCmd::new("a", "b", ArbData::default())) + ), + "arb:a:a.b:{}" + ); + assert_eq!(format!("{}", HostCall::Wait), "wait"); + assert_eq!(format!("{}", HostCall::Recv), "recv"); + assert_eq!(format!("{}", HostCall::Yield), "yield"); + } + }