Merge pull request parallaxsecond#160 from ionut-arm/e2e-tests

Move end to end tests to own crate

.github/workflows/ci.yml

@@ -7,38 +7,38 @@ jobs:
     name: Various tests targeting a Parsec image with all providers included
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Build the container
-      run: docker build -t all-providers tests/all_providers
-    - name: Run the container to execute the test script
-      run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec all-providers /tmp/parsec/tests/ci.sh all
+      - uses: actions/checkout@v1
+      - name: Build the container
+        run: docker build -t all-providers e2e_tests/provider_cfg/all
+      - name: Run the container to execute the test script
+        run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec all-providers /tmp/parsec/tests/ci.sh all
 
   mbed-crypto-provider:
     name: Integration tests using Mbed Crypto provider
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Build the container
-      run: docker build -t mbed-crypto-provider tests/per_provider/provider_cfg/mbed-crypto
-    - name: Run the container to execute the test script
-      run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec mbed-crypto-provider /tmp/parsec/tests/ci.sh mbed-crypto
+      - uses: actions/checkout@v1
+      - name: Build the container
+        run: docker build -t mbed-crypto-provider e2e_tests/provider_cfg/mbed-crypto
+      - name: Run the container to execute the test script
+        run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec mbed-crypto-provider /tmp/parsec/tests/ci.sh mbed-crypto
 
   pkcs11-provider:
     name: Integration tests using PKCS 11 provider
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Build the container
-      run: docker build -t pkcs11-provider tests/per_provider/provider_cfg/pkcs11
-    - name: Run the container to execute the test script
-      run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec pkcs11-provider /tmp/parsec/tests/ci.sh pkcs11
+      - uses: actions/checkout@v1
+      - name: Build the container
+        run: docker build -t pkcs11-provider e2e_tests/provider_cfg/pkcs11
+      - name: Run the container to execute the test script
+        run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec pkcs11-provider /tmp/parsec/tests/ci.sh pkcs11
 
   tpm-provider:
     name: Integration tests using TPM provider
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Build the container
-      run: docker build -t tpm-provider tests/per_provider/provider_cfg/tpm
-    - name: Run the container to execute the test script
-      run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec tpm-provider /tmp/parsec/tests/ci.sh tpm
+      - uses: actions/checkout@v1
+      - name: Build the container
+        run: docker build -t tpm-provider e2e_tests/provider_cfg/tpm
+      - name: Run the container to execute the test script
+        run: docker run -v $(pwd):/tmp/parsec -w /tmp/parsec tpm-provider /tmp/parsec/tests/ci.sh tpm

.travis.yml

@@ -7,14 +7,14 @@ services:
 jobs:
   include:
     - name: "Integration tests using Mbed Crypto provider"
-      env: DOCKER_IMAGE_NAME=mbed-crypto-provider DOCKER_IMAGE_PATH=tests/per_provider/provider_cfg/mbed-crypto SCRIPT="tests/ci.sh mbed-crypto"
+      env: DOCKER_IMAGE_NAME=mbed-crypto-provider DOCKER_IMAGE_PATH=e2e_tests/provider_cfg/mbed-crypto SCRIPT="tests/ci.sh mbed-crypto"
     - name: "Integration tests using PKCS 11 provider"
-      env: DOCKER_IMAGE_NAME=pkcs11-provider DOCKER_IMAGE_PATH=tests/per_provider/provider_cfg/pkcs11 SCRIPT="tests/ci.sh pkcs11"
+      env: DOCKER_IMAGE_NAME=pkcs11-provider DOCKER_IMAGE_PATH=e2e_tests/provider_cfg/pkcs11 SCRIPT="tests/ci.sh pkcs11"
   # PKCS11 tests are failing because of unidentified issues.
   # See https://github.com/parallaxsecond/parsec/issues/116
   allow_failures:
-    - env: DOCKER_IMAGE_NAME=mbed-crypto-provider DOCKER_IMAGE_PATH=tests/per_provider/provider_cfg/mbed-crypto SCRIPT="tests/ci.sh mbed-crypto"
-    - env: DOCKER_IMAGE_NAME=pkcs11-provider DOCKER_IMAGE_PATH=tests/per_provider/provider_cfg/pkcs11 SCRIPT="tests/ci.sh pkcs11"
+    - env: DOCKER_IMAGE_NAME=mbed-crypto-provider DOCKER_IMAGE_PATH=e2e_tests/provider_cfg/mbed-crypto SCRIPT="tests/ci.sh mbed-crypto"
+    - env: DOCKER_IMAGE_NAME=pkcs11-provider DOCKER_IMAGE_PATH=e2e_tests/provider_cfg/pkcs11 SCRIPT="tests/ci.sh pkcs11"
 script:
   - docker build -t $DOCKER_IMAGE_NAME $DOCKER_IMAGE_PATH
   - docker run -v $(pwd):/tmp/parsec -w /tmp/parsec $DOCKER_IMAGE_NAME /tmp/parsec/$SCRIPT

e2e_tests/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

e2e_tests/Cargo.toml

@@ -0,0 +1,27 @@
+# This crate should NOT be upstreamed to crates.io; it is only intended for 
+# PARSEC testing purposes (both locally and on CI)
+[package]
+name = "e2e_tests"
+version = "0.1.0"
+authors = ["Anton Antonov <[email protected]>",
+           "Paul Howard <[email protected]>",
+           "Ionut Mihalcea <[email protected]>",
+           "Hugues de Valon <[email protected]>"]
+description = "End to end testing framework for PARSEC"
+license = "Apache-2.0"
+edition = "2018"
+publish = false
+
+[dependencies]
+num_cpus = "1.13.0"
+picky-asn1-der = "0.2.2"
+picky-asn1 = "0.2.1"
+serde = { version = "1.0", features = ["derive"] }
+sha2 = "0.8.1"
+parsec-client = { version = "0.2.0", features = ["testing"] }
+log = "0.4.8"
+rand = "0.7.3"
+
+[dev-dependencies]
+env_logger = "0.7.1"
+uuid = "0.7.4"

tests/test_clients/mod.rs → e2e_tests/src/lib.rs

@@ -11,12 +11,15 @@ pub use parsec_client::error;
 use log::error;
 use parsec_client::auth::AuthenticationData;
 use parsec_client::core::basic_client::BasicClient;
+use parsec_client::core::interface::operations::list_providers::ProviderInfo;
+use parsec_client::core::interface::operations::psa_algorithm::{
+    Algorithm, AsymmetricSignature, Hash,
+};
+use parsec_client::core::interface::operations::psa_key_attributes::{
+    KeyAttributes, KeyPolicy, KeyType, UsageFlags,
+};
+use parsec_client::core::interface::requests::{Opcode, ProviderID, ResponseStatus, Result};
 use parsec_client::error::Error;
-use parsec_interface::operations::list_providers::ProviderInfo;
-use parsec_interface::operations::psa_algorithm::{Algorithm, AsymmetricSignature, Hash};
-use parsec_interface::operations::psa_key_attributes::*;
-use parsec_interface::operations::psa_key_attributes::{KeyAttributes, KeyPolicy, UsageFlags};
-use parsec_interface::requests::{Opcode, ProviderID, ResponseStatus, Result};
 use std::collections::HashSet;
 use std::time::Duration;
 

tests/test_clients/raw_request.rs → e2e_tests/src/raw_request.rs

@@ -1,7 +1,7 @@
 // Copyright 2020 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use parsec_interface::requests::request::RawHeader;
-use parsec_interface::requests::{Response, Result};
+use parsec_client::core::interface::requests::request::RawHeader;
+use parsec_client::core::interface::requests::{Response, Result};
 use std::io::Write;
 use std::os::unix::net::UnixStream;
 use std::thread;

tests/test_clients/stress.rs → e2e_tests/src/stress.rs

@@ -1,8 +1,8 @@
 // Copyright 2020 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
+use super::TestClient;
 use log::info;
-use parsec_interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::ResponseStatus;
 use rand::Rng;
 use rand::{
     distributions::{Alphanumeric, Distribution, Standard},

tests/all_providers/mod.rs → e2e_tests/tests/all_providers/mod.rs

@@ -1,7 +1,7 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::requests::{Opcode, ProviderID, Result};
+use e2e_tests::TestClient;
+use parsec_client::core::interface::requests::{Opcode, ProviderID, Result};
 use std::collections::HashSet;
 use uuid::Uuid;
 

tests/mod.rs → e2e_tests/tests/mod.rs

@@ -32,4 +32,3 @@
 
 mod all_providers;
 mod per_provider;
-mod test_clients;

tests/per_provider/normal_tests/asym_sign_verify.rs → e2e_tests/tests/per_provider/normal_tests/asym_sign_verify.rs

@@ -1,10 +1,10 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::operations::psa_algorithm::*;
-use parsec_interface::operations::psa_key_attributes::*;
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::operations::psa_algorithm::*;
+use parsec_client::core::interface::operations::psa_key_attributes::*;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 use sha2::{Digest, Sha256};
 
 const HASH: [u8; 32] = [

tests/per_provider/normal_tests/auth.rs → e2e_tests/tests/per_provider/normal_tests/auth.rs

@@ -1,8 +1,8 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 
 #[test]
 fn two_auths_same_key_name() -> Result<()> {

tests/per_provider/normal_tests/basic.rs → e2e_tests/tests/per_provider/normal_tests/basic.rs

@@ -1,8 +1,8 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::RawRequestClient;
-use parsec_interface::requests::request::RawHeader;
-use parsec_interface::requests::{Opcode, ProviderID, ResponseStatus};
+use e2e_tests::RawRequestClient;
+use parsec_client::core::interface::requests::request::RawHeader;
+use parsec_client::core::interface::requests::{Opcode, ProviderID, ResponseStatus};
 
 #[test]
 fn invalid_provider() {

tests/per_provider/normal_tests/create_destroy_key.rs → e2e_tests/tests/per_provider/normal_tests/create_destroy_key.rs

@@ -1,12 +1,14 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::operations::psa_algorithm::{Algorithm, AsymmetricSignature, Hash};
-use parsec_interface::operations::psa_key_attributes::{
+use e2e_tests::TestClient;
+use parsec_client::core::interface::operations::psa_algorithm::{
+    Algorithm, AsymmetricSignature, Hash,
+};
+use parsec_client::core::interface::operations::psa_key_attributes::{
     KeyAttributes, KeyPolicy, KeyType, UsageFlags,
 };
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 use picky_asn1::wrapper::IntegerAsn1;
 use serde::{Deserialize, Serialize};
 

tests/per_provider/normal_tests/export_public_key.rs → e2e_tests/tests/per_provider/normal_tests/export_public_key.rs

@@ -1,10 +1,10 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::operations::psa_algorithm::*;
-use parsec_interface::operations::psa_key_attributes::*;
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::operations::psa_algorithm::*;
+use parsec_client::core::interface::operations::psa_key_attributes::*;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 use picky_asn1::wrapper::IntegerAsn1;
 use serde::{Deserialize, Serialize};
 

tests/per_provider/normal_tests/import_key.rs → e2e_tests/tests/per_provider/normal_tests/import_key.rs

@@ -1,10 +1,10 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::operations::psa_algorithm::*;
-use parsec_interface::operations::psa_key_attributes::*;
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::operations::psa_algorithm::*;
+use parsec_client::core::interface::operations::psa_key_attributes::*;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 use picky_asn1::wrapper::IntegerAsn1;
 use serde::{Deserialize, Serialize};
 

tests/per_provider/normal_tests/key_attributes.rs → e2e_tests/tests/per_provider/normal_tests/key_attributes.rs

@@ -1,11 +1,13 @@
 // Copyright 2020 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::TestClient;
-use parsec_interface::operations::psa_algorithm::{Algorithm, AsymmetricSignature, Cipher, Hash};
-use parsec_interface::operations::psa_key_attributes::{
+use e2e_tests::TestClient;
+use parsec_client::core::interface::operations::psa_algorithm::{
+    Algorithm, AsymmetricSignature, Cipher, Hash,
+};
+use parsec_client::core::interface::operations::psa_key_attributes::{
     KeyAttributes, KeyPolicy, KeyType, UsageFlags,
 };
-use parsec_interface::requests::{ProviderID, ResponseStatus};
+use parsec_client::core::interface::requests::{ProviderID, ResponseStatus};
 
 // Ignored as only RSA key types are supported for now.
 #[ignore]

tests/per_provider/normal_tests/ping.rs → e2e_tests/tests/per_provider/normal_tests/ping.rs

@@ -1,12 +1,12 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::RequestClient;
-use crate::test_clients::TestClient;
-use parsec_interface::requests::request::{Request, RequestAuth, RequestBody};
-use parsec_interface::requests::Opcode;
-use parsec_interface::requests::ProviderID;
-use parsec_interface::requests::ResponseStatus;
-use parsec_interface::requests::Result;
+use e2e_tests::RequestClient;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::requests::request::{Request, RequestAuth, RequestBody};
+use parsec_client::core::interface::requests::Opcode;
+use parsec_client::core::interface::requests::ProviderID;
+use parsec_client::core::interface::requests::ResponseStatus;
+use parsec_client::core::interface::requests::Result;
 
 #[test]
 fn test_ping() -> Result<()> {

tests/per_provider/persistent_after.rs → e2e_tests/tests/per_provider/persistent_after.rs

@@ -3,9 +3,9 @@
 
 // These functions test for the service persistency to shutdown. They will be executed after the
 // service is shutdown, after the persistent_before tests are executed.
-use crate::test_clients::TestClient;
-use parsec_interface::requests::Result;
-use parsec_interface::requests::{ProviderID, ResponseStatus};
+use e2e_tests::TestClient;
+use parsec_client::core::interface::requests::Result;
+use parsec_client::core::interface::requests::{ProviderID, ResponseStatus};
 
 const HASH: [u8; 32] = [
     0x69, 0x3E, 0xDB, 0x1B, 0x22, 0x79, 0x03, 0xF4, 0xC0, 0xBF, 0xD6, 0x91, 0x76, 0x37, 0x84, 0xA2,

tests/per_provider/persistent_before.rs → e2e_tests/tests/per_provider/persistent_before.rs

@@ -3,8 +3,8 @@
 
 // These functions test for the service persistency to shutdown. They will be executed before the
 // service is shutdown and before the persistent_after tests are executed.
-use crate::test_clients::TestClient;
-use parsec_interface::requests::Result;
+use e2e_tests::TestClient;
+use parsec_client::core::interface::requests::Result;
 
 const HASH: [u8; 32] = [
     0x69, 0x3E, 0xDB, 0x1B, 0x22, 0x79, 0x03, 0xF4, 0xC0, 0xBF, 0xD6, 0x91, 0x76, 0x37, 0x84, 0xA2,

tests/per_provider/stress_test.rs → e2e_tests/tests/per_provider/stress_test.rs

@@ -1,6 +1,6 @@
 // Copyright 2019 Contributors to the Parsec project.
 // SPDX-License-Identifier: Apache-2.0
-use crate::test_clients::stress::{StressClient, StressTestConfig};
+use e2e_tests::stress::{StressClient, StressTestConfig};
 use std::time::Duration;
 
 #[test]

tests/ci.sh

@@ -63,12 +63,11 @@ while [ "$#" -gt 0 ]; do
                 error_msg "Only one provider name must be given"
             fi
             PROVIDER_NAME=$1
+            CONFIG_PATH="e2e_tests/provider_cfg/$1/config.toml"
             if [ "$PROVIDER_NAME" = "all" ]; then
                 FEATURES="--features=all-providers"
-                CONFIG_PATH="tests/all_providers/config.toml"
             else
                 FEATURES="--features=$1-provider"
-                CONFIG_PATH="tests/per_provider/provider_cfg/$1/config.toml"
             fi
         ;;
         *)
@@ -96,7 +95,7 @@ fi
 
 if [ "$PROVIDER_NAME" = "pkcs11" ] || [ "$PROVIDER_NAME" = "all" ]; then
     # Find and append the slot number at the end of the configuration file.
-    tests/per_provider/provider_cfg/pkcs11/find_slot_number.sh $CONFIG_PATH
+    e2e_tests/provider_cfg/pkcs11/find_slot_number.sh $CONFIG_PATH
 fi
 
 echo "Build test"
@@ -127,14 +126,14 @@ pgrep -f target/debug/parsec >/dev/null
 
 if [ "$PROVIDER_NAME" = "all" ]; then
     echo "Execute all-providers tests"
-    RUST_BACKTRACE=1 cargo test $FEATURES all_providers
+    RUST_BACKTRACE=1 cargo test --manifest-path ./e2e_tests/Cargo.toml all_providers
 else
     # Per provider tests
     echo "Execute normal tests"
-    RUST_BACKTRACE=1 cargo test $FEATURES normal_tests
+    RUST_BACKTRACE=1 cargo test --manifest-path ./e2e_tests/Cargo.toml normal_tests
 
     echo "Execute persistent test, before the reload"
-    RUST_BACKTRACE=1 cargo test $FEATURES persistent_before
+    RUST_BACKTRACE=1 cargo test --manifest-path ./e2e_tests/Cargo.toml persistent_before
 
     # Create a fake mapping file for the root application, the provider and a
     # key name of "Test Key". It contains a valid KeyInfo structure.
@@ -160,7 +159,7 @@ else
     sleep 5
 
     echo "Execute persistent test, after the reload"
-    RUST_BACKTRACE=1 cargo test $FEATURES persistent_after
+    RUST_BACKTRACE=1 cargo test --manifest-path ./e2e_tests/Cargo.toml persistent_after
 
     if [ -z "$NO_STRESS_TEST" ]; then
         echo "Shutdown Parsec"
@@ -176,6 +175,6 @@ else
         sleep 5
 
         echo "Execute stress tests"
-        RUST_BACKTRACE=1 cargo test $FEATURES stress_test
+        RUST_BACKTRACE=1 cargo test --manifest-path ./e2e_tests/Cargo.toml stress_test
 	fi
 fi