From 1373e4bc0f0c8d3c51f7fdcfd0a0517d20fddac0 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Thu, 26 Sep 2024 16:46:40 +0200 Subject: [PATCH 01/17] Using github matrix to run mock tests --- .github/workflows/test.yml | 68 ++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67351d933..e3822dae6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,6 +46,32 @@ jobs: - dotnet: version: 8.0 handler: GrpcWebHandler + endpoint: + - mock_env: + Grpc__Port: 5000 + Http__Port: 4999 + - mock_env: + Grpc__Port: 5001 + Http__Port: 5001 + Http__Cert: certs/server1.pem + Http__Key: certs/server1.key + - mock_env: + Grpc__Port: 5002 + Http__Port: 5002 + Http__Cert: certs/server2.pem + Http__Key: certs/server2.key + - mock_env: + Grpc__Port: 5003 + Http__Port: 5003 + Http__Cert: certs/server1.pem + Http__Key: certs/server1.key + Http__ClientCert: certs/client-ca.pem + - mock_env: + Grpc__Port: 5004 + Http__Port: 5004 + Http__Cert: certs/server2.pem + Http__Key: certs/server2.key + Http__ClientCert: certs/client-ca.pem runs-on: ${{ matrix.platform.os }} steps: @@ -88,42 +114,34 @@ jobs: dotnet publish -o ../out [ -e ../out/ArmoniK.Api.Mock.exe ] || ln -s ArmoniK.Api.Mock ../out/ArmoniK.Api.Mock.exe - - name: Test + - name: Run Mock servers working-directory: packages/csharp/ shell: bash - env: - GrpcClient__HttpMessageHandler: ${{ matrix.handler }} + env: ${{ matrix.endpoint.mock_env }} run: | - set +e - set -x export CertFolder="$PWD/certs" - ./out/ArmoniK.Api.Mock.exe \ - grpc:port=5000 http:port=4999 \ - & notls_pid=$! - ./out/ArmoniK.Api.Mock.exe \ - grpc:port=5001 http:port=5001 \ - http:cert="$CertFolder/server1.pem" http:key="$CertFolder/server1.key" \ - & tls_pid=$! - ./out/ArmoniK.Api.Mock.exe \ - grpc:port=5002 http:port=5002 \ - http:cert="$CertFolder/server2.pem" http:key="$CertFolder/server2.key" \ - & tlsstore_pid=$! - ./out/ArmoniK.Api.Mock.exe \ - grpc:port=5003 http:port=5003 \ - http:cert="$CertFolder/server1.pem" http:key="$CertFolder/server1.key" http:clientcert="$CertFolder/client-ca.pem" \ - & mtls_pid=$! - ./out/ArmoniK.Api.Mock.exe \ - grpc:port=5004 http:port=5004 \ - http:cert="$CertFolder/server2.pem" http:key="$CertFolder/server2.key" http:clientcert="$CertFolder/client-ca.pem" \ - & mtlsstore_pid=$! + printenv | grep Grpc + ./out/ArmoniK.Api.Mock.exe & + echo "server_pid=$!" >> $GITHUB_ENV sleep 5 + - name: Run Test + working-directory: packages/csharp/ + shell: bash + env: + GrpcClient__HttpMessageHandler: ${{ matrix.handler }} + run: | cd ArmoniK.Api.Client.Test dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$? - kill $notls_pid $tls_pid $tlsstore_pid $mtls_pid $mtlsstore_pid exit $ret + - name: Stop Mock Server + if: always() + working-directory: packages/csharp/ + run: | + kill $server_pid || true + - name: Test Report uses: dorny/test-reporter@v1 if: always() From 2723421f82302fcc34526f944233a6d9cc81fd09 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Thu, 10 Oct 2024 09:29:03 +0200 Subject: [PATCH 02/17] Modify test to read parameters from environnement variables --- .../ConnectivityTest.cs | 55 +++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index 6552692a9..b0610b1bc 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -21,9 +21,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using System.Linq; +using System.Runtime.InteropServices; using System.Threading.Tasks; +using ArmoniK.Api.Client.Options; +using ArmoniK.Api.Client.Submitter; using ArmoniK.Api.gRPC.V1; using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Utils; @@ -35,10 +39,44 @@ namespace ArmoniK.Api.Client.Tests; [TestFixture] public class ConnectivityTests { + [SetUp] + public void SetUp() + { + certPath_ = Environment.GetEnvironmentVariable("Grpc__ClientCert") ?? ""; + keyPath_ = Environment.GetEnvironmentVariable("Grpc__ClientKey") ?? ""; + CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; + MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; + endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; + } + + [TearDown] + public void TearDown() + { + } + + private static string CertFolder + => Environment.GetEnvironmentVariable("CertFolder") ?? "../../../../certs"; + + private static string? endpoint_; + private static string? certPath_; + private static string? keyPath_; + private static string? CaCertPath_; + private static string? MessageHandler_; + [Test] - public void ResultsGetServiceConfiguration([Values] ConnectivityKind connectivityKind) + public void ResultsGetServiceConfiguration() { - var channel = connectivityKind.GetChannel(); + Console.WriteLine("Endpoint: " + endpoint_); + var channel = GrpcChannelFactory.CreateChannel(new GrpcClient + { + Endpoint = endpoint_, + AllowUnsafeConnection = true, + CertPem = certPath_!, + KeyPem = keyPath_!, + CaCert = CaCertPath_!, + HttpMessageHandler = MessageHandler_!, + }); + var resultClient = new Results.ResultsClient(channel); Assert.That(() => resultClient.GetServiceConfiguration(new Empty()), @@ -46,8 +84,7 @@ public void ResultsGetServiceConfiguration([Values] ConnectivityKind connectivit } [Test] - public async Task MultipleChannels([Values] ConnectivityKind connectivityKind, - [Values(1, + public async Task MultipleChannels([Values(1, 2, 10, 100)] @@ -56,7 +93,15 @@ public async Task MultipleChannels([Values] ConnectivityKind connectivityKind, var channels = await Enumerable.Range(0, concurrency) .ParallelSelect(new ParallelTaskOptions(-1), - i => Task.FromResult(connectivityKind.GetChannel())) + i => Task.FromResult(GrpcChannelFactory.CreateChannel(new GrpcClient + { + Endpoint = endpoint_, + AllowUnsafeConnection = true, + CertPem = certPath_!, + KeyPem = keyPath_!, + CaCert = CaCertPath_!, + HttpMessageHandler = MessageHandler_!, + }))) .ToListAsync() .ConfigureAwait(false); From cb28ab7bd264a0ff717885586b29263fc4179d75 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Thu, 10 Oct 2024 09:46:49 +0200 Subject: [PATCH 03/17] Configure test env in ci --- .github/workflows/test.yml | 43 +++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3822dae6..6364f74b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,28 +50,44 @@ jobs: - mock_env: Grpc__Port: 5000 Http__Port: 4999 + test_env: + Grpc__Endpoint: http://localhost:4999 - mock_env: Grpc__Port: 5001 Http__Port: 5001 Http__Cert: certs/server1.pem Http__Key: certs/server1.key + test_env: + Grpc__Endpoint: https://localhost:5001 + Grpc__CaCert: /home/runner/work/ArmoniK.Api/ArmoniK.Api/packages/csharp/certs/server1-ca.pem - mock_env: Grpc__Port: 5002 Http__Port: 5002 Http__Cert: certs/server2.pem Http__Key: certs/server2.key + test_env: + Grpc__Endpoint: https://localhost:5002 - mock_env: Grpc__Port: 5003 Http__Port: 5003 Http__Cert: certs/server1.pem Http__Key: certs/server1.key Http__ClientCert: certs/client-ca.pem + test_env: + Grpc__Endpoint: https://localhost:5003 + Grpc__CaCert: packages/csharp/certs/server1-ca.pem + Grpc__ClientCert: packages/csharp/certs/client.pem + Grpc__ClientKey: packages/csharp/certs/client.key - mock_env: Grpc__Port: 5004 Http__Port: 5004 Http__Cert: certs/server2.pem Http__Key: certs/server2.key Http__ClientCert: certs/client-ca.pem + test_env: + Grpc__Endpoint: https://localhost:5004 + Grpc__ClientCert: packages/csharp/certs/client.pem + Grpc__ClientKey: packages/csharp/certs/client.key runs-on: ${{ matrix.platform.os }} steps: @@ -119,8 +135,6 @@ jobs: shell: bash env: ${{ matrix.endpoint.mock_env }} run: | - export CertFolder="$PWD/certs" - printenv | grep Grpc ./out/ArmoniK.Api.Mock.exe & echo "server_pid=$!" >> $GITHUB_ENV sleep 5 @@ -128,20 +142,29 @@ jobs: - name: Run Test working-directory: packages/csharp/ shell: bash - env: - GrpcClient__HttpMessageHandler: ${{ matrix.handler }} + env: ${{ matrix.endpoint.test_env }} run: | + echo "Grpc__CaCert: $Grpc__CaCert" + echo "Grpc__ClientCert: $Grpc__ClientCert" + echo "Grpc__ClientKey: $Grpc__ClientKey" + export GrpcClient__HttpMessageHandler=${{ matrix.handler }} + if [ -n "$Grpc__CaCert" ]; then + export Grpc__CaCert=${{ github.workspace }}/$Grpc__CaCert + fi + if [ -n "$Grpc__ClientCert" ]; then + export Grpc__ClientCert=${{ github.workspace }}/$Grpc__ClientCert + fi + if [ -n "$Grpc__ClientKey" ]; then + export Grpc__ClientKey=${{ github.workspace }}/$Grpc__ClientKey + fi + echo "Grpc__CaCert: $Grpc__CaCert" + echo "Grpc__ClientCert: $Grpc__ClientCert" + echo "Grpc__ClientKey: $Grpc__ClientKey" cd ArmoniK.Api.Client.Test dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$? exit $ret - - name: Stop Mock Server - if: always() - working-directory: packages/csharp/ - run: | - kill $server_pid || true - - name: Test Report uses: dorny/test-reporter@v1 if: always() From 82321863f262df35442d43ef6f49353121ad6b2c Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Thu, 10 Oct 2024 17:21:43 +0200 Subject: [PATCH 04/17] Handle Net framework and wed case --- .github/workflows/test.yml | 6 ------ .../csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs | 8 +++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6364f74b3..428cec0c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,9 +144,6 @@ jobs: shell: bash env: ${{ matrix.endpoint.test_env }} run: | - echo "Grpc__CaCert: $Grpc__CaCert" - echo "Grpc__ClientCert: $Grpc__ClientCert" - echo "Grpc__ClientKey: $Grpc__ClientKey" export GrpcClient__HttpMessageHandler=${{ matrix.handler }} if [ -n "$Grpc__CaCert" ]; then export Grpc__CaCert=${{ github.workspace }}/$Grpc__CaCert @@ -157,9 +154,6 @@ jobs: if [ -n "$Grpc__ClientKey" ]; then export Grpc__ClientKey=${{ github.workspace }}/$Grpc__ClientKey fi - echo "Grpc__CaCert: $Grpc__CaCert" - echo "Grpc__ClientCert: $Grpc__ClientCert" - echo "Grpc__ClientKey: $Grpc__ClientKey" cd ArmoniK.Api.Client.Test dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$? diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index b0610b1bc..dcc1a3fef 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -46,7 +46,13 @@ public void SetUp() keyPath_ = Environment.GetEnvironmentVariable("Grpc__ClientKey") ?? ""; CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; - endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; + endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint"); + if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || MessageHandler_.ToLower() + .Contains("web")) + { + endpoint_ = endpoint_ ?? "http://localhost:4999"; + } + endpoint_ = endpoint_ ?? "http://localhost:5000"; } [TearDown] From ccf3305ca5bce84fceb2a6af54813b804fa1ca05 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Sat, 12 Oct 2024 14:57:19 +0200 Subject: [PATCH 05/17] Adding script for test in ci --- .../ArmoniK.Api.Client.Test/mock_test.sh | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh diff --git a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh new file mode 100644 index 000000000..ba0ba50f4 --- /dev/null +++ b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +if [ $# -lt 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +script_path="$(dirname "${BASH_SOURCE:-$0}")" +script_dir="$(realpath "$script_path/" )" +working_dir="$(realpath "$script_path/../" )" + +MOCK_ENV=$1 +TEST_ENV=$2 +TEST_COMMAND=$3 + +if [ ! -f "$MOCK_ENV" ]; then + echo "Fichier d'environnement $MOCK_ENV introuvable" + exit 1 +fi + +if [ ! -f "$TEST_ENV" ]; then + echo "Fichier d'environnement $TEST_ENV introuvable" + exit 1 +fi + +set -a +source "$MOCK_ENV" +set +a + +cd $working_dir +set +e +set -x +./out/ArmoniK.Api.Mock.exe & + server_pid=$! + sleep 5 + +set -e + +cd $script_dir +set -a +source "$TEST_ENV" +set +a + +# $TEST_COMMAND +set +e + +echo $server_pid +kill $server_pid From a28e5202ec9d5966c41c86340573c90c515ec99b Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Sat, 12 Oct 2024 15:20:47 +0200 Subject: [PATCH 06/17] Use test script in ci --- .github/workflows/test.yml | 36 +++++++++---------- .../ConnectivityTest.cs | 9 ++--- .../ArmoniK.Api.Client.Test/mock_test.sh | 2 +- 3 files changed, 20 insertions(+), 27 deletions(-) mode change 100644 => 100755 packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 428cec0c6..c0bb262bc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,22 +50,19 @@ jobs: - mock_env: Grpc__Port: 5000 Http__Port: 4999 - test_env: Grpc__Endpoint: http://localhost:4999 - mock_env: Grpc__Port: 5001 Http__Port: 5001 Http__Cert: certs/server1.pem Http__Key: certs/server1.key - test_env: Grpc__Endpoint: https://localhost:5001 - Grpc__CaCert: /home/runner/work/ArmoniK.Api/ArmoniK.Api/packages/csharp/certs/server1-ca.pem + Grpc__CaCert: packages/csharp/certs/server1-ca.pem - mock_env: Grpc__Port: 5002 Http__Port: 5002 Http__Cert: certs/server2.pem Http__Key: certs/server2.key - test_env: Grpc__Endpoint: https://localhost:5002 - mock_env: Grpc__Port: 5003 @@ -73,7 +70,6 @@ jobs: Http__Cert: certs/server1.pem Http__Key: certs/server1.key Http__ClientCert: certs/client-ca.pem - test_env: Grpc__Endpoint: https://localhost:5003 Grpc__CaCert: packages/csharp/certs/server1-ca.pem Grpc__ClientCert: packages/csharp/certs/client.pem @@ -84,7 +80,6 @@ jobs: Http__Cert: certs/server2.pem Http__Key: certs/server2.key Http__ClientCert: certs/client-ca.pem - test_env: Grpc__Endpoint: https://localhost:5004 Grpc__ClientCert: packages/csharp/certs/client.pem Grpc__ClientKey: packages/csharp/certs/client.key @@ -130,19 +125,10 @@ jobs: dotnet publish -o ../out [ -e ../out/ArmoniK.Api.Mock.exe ] || ln -s ArmoniK.Api.Mock ../out/ArmoniK.Api.Mock.exe - - name: Run Mock servers - working-directory: packages/csharp/ - shell: bash - env: ${{ matrix.endpoint.mock_env }} - run: | - ./out/ArmoniK.Api.Mock.exe & - echo "server_pid=$!" >> $GITHUB_ENV - sleep 5 - - name: Run Test working-directory: packages/csharp/ shell: bash - env: ${{ matrix.endpoint.test_env }} + env: ${{ matrix.endpoint.mock_env }} run: | export GrpcClient__HttpMessageHandler=${{ matrix.handler }} if [ -n "$Grpc__CaCert" ]; then @@ -155,9 +141,21 @@ jobs: export Grpc__ClientKey=${{ github.workspace }}/$Grpc__ClientKey fi cd ArmoniK.Api.Client.Test - dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$? - - exit $ret + mock_file="mock_env.env" + test_file="test_env.env" + mock_conf=("Grpc__Port" "Http__Port" "Http__Cert" "Http__Key" "Http__ClientCert") + test_conf=("Grpc__Endpoint" "Grpc__CaCert" "Grpc__ClientCert" "Grpc__ClientKey" "GrpcClient__HttpMessageHandler") + for var in "${test_conf[@]}"; do + if [ -n "${!var}" ]; then + echo "$var=${!var}" >> "$mock_file" + fi + done + for var in "${mock_conf[@]}"; do + if [ -n "${!var}" ]; then + echo "$var=${!var}" >> "$test_file" + fi + done + ./mock_test.sh mock_env.env test_env.env "dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$?" - name: Test Report uses: dorny/test-reporter@v1 diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index dcc1a3fef..8268ab6d7 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -46,13 +46,7 @@ public void SetUp() keyPath_ = Environment.GetEnvironmentVariable("Grpc__ClientKey") ?? ""; CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; - endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint"); - if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || MessageHandler_.ToLower() - .Contains("web")) - { - endpoint_ = endpoint_ ?? "http://localhost:4999"; - } - endpoint_ = endpoint_ ?? "http://localhost:5000"; + endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; } [TearDown] @@ -68,6 +62,7 @@ private static string CertFolder private static string? keyPath_; private static string? CaCertPath_; private static string? MessageHandler_; + private bool isSecure; [Test] public void ResultsGetServiceConfiguration() diff --git a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh old mode 100644 new mode 100755 index ba0ba50f4..89299ad9c --- a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh +++ b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh @@ -41,7 +41,7 @@ set -a source "$TEST_ENV" set +a -# $TEST_COMMAND +$TEST_COMMAND set +e echo $server_pid From f4f2fd984463aa1e02b354e7356ca854717f4dc9 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 14 Oct 2024 10:16:37 +0200 Subject: [PATCH 07/17] Remove test report step --- .github/workflows/test.yml | 23 ++++++++++--------- .../ArmoniK.Api.Client.Test/mock_test.sh | 2 ++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0bb262bc..24382feb9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,14 +50,14 @@ jobs: - mock_env: Grpc__Port: 5000 Http__Port: 4999 - Grpc__Endpoint: http://localhost:4999 + Grpc__Endpoint: http://localhost:5000 - mock_env: Grpc__Port: 5001 Http__Port: 5001 Http__Cert: certs/server1.pem Http__Key: certs/server1.key Grpc__Endpoint: https://localhost:5001 - Grpc__CaCert: packages/csharp/certs/server1-ca.pem + Grpc__CaCert: server1-ca.pem - mock_env: Grpc__Port: 5002 Http__Port: 5002 @@ -71,9 +71,9 @@ jobs: Http__Key: certs/server1.key Http__ClientCert: certs/client-ca.pem Grpc__Endpoint: https://localhost:5003 - Grpc__CaCert: packages/csharp/certs/server1-ca.pem - Grpc__ClientCert: packages/csharp/certs/client.pem - Grpc__ClientKey: packages/csharp/certs/client.key + Grpc__CaCert: server1-ca.pem + Grpc__ClientCert: client.pem + Grpc__ClientKey: client.key - mock_env: Grpc__Port: 5004 Http__Port: 5004 @@ -81,8 +81,8 @@ jobs: Http__Key: certs/server2.key Http__ClientCert: certs/client-ca.pem Grpc__Endpoint: https://localhost:5004 - Grpc__ClientCert: packages/csharp/certs/client.pem - Grpc__ClientKey: packages/csharp/certs/client.key + Grpc__ClientCert: client.pem + Grpc__ClientKey: client.key runs-on: ${{ matrix.platform.os }} steps: @@ -131,14 +131,15 @@ jobs: env: ${{ matrix.endpoint.mock_env }} run: | export GrpcClient__HttpMessageHandler=${{ matrix.handler }} + export CertFolder="$PWD/certs" if [ -n "$Grpc__CaCert" ]; then - export Grpc__CaCert=${{ github.workspace }}/$Grpc__CaCert + export Grpc__CaCert=$CertFolder/$Grpc__CaCert fi if [ -n "$Grpc__ClientCert" ]; then - export Grpc__ClientCert=${{ github.workspace }}/$Grpc__ClientCert + export Grpc__ClientCert=$CertFolder/$Grpc__ClientCert fi if [ -n "$Grpc__ClientKey" ]; then - export Grpc__ClientKey=${{ github.workspace }}/$Grpc__ClientKey + export Grpc__ClientKey=$CertFolder/$Grpc__ClientKey fi cd ArmoniK.Api.Client.Test mock_file="mock_env.env" @@ -155,7 +156,7 @@ jobs: echo "$var=${!var}" >> "$test_file" fi done - ./mock_test.sh mock_env.env test_env.env "dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"; ret=$?" + ./mock_test.sh mock_env.env test_env.env 'dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"' - name: Test Report uses: dorny/test-reporter@v1 diff --git a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh index 89299ad9c..6e701a8df 100755 --- a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh +++ b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh @@ -42,6 +42,8 @@ source "$TEST_ENV" set +a $TEST_COMMAND +ret=$? +exit $ret set +e echo $server_pid From d45b2b0bd2e9f67aedd81249b3363b4f68236d93 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 14 Oct 2024 11:51:38 +0200 Subject: [PATCH 08/17] Fix unsecure connexion tests --- .../ConnectivityTest.cs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index 8268ab6d7..ec2cbcbf9 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -47,6 +47,15 @@ public void SetUp() CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; + isInsecure_ = IsInsecure(endpoint_); + + if (isInsecure_) + { + endpoint_ = RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || MessageHandler_.ToLower() + .Contains("web") + ? "http://localhost:4999" + : endpoint_; + } } [TearDown] @@ -62,7 +71,13 @@ private static string CertFolder private static string? keyPath_; private static string? CaCertPath_; private static string? MessageHandler_; - private bool isSecure; + private bool isInsecure_; + + private static bool IsInsecure(string endpoint) + { + var uri = new Uri(endpoint); + return uri.Scheme == Uri.UriSchemeHttp; + } [Test] public void ResultsGetServiceConfiguration() @@ -71,7 +86,7 @@ public void ResultsGetServiceConfiguration() var channel = GrpcChannelFactory.CreateChannel(new GrpcClient { Endpoint = endpoint_, - AllowUnsafeConnection = true, + AllowUnsafeConnection = isInsecure_, CertPem = certPath_!, KeyPem = keyPath_!, CaCert = CaCertPath_!, @@ -97,7 +112,7 @@ public async Task MultipleChannels([Values(1, i => Task.FromResult(GrpcChannelFactory.CreateChannel(new GrpcClient { Endpoint = endpoint_, - AllowUnsafeConnection = true, + AllowUnsafeConnection = isInsecure_, CertPem = certPath_!, KeyPem = keyPath_!, CaCert = CaCertPath_!, From 85b6c7e30c36e6a9d7fc2be7596b14c6cb30f8a6 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 14 Oct 2024 13:34:33 +0200 Subject: [PATCH 09/17] Remove connectivity kind --- .../ConnectivityKind.cs | 123 ------------------ 1 file changed, 123 deletions(-) delete mode 100644 packages/csharp/ArmoniK.Api.Client.Test/ConnectivityKind.cs diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityKind.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityKind.cs deleted file mode 100644 index d054313e3..000000000 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityKind.cs +++ /dev/null @@ -1,123 +0,0 @@ -// This file is part of the ArmoniK project -// -// Copyright (C) ANEO, 2021-2024. All rights reserved. -// W. Kirschenmann -// J. Gurhem -// D. Dubuc -// L. Ziane Khodja -// F. Lemaitre -// S. Djebbar -// J. Fonseca -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published -// by the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY, without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -using System; -using System.IO; -using System.Runtime.InteropServices; - -using ArmoniK.Api.Client.Options; -using ArmoniK.Api.Client.Submitter; - -using Grpc.Net.Client; - -namespace ArmoniK.Api.Client.Tests; - -public enum ConnectivityKind -{ - Unencrypted, - TlsInsecure, - TlsCert, - TlsStore, - MTlsInsecure, - MTlsCert, - MTlsStore, -} - -internal static class ConnectivityKindExt -{ - private static string CertFolder - => Environment.GetEnvironmentVariable("CertFolder") ?? "../../../../certs"; - - private static string MessageHandler - => Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; - - internal static bool IsTls(this ConnectivityKind kind) - => kind switch - { - ConnectivityKind.Unencrypted => false, - _ => true, - }; - - internal static bool IsInsecure(this ConnectivityKind kind) - => kind switch - { - ConnectivityKind.Unencrypted or ConnectivityKind.TlsInsecure or ConnectivityKind.MTlsInsecure => true, - _ => false, - }; - - internal static bool IsMTls(this ConnectivityKind kind) - => kind switch - { - ConnectivityKind.MTlsInsecure => true, - ConnectivityKind.MTlsCert => true, - ConnectivityKind.MTlsStore => true, - _ => false, - }; - - internal static string? GetCaCertPath(this ConnectivityKind kind) - => kind switch - { - ConnectivityKind.TlsCert or ConnectivityKind.MTlsCert => Path.Combine(CertFolder, - "server1-ca.pem"), - _ => null, - }; - - internal static (string?, string?) GetClientCertPath(this ConnectivityKind kind) - => kind.IsMTls() - ? (Path.Combine(CertFolder, - "client.pem"), Path.Combine(CertFolder, - "client.key")) - : (null, null); - - internal static string GetEndpoint(this ConnectivityKind kind) - => kind switch - { - ConnectivityKind.Unencrypted => RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || MessageHandler.ToLower() - .Contains("web") - ? "http://localhost:4999" - : "http://localhost:5000", - ConnectivityKind.TlsInsecure => "https://localhost:5001", - ConnectivityKind.TlsCert => "https://localhost:5001", - ConnectivityKind.TlsStore => "https://localhost:5002", - ConnectivityKind.MTlsInsecure => "https://localhost:5003", - ConnectivityKind.MTlsCert => "https://localhost:5003", - ConnectivityKind.MTlsStore => "https://localhost:5004", - _ => "http://localhost:4999", - }; - - internal static GrpcChannel GetChannel(this ConnectivityKind kind) - { - var (certPath, keyPath) = kind.GetClientCertPath(); - - return GrpcChannelFactory.CreateChannel(new GrpcClient - { - Endpoint = kind.GetEndpoint(), - AllowUnsafeConnection = kind.IsInsecure(), - CertPem = certPath ?? "", - KeyPem = keyPath ?? "", - CaCert = kind.GetCaCertPath() ?? "", - HttpMessageHandler = MessageHandler, - }); - } -} From e19159012c39975b89caf2b2b292cca48f41b3fd Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 14 Oct 2024 13:38:08 +0200 Subject: [PATCH 10/17] Cleanup code --- .../csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index ec2cbcbf9..650f4661e 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -58,14 +58,6 @@ public void SetUp() } } - [TearDown] - public void TearDown() - { - } - - private static string CertFolder - => Environment.GetEnvironmentVariable("CertFolder") ?? "../../../../certs"; - private static string? endpoint_; private static string? certPath_; private static string? keyPath_; @@ -82,7 +74,6 @@ private static bool IsInsecure(string endpoint) [Test] public void ResultsGetServiceConfiguration() { - Console.WriteLine("Endpoint: " + endpoint_); var channel = GrpcChannelFactory.CreateChannel(new GrpcClient { Endpoint = endpoint_, From c146cd93acd3d11015c5d0d7ca5a93d713797d16 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 21 Oct 2024 14:57:06 +0200 Subject: [PATCH 11/17] Refactor test ci --- .github/workflows/ci.yml | 96 ---- .github/workflows/test.yml | 422 +++++++++--------- packages/cpp/tools/build_test.sh | 7 + packages/cpp/tools/run_test.sh | 15 +- .../ConnectivityTest.cs | 10 +- .../ArmoniK.Api.Client.Test/mock_test.sh | 50 --- scripts/mock_test.sh | 42 ++ 7 files changed, 269 insertions(+), 373 deletions(-) create mode 100755 packages/cpp/tools/build_test.sh delete mode 100755 packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh create mode 100755 scripts/mock_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89c98a0d7..dd95df872 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,102 +264,6 @@ jobs: - name: Build run: nr build - build-test-python: - name: Build and test Python - runs-on: ubuntu-latest - defaults: - run: - working-directory: packages/python - steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - fetch-depth: 0 - - - name: pip update and add build package - run: bash proto2python.sh ~/pyvenv - - - name: Install dependencies - run: pip install "$(echo pkg/armonik*.whl)[tests]" - - - name: Generate certs - working-directory: packages/csharp/ - shell: bash - run: | - mkdir certs - cd certs - ../../../scripts/certs.sh - - name: Install certs - working-directory: packages/csharp/certs - run: | - sudo apt install ca-certificates - sudo mkdir -p /usr/local/share/ca-certificates/ - sudo cp server2-ca.pem /usr/local/share/ca-certificates/ca.crt - sudo update-ca-certificates - - - name: Build Mock server - working-directory: packages/csharp/ArmoniK.Api.Mock - shell: bash - run: | - dotnet publish -o ../out - [ -e ../out/ArmoniK.Api.Mock.exe ] || ln -s ArmoniK.Api.Mock ../out/ArmoniK.Api.Mock.exe - - - name: Test - working-directory: packages/python/ - shell: bash - run: | - set +e - set -x - export CertFolder="$PWD/../csharp/certs" - $PWD/../csharp/out/ArmoniK.Api.Mock.exe \ - grpc:port=5000 http:port=4999 logging:loglevel:default="Warning" \ - & notls_pid=$! - $PWD/../csharp/out/ArmoniK.Api.Mock.exe \ - grpc:port=5001 http:port=5001 logging:loglevel:default="Warning" \ - http:cert="$CertFolder/server1.pem" http:key="$CertFolder/server1.key" \ - & tls_pid=$! - $PWD/../csharp/out/ArmoniK.Api.Mock.exe \ - grpc:port=5002 http:port=5002 logging:loglevel:default="Warning" \ - http:cert="$CertFolder/server2.pem" http:key="$CertFolder/server2.key" \ - & tlsstore_pid=$! - $PWD/../csharp/out/ArmoniK.Api.Mock.exe \ - grpc:port=5003 http:port=5003 logging:loglevel:default="Warning" \ - http:cert="$CertFolder/server1.pem" http:key="$CertFolder/server1.key" http:clientcert="$CertFolder/client-ca.pem" \ - & mtls_pid=$! - $PWD/../csharp/out/ArmoniK.Api.Mock.exe \ - grpc:port=5004 http:port=5004 logging:loglevel:default="Warning" \ - http:cert="$CertFolder/server2.pem" http:key="$CertFolder/server2.key" http:clientcert="$CertFolder/client-ca.pem" \ - & mtlsstore_pid=$! - sleep 5 - set -e - Grpc__Endpoint=http://localhost:5000 Http__Endpoint=http://localhost:4999 pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing - Grpc__Endpoint=https://localhost:5001 Http__Endpoint=https://localhost:5001 Grpc__CaCert="$CertFolder/server1-ca.pem" pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append - Grpc__Endpoint=https://localhost:5002 Http__Endpoint=https://localhost:5002 pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append - Grpc__Endpoint=https://localhost:5003 Http__Endpoint=https://localhost:5003 Grpc__CaCert="$CertFolder/server1-ca.pem" Grpc__ClientCert="$CertFolder/client.pem" Grpc__ClientKey="$CertFolder/client.key" pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append - Grpc__Endpoint=https://localhost:5004 Http__Endpoint=https://localhost:5004 Grpc__ClientCert="$CertFolder/client.pem" Grpc__ClientKey="$CertFolder/client.key" pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append - Grpc__Endpoint=https://localhost:5004 Http__Endpoint=https://localhost:5004 Grpc__ClientCert="$CertFolder/client-client.pem" pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append - Grpc__Endpoint=https://localhost:5004 Http__Endpoint=https://localhost:5004 Grpc__ClientCert="$CertFolder/client.p12" pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append --cov-report xml:coverage.xml --cov-report html:coverage_report - set +e - kill $notls_pid $tls_pid $tlsstore_pid $mtls_pid $mtlsstore_pid - exit $ret - - - name: Get Cover - uses: orgoro/coverage@3f13a558c5af7376496aa4848bf0224aead366ac - with: - coverageFile: packages/python/coverage.xml - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Archive code coverage results html - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a - with: - name: code-coverage-report-html - path: packages/python/coverage_report - - - name: Archive code coverage results xml - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a - with: - name: code-coverage-report-xml - path: packages/python/coverage.xml - build-cpp-packages: strategy: fail-fast: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24382feb9..4677434b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,79 +11,75 @@ concurrency: cancel-in-progress: true jobs: - csharp: - name: Test C# + test: + name: Test API strategy: fail-fast: false matrix: - dotnet: - - version: '' - framework: net4.7 - - version: '' - framework: net4.8 - - version: 6.0 - framework: net6.0 - - version: 8.0 - framework: net8.0 - platform: + language: + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" + dotnet: '' + handler: '' + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" + dotnet: '' + handler: GrpcWebHandler + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" + dotnet: '' + handler: '' + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" + dotnet: '' + handler: GrpcWebHandler + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net6.0 --logger "trx;LogFileName=test-results.trx" + dotnet: 6.0 + handler: '' + - lang: C# + dir: csharp/ArmoniK.Api.Client.Test + cmd: dotnet test -f net8.0 --logger "trx;LogFileName=test-results.trx" + dotnet: 8.0 + handler: '' + - lang: C++ + dir: cpp + cmd: ./tools/run_test.sh + - lang: Python + dir: python + cmd: pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append --cov-report xml:coverage.xml --cov-report html:coverage_report + plateform: - os: ubuntu-latest - runtime: linux-x64 + - os: windows-latest - os: windows-2019 - runtime: windows-x64 - - os: windows-2022 - runtime: windows-x64 - handler: - - '' - - GrpcWebHandler exclude: - - dotnet: - version: '' - platform: + - language: + lang: Python + plateform: + os: windows-latest + - language: + lang: C++ + plateform: + os: windows-latest + - language: + lang: C++ + plateform: + os: windows-2019 + - language: + lang: Python + plateform: + os: windows-2019 + - language: + lang: C# + dotnet: '' + plateform: os: ubuntu-latest - - dotnet: - version: 6.0 - handler: GrpcWebHandler - - dotnet: - version: 8.0 - handler: GrpcWebHandler - endpoint: - - mock_env: - Grpc__Port: 5000 - Http__Port: 4999 - Grpc__Endpoint: http://localhost:5000 - - mock_env: - Grpc__Port: 5001 - Http__Port: 5001 - Http__Cert: certs/server1.pem - Http__Key: certs/server1.key - Grpc__Endpoint: https://localhost:5001 - Grpc__CaCert: server1-ca.pem - - mock_env: - Grpc__Port: 5002 - Http__Port: 5002 - Http__Cert: certs/server2.pem - Http__Key: certs/server2.key - Grpc__Endpoint: https://localhost:5002 - - mock_env: - Grpc__Port: 5003 - Http__Port: 5003 - Http__Cert: certs/server1.pem - Http__Key: certs/server1.key - Http__ClientCert: certs/client-ca.pem - Grpc__Endpoint: https://localhost:5003 - Grpc__CaCert: server1-ca.pem - Grpc__ClientCert: client.pem - Grpc__ClientKey: client.key - - mock_env: - Grpc__Port: 5004 - Http__Port: 5004 - Http__Cert: certs/server2.pem - Http__Key: certs/server2.key - Http__ClientCert: certs/client-ca.pem - Grpc__Endpoint: https://localhost:5004 - Grpc__ClientCert: client.pem - Grpc__ClientKey: client.key - runs-on: ${{ matrix.platform.os }} + runs-on: ${{ matrix.plateform.os }} steps: - name: Checkout @@ -99,7 +95,7 @@ jobs: cd certs ../../../scripts/certs.sh - name: Install certs - if: ${{ matrix.platform.os == 'ubuntu-latest' }} + if: ${{ matrix.plateform.os == 'ubuntu-latest' }} working-directory: packages/csharp/certs run: | sudo apt install ca-certificates @@ -107,16 +103,38 @@ jobs: sudo cp server2-ca.pem /usr/local/share/ca-certificates/ca.crt sudo update-ca-certificates - name: Install certs - if: ${{ contains(matrix.platform.os, 'windows') }} + if: ${{ contains(matrix.plateform.os, 'windows') }} working-directory: packages/csharp/certs run: | certutil -addstore -f "ROOT" server2-ca.pem - name: Install .NET Core uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee # v4 - if: ${{ matrix.dotnet.version }} != "" + if: ${{ matrix.language.dotnet }} != "" with: - dotnet-version: ${{ matrix.dotnet.version }} + dotnet-version: ${{ matrix.language.dotnet }} + + - name: Print language + run: | + echo "Current language: ${{ matrix.language.lang }}" + + - name: pip update and add build package + if: ${{ matrix.language.lang == 'Python' }} + working-directory: packages/python + run: bash proto2python.sh ~/pyvenv + + - name: Install python dependencies + if: ${{ matrix.language.lang == 'Python' }} + working-directory: packages/python + run: | + pwd + pip install "$(echo pkg/armonik*.whl)[tests]" + + - name: Build C++ package + if: ${{ matrix.language.lang == 'C++' }} + working-directory: packages/cpp + run: | + ./tools/build_test.sh - name: Build Mock server working-directory: packages/csharp/ArmoniK.Api.Mock @@ -125,169 +143,141 @@ jobs: dotnet publish -o ../out [ -e ../out/ArmoniK.Api.Mock.exe ] || ln -s ArmoniK.Api.Mock ../out/ArmoniK.Api.Mock.exe - - name: Run Test + - name: noTLS working-directory: packages/csharp/ shell: bash - env: ${{ matrix.endpoint.mock_env }} run: | - export GrpcClient__HttpMessageHandler=${{ matrix.handler }} - export CertFolder="$PWD/certs" - if [ -n "$Grpc__CaCert" ]; then - export Grpc__CaCert=$CertFolder/$Grpc__CaCert - fi - if [ -n "$Grpc__ClientCert" ]; then - export Grpc__ClientCert=$CertFolder/$Grpc__ClientCert - fi - if [ -n "$Grpc__ClientKey" ]; then - export Grpc__ClientKey=$CertFolder/$Grpc__ClientKey - fi - cd ArmoniK.Api.Client.Test - mock_file="mock_env.env" - test_file="test_env.env" - mock_conf=("Grpc__Port" "Http__Port" "Http__Cert" "Http__Key" "Http__ClientCert") - test_conf=("Grpc__Endpoint" "Grpc__CaCert" "Grpc__ClientCert" "Grpc__ClientKey" "GrpcClient__HttpMessageHandler") - for var in "${test_conf[@]}"; do - if [ -n "${!var}" ]; then - echo "$var=${!var}" >> "$mock_file" - fi - done - for var in "${mock_conf[@]}"; do - if [ -n "${!var}" ]; then - echo "$var=${!var}" >> "$test_file" - fi - done - ./mock_test.sh mock_env.env test_env.env 'dotnet test -f ${{ matrix.dotnet.framework }} --logger "trx;LogFileName=test-results.trx"' - - - name: Test Report - uses: dorny/test-reporter@v1 - if: always() - with: - name: ConnectivityTests ${{ matrix.platform.os }} ${{ matrix.dotnet.framework }} ${{ matrix.handler }} - path: ./packages/csharp/ArmoniK.Api.Client.Test/TestResults/test-results.trx - reporter: dotnet-trx - - cpp: - strategy: - fail-fast: false - matrix: - endpoint: - - mock_env: - Grpc__Port: 5000 - Http__Port: 4999 - test_env: - Grpc__Endpoint: http://localhost:5000 - Http__Endpoint: http://localhost:4999 - - mock_env: - Grpc__Port: 5001 - Http__Port: 5001 - Http__Cert: ../cpp/certs/server1.pem - Http__Key: ../cpp/certs/server1.key - test_env: - Grpc__Endpoint: https://localhost:5001 - Grpc__SSLValidation: enable - Grpc__CaCert: /app/source/certs/server1-ca.pem - - mock_env: - Grpc__Port: 5002 - Http__Port: 5002 - Http__Cert: ../cpp/certs/server2.pem - Http__Key: ../cpp/certs/server2.key - test_env: - Grpc__Endpoint: https://localhost:5002 - Grpc__SSLValidation: enable - - mock_env: - Grpc__Port: 5003 - Http__Port: 5003 - Http__Cert: ../cpp/certs/server1.pem - Http__Key: ../cpp/certs/server1.key - Http__ClientCert: ../cpp/certs/client-ca.pem - test_env: - Grpc__Endpoint: https://localhost:5003 - Grpc__SSLValidation: enable - Grpc__mTLS: true - Grpc__CaCert: /app/source/certs/server1-ca.pem - Grpc__ClientCert: /app/source/certs/client.pem - Grpc__ClientKey: /app/source/certs/client.key - - mock_env: - Grpc__Port: 5004 - Http__Port: 5004 - Http__Cert: ../cpp/certs/server2.pem - Http__Key: ../cpp/certs/server2.key - Http__ClientCert: ../cpp/certs/client-ca.pem - test_env: - Grpc__Endpoint: https://localhost:5004 - Grpc__SSLValidation: enable - Grpc__mTLS: true - Grpc__ClientCert: /app/source/certs/client.pem - Grpc__ClientKey: /app/source/certs/client.key - - name: Test C++ - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - ref: ${{ github.ref }} + export Grpc__Port=5000 + export Http__Port=4999 + export Grpc__Endpoint=http://localhost:5000 + export Http__Endpoint=http://localhost:4999 + export Grpc__AllowUnsafeConnection=true + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Generate certs - working-directory: packages/cpp/ + - name: TLS Insecure + if: ${{ matrix.language.lang == 'C#' }} + working-directory: packages/csharp/ shell: bash run: | - mkdir certs - cd certs - ../../../scripts/certs.sh - - - name: Install certs - working-directory: packages/cpp/certs - run: | - sudo apt install ca-certificates - sudo mkdir -p /usr/local/share/ca-certificates/ - sudo cp server2-ca.pem /usr/local/share/ca-certificates/ca.crt - sudo update-ca-certificates - - - name: Install .NET Core - uses: actions/setup-dotnet@4d6c8fcf3c8f7a60068d26b594648e99df24cee3 # v4 - with: - dotnet-version: 8.x + export Grpc__Port=5001 + export Http__Port=5001 + export Http__Cert=certs/server1.pem + export Http__Key=certs/server1.key + export Grpc__Endpoint=https://localhost:5001 + export Http__Endpoint=https://localhost:5001 + export Grpc__AllowUnsafeConnection=true + export Grpc__SSLValidation=enable + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Build Mock server - working-directory: packages/csharp/ArmoniK.Api.Mock + - name: TLS secure + working-directory: packages/csharp/ shell: bash run: | - dotnet publish -o ../out - [ -e ../out/ArmoniK.Api.Mock.exe ] || ln -s ArmoniK.Api.Mock ../out/ArmoniK.Api.Mock.exe + export Grpc__Port=5001 + export Http__Port=5001 + export Http__Cert=certs/server1.pem + export Http__Key=certs/server1.key + export Grpc__Endpoint=https://localhost:5001 + export Http__Endpoint=https://localhost:5001 + export Grpc__AllowUnsafeConnection=false + export Grpc__CaCert=certs/server1-ca.pem + export Grpc__SSLValidation=enable + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Run Mock servers + - name: TLS store working-directory: packages/csharp/ shell: bash - env: ${{ matrix.endpoint.mock_env }} run: | - export CertFolder="$PWD/../cpp/certs" - ./out/ArmoniK.Api.Mock.exe > mock.log 2>&1 & - echo "server_pid=$!" >> $GITHUB_ENV - sleep 5 + export Grpc__Port=5002 + export Http__Port=5002 + export Http__Cert=certs/server2.pem + export Http__Key=certs/server2.key + export Grpc__Endpoint=https://localhost:5002 + export Http__Endpoint=https://localhost:5002 + export Grpc__AllowUnsafeConnection=false + export Grpc__SSLValidation=enable + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Build Test - working-directory: packages/cpp + - name: mTLS Insecure + if: ${{ matrix.language.lang == 'C#' }} + working-directory: packages/csharp/ + shell: bash run: | - ./tools/run_test.sh + export Grpc__Port=5003 + export Http__Port=5003 + export Http__Cert=certs/server1.pem + export Http__Key=certs/server1.key + export Grpc__Endpoint=https://localhost:5003 + export Http__Endpoint=https://localhost:5003 + export Grpc__AllowUnsafeConnection=true + export Grpc__ClientCert=certs/client.pem + export Grpc__ClientKey=certs/client.key + export Grpc__SSLValidation=enable + export Grpc__mTLS=true + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Run Test - env: ${{ matrix.endpoint.test_env }} + - name: mTLS secure + working-directory: packages/csharp/ + shell: bash run: | - echo "$PWD/packages/cpp/certs" - docker run --rm -t --network host -v "$PWD/packages/cpp/certs:/app/source/certs" -v "/usr/local/share/ca-certificates/:/usr/local/share/ca-certificates" \ - -e Grpc__EndPoint="$Grpc__Endpoint" \ - -e Http__EndPoint="${Http__Endpoint:-$Grpc__Endpoint}" \ - ${Grpc__SSLValidation:+-e Grpc__SSLValidation="$Grpc__SSLValidation"} \ - ${Grpc__CaCert:+-e Grpc__CaCert="$Grpc__CaCert"} \ - ${Grpc__mTLS:+-e Grpc__mTLS="$Grpc__mTLS"} \ - ${Grpc__ClientCert:+-e Grpc__ClientCert="$Grpc__ClientCert"} \ - ${Grpc__ClientKey:+-e Grpc__ClientKey="$Grpc__ClientKey"} \ - "armonik-api-cpp:0.1.0" + export Grpc__Port=5003 + export Http__Port=5003 + export Http__Cert=certs/server1.pem + export Http__Key=certs/server1.key + export Http__ClientCert=certs/client-ca.pem + export Grpc__Endpoint=https://localhost:5003 + export Http__Endpoint=https://localhost:5003 + export Grpc__AllowUnsafeConnection=false + export Grpc__CaCert=certs/server1-ca.pem + export Grpc__ClientCert=certs/client.pem + export Grpc__ClientKey=certs/client.key + export Grpc__SSLValidation=enable + export Grpc__mTLS=true + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - - name: Stop Mock Server - if: always() + - name: mTLS store working-directory: packages/csharp/ + shell: bash run: | - kill $server_pid || true - cat mock.log + export Grpc__Port=5004 + export Http__Port=5004 + export Http__Cert=certs/server2.pem + export Http__Key=certs/server2.key + export Http__ClientCert=certs/client-ca.pem + export Grpc__Endpoint=https://localhost:5004 + export Http__Endpoint=https://localhost:5004 + export Grpc__AllowUnsafeConnection=false + export Grpc__ClientCert=certs/client.pem + export Grpc__ClientKey=certs/client.key + export Grpc__SSLValidation=enable + export Grpc__mTLS=true + ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' + + - name: Test Report + uses: dorny/test-reporter@v1 + if: ${{ matrix.language.lang == 'C#' && always() }} + with: + name: ConnectivityTests ${{ matrix.plateform.os }} ${{ matrix.dotnet.framework }} ${{ matrix.language.handler }} + path: ./packages/csharp/ArmoniK.Api.Client.Test/TestResults/test-results.trx + reporter: dotnet-trx + + - name: Get Cover Python + uses: orgoro/coverage@3f13a558c5af7376496aa4848bf0224aead366ac + if: ${{ matrix.language.lang == 'Python' && always() }} + with: + coverageFile: packages/python/coverage.xml + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Archive code coverage results html + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + if: ${{ matrix.language.lang == 'Python' && always() }} + with: + name: code-coverage-report-html + path: packages/python/coverage_report + + - name: Archive code coverage results xml + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + if: ${{ matrix.language.lang == 'Python' && always() }} + with: + name: code-coverage-report-xml + path: packages/python/coverage.xml diff --git a/packages/cpp/tools/build_test.sh b/packages/cpp/tools/build_test.sh new file mode 100755 index 000000000..b829bad2e --- /dev/null +++ b/packages/cpp/tools/build_test.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +script_path="$(dirname "${BASH_SOURCE:-$0}")" +working_dir="$(realpath "${script_path}/../../../" )" +dockerfile="${1:-"${working_dir}/packages/cpp/ArmoniK.Api.Tests/Dockerfile"}" +image_tag="${2:-"armonik-api-cpp:0.1.0"}" +docker build --rm -t "$image_tag" -f "$dockerfile" --progress plain "$working_dir" diff --git a/packages/cpp/tools/run_test.sh b/packages/cpp/tools/run_test.sh index b829bad2e..bda55b3ba 100755 --- a/packages/cpp/tools/run_test.sh +++ b/packages/cpp/tools/run_test.sh @@ -1,7 +1,14 @@ #! /bin/sh +set -x script_path="$(dirname "${BASH_SOURCE:-$0}")" -working_dir="$(realpath "${script_path}/../../../" )" -dockerfile="${1:-"${working_dir}/packages/cpp/ArmoniK.Api.Tests/Dockerfile"}" -image_tag="${2:-"armonik-api-cpp:0.1.0"}" -docker build --rm -t "$image_tag" -f "$dockerfile" --progress plain "$working_dir" +CertFolder="$(realpath "$script_path/../../csharp/certs" )" +docker run --rm -t --network host -v "$CertFolder:/app/source/certs" -v "/usr/local/share/ca-certificates/:/usr/local/share/ca-certificates" \ + -e Grpc__EndPoint="$Grpc__Endpoint" \ + -e Http__EndPoint="${Http__Endpoint:-$Grpc__Endpoint}" \ + ${Grpc__SSLValidation:+-e Grpc__SSLValidation="$Grpc__SSLValidation"} \ + ${Grpc__CaCert:+-e Grpc__CaCert="/app/source/certs/server1-ca.pem"} \ + ${Grpc__mTLS:+-e Grpc__mTLS="$Grpc__mTLS"} \ + ${Grpc__ClientCert:+-e Grpc__ClientCert="/app/source/certs/client.pem"} \ + ${Grpc__ClientKey:+-e Grpc__ClientKey="/app/source/certs/client.key"} \ + "armonik-api-cpp:0.1.0" diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index 650f4661e..2807a5fbb 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -47,7 +47,9 @@ public void SetUp() CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; - isInsecure_ = IsInsecure(endpoint_); + isInsecure_ = Environment.GetEnvironmentVariable("Grpc__AllowUnsafeConnection") == "true" + ? true + : false; if (isInsecure_) { @@ -65,12 +67,6 @@ public void SetUp() private static string? MessageHandler_; private bool isInsecure_; - private static bool IsInsecure(string endpoint) - { - var uri = new Uri(endpoint); - return uri.Scheme == Uri.UriSchemeHttp; - } - [Test] public void ResultsGetServiceConfiguration() { diff --git a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh b/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh deleted file mode 100755 index 6e701a8df..000000000 --- a/packages/csharp/ArmoniK.Api.Client.Test/mock_test.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -if [ $# -lt 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -script_path="$(dirname "${BASH_SOURCE:-$0}")" -script_dir="$(realpath "$script_path/" )" -working_dir="$(realpath "$script_path/../" )" - -MOCK_ENV=$1 -TEST_ENV=$2 -TEST_COMMAND=$3 - -if [ ! -f "$MOCK_ENV" ]; then - echo "Fichier d'environnement $MOCK_ENV introuvable" - exit 1 -fi - -if [ ! -f "$TEST_ENV" ]; then - echo "Fichier d'environnement $TEST_ENV introuvable" - exit 1 -fi - -set -a -source "$MOCK_ENV" -set +a - -cd $working_dir -set +e -set -x -./out/ArmoniK.Api.Mock.exe & - server_pid=$! - sleep 5 - -set -e - -cd $script_dir -set -a -source "$TEST_ENV" -set +a - -$TEST_COMMAND -ret=$? -exit $ret -set +e - -echo $server_pid -kill $server_pid diff --git a/scripts/mock_test.sh b/scripts/mock_test.sh new file mode 100755 index 000000000..2dccffbb9 --- /dev/null +++ b/scripts/mock_test.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +if [ $# -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +script_path="$(dirname "${BASH_SOURCE:-$0}")" +working_dir="$(realpath "$script_path/../packages" )" + +TEST_DIR=$1 +TEST_COMMAND=$2 + +if [ -n "$Grpc__CaCert" ]; then + export Grpc__CaCert=$working_dir/csharp/$Grpc__CaCert +fi +if [ -n "$Grpc__ClientCert" ]; then + export Grpc__ClientCert=$working_dir/csharp/$Grpc__ClientCert +fi +if [ -n "$Grpc__ClientKey" ]; then + export Grpc__ClientKey=$working_dir/csharp/$Grpc__ClientKey +fi + +cd $working_dir/csharp +set +e +set -x +./out/ArmoniK.Api.Mock.exe & + server_pid=$! + sleep 5 + +set -e + +cd $working_dir +cd $TEST_DIR + +$TEST_COMMAND +ret=$? +exit $ret +set +e + +echo $server_pid +kill $server_pid From 5fc814e2a8d84151efafd6878befc2f4dcf031c8 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Mon, 4 Nov 2024 16:32:08 +0100 Subject: [PATCH 12/17] Reading options usin configuration object --- .github/workflows/test.yml | 32 ++++++++--- .../ArmoniK.Api.Client.Tests.csproj | 13 +++-- .../ConnectivityTest.cs | 53 ++++++------------- scripts/mock_test.sh | 2 +- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4677434b3..efdfa3139 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,8 @@ jobs: export Http__Port=4999 export Grpc__Endpoint=http://localhost:5000 export Http__Endpoint=http://localhost:4999 - export Grpc__AllowUnsafeConnection=true + export GrpcClient__Endpoint=http://localhost:5000 + export GrpcClient__AllowUnsafeConnection=true ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' - name: TLS Insecure @@ -166,6 +167,8 @@ jobs: export Grpc__Endpoint=https://localhost:5001 export Http__Endpoint=https://localhost:5001 export Grpc__AllowUnsafeConnection=true + export GrpcClient__Endpoint=https://localhost:5001 + export GrpcClient__AllowUnsafeConnection=true export Grpc__SSLValidation=enable ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' @@ -173,14 +176,17 @@ jobs: working-directory: packages/csharp/ shell: bash run: | + export CertFolder="$PWD/certs" export Grpc__Port=5001 export Http__Port=5001 export Http__Cert=certs/server1.pem export Http__Key=certs/server1.key export Grpc__Endpoint=https://localhost:5001 export Http__Endpoint=https://localhost:5001 - export Grpc__AllowUnsafeConnection=false export Grpc__CaCert=certs/server1-ca.pem + export GrpcClient__Endpoint=https://localhost:5001 + export GrpcClient__AllowUnsafeConnection=false + export GrpcClient__CaCert=$CertFolder/server1-ca.pem export Grpc__SSLValidation=enable ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' @@ -194,7 +200,8 @@ jobs: export Http__Key=certs/server2.key export Grpc__Endpoint=https://localhost:5002 export Http__Endpoint=https://localhost:5002 - export Grpc__AllowUnsafeConnection=false + export GrpcClient__Endpoint=https://localhost:5002 + export GrpcClient__AllowUnsafeConnection=false export Grpc__SSLValidation=enable ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' @@ -203,15 +210,19 @@ jobs: working-directory: packages/csharp/ shell: bash run: | + export CertFolder="$PWD/certs" export Grpc__Port=5003 export Http__Port=5003 export Http__Cert=certs/server1.pem export Http__Key=certs/server1.key export Grpc__Endpoint=https://localhost:5003 export Http__Endpoint=https://localhost:5003 - export Grpc__AllowUnsafeConnection=true export Grpc__ClientCert=certs/client.pem export Grpc__ClientKey=certs/client.key + export GrpcClient__Endpoint=https://localhost:5003 + export GrpcClient__AllowUnsafeConnection=true + export GrpcClient__CertPem=$CertFolder/client.pem + export GrpcClient__KeyPem=$CertFolder/client.key export Grpc__SSLValidation=enable export Grpc__mTLS=true ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' @@ -220,6 +231,7 @@ jobs: working-directory: packages/csharp/ shell: bash run: | + export CertFolder="$PWD/certs" export Grpc__Port=5003 export Http__Port=5003 export Http__Cert=certs/server1.pem @@ -227,10 +239,14 @@ jobs: export Http__ClientCert=certs/client-ca.pem export Grpc__Endpoint=https://localhost:5003 export Http__Endpoint=https://localhost:5003 - export Grpc__AllowUnsafeConnection=false export Grpc__CaCert=certs/server1-ca.pem export Grpc__ClientCert=certs/client.pem export Grpc__ClientKey=certs/client.key + export GrpcClient__Endpoint=https://localhost:5003 + export GrpcClient__AllowUnsafeConnection=false + export GrpcClient__CaCert=$CertFolder/server1-ca.pem + export GrpcClient__CertPem=$CertFolder/client.pem + export GrpcClient__KeyPem=$CertFolder/client.key export Grpc__SSLValidation=enable export Grpc__mTLS=true ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' @@ -239,6 +255,7 @@ jobs: working-directory: packages/csharp/ shell: bash run: | + export CertFolder="$PWD/certs" export Grpc__Port=5004 export Http__Port=5004 export Http__Cert=certs/server2.pem @@ -246,9 +263,12 @@ jobs: export Http__ClientCert=certs/client-ca.pem export Grpc__Endpoint=https://localhost:5004 export Http__Endpoint=https://localhost:5004 - export Grpc__AllowUnsafeConnection=false export Grpc__ClientCert=certs/client.pem export Grpc__ClientKey=certs/client.key + export GrpcClient__Endpoint=https://localhost:5004 + export GrpcClient__AllowUnsafeConnection=false + export GrpcClient__CertPem=$CertFolder/client.pem + export GrpcClient__KeyPem=$CertFolder/client.key export Grpc__SSLValidation=enable export Grpc__mTLS=true ../../scripts/mock_test.sh ${{ matrix.language.dir }} '${{ matrix.language.cmd }}' diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ArmoniK.Api.Client.Tests.csproj b/packages/csharp/ArmoniK.Api.Client.Test/ArmoniK.Api.Client.Tests.csproj index b8f03cdc7..038ea58e2 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ArmoniK.Api.Client.Tests.csproj +++ b/packages/csharp/ArmoniK.Api.Client.Test/ArmoniK.Api.Client.Tests.csproj @@ -20,17 +20,20 @@ - - - + + + + + + - + - + diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index 2807a5fbb..d6c535f04 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -32,6 +32,8 @@ using ArmoniK.Api.gRPC.V1.Results; using ArmoniK.Utils; +using Microsoft.Extensions.Configuration; + using NUnit.Framework; namespace ArmoniK.Api.Client.Tests; @@ -42,43 +44,26 @@ public class ConnectivityTests [SetUp] public void SetUp() { - certPath_ = Environment.GetEnvironmentVariable("Grpc__ClientCert") ?? ""; - keyPath_ = Environment.GetEnvironmentVariable("Grpc__ClientKey") ?? ""; - CaCertPath_ = Environment.GetEnvironmentVariable("Grpc__CaCert") ?? ""; - MessageHandler_ = Environment.GetEnvironmentVariable("GrpcClient__HttpMessageHandler") ?? ""; - endpoint_ = Environment.GetEnvironmentVariable("Grpc__Endpoint") ?? ""; - isInsecure_ = Environment.GetEnvironmentVariable("Grpc__AllowUnsafeConnection") == "true" - ? true - : false; - - if (isInsecure_) + var builder = new ConfigurationBuilder().AddEnvironmentVariables(); + var configuration = builder.Build(); + options_ = configuration.GetRequiredSection(GrpcClient.SettingSection) + .Get(); + if (options_!.AllowUnsafeConnection) { - endpoint_ = RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || MessageHandler_.ToLower() - .Contains("web") - ? "http://localhost:4999" - : endpoint_; + if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || options_.HttpMessageHandler.ToLower() + .Contains("web")) + { + options_!.Endpoint = Environment.GetEnvironmentVariable("Http__Endpoint"); + } } } - private static string? endpoint_; - private static string? certPath_; - private static string? keyPath_; - private static string? CaCertPath_; - private static string? MessageHandler_; - private bool isInsecure_; + private GrpcClient? options_; [Test] public void ResultsGetServiceConfiguration() { - var channel = GrpcChannelFactory.CreateChannel(new GrpcClient - { - Endpoint = endpoint_, - AllowUnsafeConnection = isInsecure_, - CertPem = certPath_!, - KeyPem = keyPath_!, - CaCert = CaCertPath_!, - HttpMessageHandler = MessageHandler_!, - }); + var channel = GrpcChannelFactory.CreateChannel(options_!); var resultClient = new Results.ResultsClient(channel); @@ -96,15 +81,7 @@ public async Task MultipleChannels([Values(1, var channels = await Enumerable.Range(0, concurrency) .ParallelSelect(new ParallelTaskOptions(-1), - i => Task.FromResult(GrpcChannelFactory.CreateChannel(new GrpcClient - { - Endpoint = endpoint_, - AllowUnsafeConnection = isInsecure_, - CertPem = certPath_!, - KeyPem = keyPath_!, - CaCert = CaCertPath_!, - HttpMessageHandler = MessageHandler_!, - }))) + i => Task.FromResult(GrpcChannelFactory.CreateChannel(options_!))) .ToListAsync() .ConfigureAwait(false); diff --git a/scripts/mock_test.sh b/scripts/mock_test.sh index 2dccffbb9..c4b67f445 100755 --- a/scripts/mock_test.sh +++ b/scripts/mock_test.sh @@ -35,8 +35,8 @@ cd $TEST_DIR $TEST_COMMAND ret=$? -exit $ret set +e echo $server_pid kill $server_pid +exit $ret From 30f0b5e856ebf75960bf374f5dcd7a50c71c2bd8 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Wed, 6 Nov 2024 09:31:48 +0100 Subject: [PATCH 13/17] Cleanup code --- .../csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index d6c535f04..d3566dfb4 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -48,13 +48,10 @@ public void SetUp() var configuration = builder.Build(); options_ = configuration.GetRequiredSection(GrpcClient.SettingSection) .Get(); - if (options_!.AllowUnsafeConnection) + if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || options_.HttpMessageHandler.ToLower() + .Contains("web")) { - if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || options_.HttpMessageHandler.ToLower() - .Contains("web")) - { - options_!.Endpoint = Environment.GetEnvironmentVariable("Http__Endpoint"); - } + options_!.Endpoint = Environment.GetEnvironmentVariable("Http__Endpoint"); } } From 2251e886571e32a74ab01904de6ef14125b7d77f Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Wed, 6 Nov 2024 11:55:46 +0100 Subject: [PATCH 14/17] Using http handler in ci --- .github/workflows/test.yml | 11 +++++------ packages/cpp/tools/run_test.sh | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efdfa3139..5c2fb54b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,32 +21,30 @@ jobs: dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" dotnet: '' - handler: '' - lang: C# dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" dotnet: '' - handler: GrpcWebHandler + env: + GrpcClient__HttpMessageHandler: GrpcWebHandler - lang: C# dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" dotnet: '' - handler: '' - lang: C# dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" dotnet: '' - handler: GrpcWebHandler + env: + GrpcClient__HttpMessageHandler: GrpcWebHandler - lang: C# dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net6.0 --logger "trx;LogFileName=test-results.trx" dotnet: 6.0 - handler: '' - lang: C# dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net8.0 --logger "trx;LogFileName=test-results.trx" dotnet: 8.0 - handler: '' - lang: C++ dir: cpp cmd: ./tools/run_test.sh @@ -80,6 +78,7 @@ jobs: plateform: os: ubuntu-latest runs-on: ${{ matrix.plateform.os }} + env: ${{ matrix.language.env }} steps: - name: Checkout diff --git a/packages/cpp/tools/run_test.sh b/packages/cpp/tools/run_test.sh index bda55b3ba..1c4ef1f21 100755 --- a/packages/cpp/tools/run_test.sh +++ b/packages/cpp/tools/run_test.sh @@ -4,11 +4,11 @@ set -x script_path="$(dirname "${BASH_SOURCE:-$0}")" CertFolder="$(realpath "$script_path/../../csharp/certs" )" docker run --rm -t --network host -v "$CertFolder:/app/source/certs" -v "/usr/local/share/ca-certificates/:/usr/local/share/ca-certificates" \ - -e Grpc__EndPoint="$Grpc__Endpoint" \ + -e Grpc__EndPoint \ -e Http__EndPoint="${Http__Endpoint:-$Grpc__Endpoint}" \ - ${Grpc__SSLValidation:+-e Grpc__SSLValidation="$Grpc__SSLValidation"} \ + ${Grpc__SSLValidation:+-e Grpc__SSLValidation} \ ${Grpc__CaCert:+-e Grpc__CaCert="/app/source/certs/server1-ca.pem"} \ - ${Grpc__mTLS:+-e Grpc__mTLS="$Grpc__mTLS"} \ + ${Grpc__mTLS:+-e Grpc__mTLS} \ ${Grpc__ClientCert:+-e Grpc__ClientCert="/app/source/certs/client.pem"} \ ${Grpc__ClientKey:+-e Grpc__ClientKey="/app/source/certs/client.key"} \ "armonik-api-cpp:0.1.0" From 0d9140c7981774186d591c585b2313b091eeceb5 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Wed, 6 Nov 2024 12:01:04 +0100 Subject: [PATCH 15/17] Rorder dir and cmd --- .github/workflows/test.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c2fb54b4..969cd350e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,39 +18,45 @@ jobs: matrix: language: - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: '' + env: [] - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: '' env: GrpcClient__HttpMessageHandler: GrpcWebHandler - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: '' + env: [] - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: '' env: GrpcClient__HttpMessageHandler: GrpcWebHandler - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net6.0 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: 6.0 + env: [] - lang: C# - dir: csharp/ArmoniK.Api.Client.Test cmd: dotnet test -f net8.0 --logger "trx;LogFileName=test-results.trx" + dir: csharp/ArmoniK.Api.Client.Test dotnet: 8.0 + env: [] - lang: C++ - dir: cpp cmd: ./tools/run_test.sh + dir: cpp + env: [] - lang: Python - dir: python cmd: pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append --cov-report xml:coverage.xml --cov-report html:coverage_report + dir: python + env: [] plateform: - os: ubuntu-latest - os: windows-latest From aeba776d9d949406d826fcec07b0effdc01f935d Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Wed, 6 Nov 2024 12:10:18 +0100 Subject: [PATCH 16/17] Cleanup scripts --- .github/workflows/test.yml | 12 ++++++------ packages/cpp/tools/run_test.sh | 2 +- scripts/mock_test.sh | 30 ++++++++++++------------------ 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 969cd350e..1ee97ed19 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test dotnet: '' - env: [] + env: {} - lang: C# cmd: dotnet test -f net4.7 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test @@ -32,7 +32,7 @@ jobs: cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test dotnet: '' - env: [] + env: {} - lang: C# cmd: dotnet test -f net4.8 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test @@ -43,20 +43,20 @@ jobs: cmd: dotnet test -f net6.0 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test dotnet: 6.0 - env: [] + env: {} - lang: C# cmd: dotnet test -f net8.0 --logger "trx;LogFileName=test-results.trx" dir: csharp/ArmoniK.Api.Client.Test dotnet: 8.0 - env: [] + env: {} - lang: C++ cmd: ./tools/run_test.sh dir: cpp - env: [] + env: {} - lang: Python cmd: pytest tests --cov=armonik --cov-config=.coveragerc --cov-report=term-missing --cov-append --cov-report xml:coverage.xml --cov-report html:coverage_report dir: python - env: [] + env: {} plateform: - os: ubuntu-latest - os: windows-latest diff --git a/packages/cpp/tools/run_test.sh b/packages/cpp/tools/run_test.sh index 1c4ef1f21..6715a22d5 100755 --- a/packages/cpp/tools/run_test.sh +++ b/packages/cpp/tools/run_test.sh @@ -4,7 +4,7 @@ set -x script_path="$(dirname "${BASH_SOURCE:-$0}")" CertFolder="$(realpath "$script_path/../../csharp/certs" )" docker run --rm -t --network host -v "$CertFolder:/app/source/certs" -v "/usr/local/share/ca-certificates/:/usr/local/share/ca-certificates" \ - -e Grpc__EndPoint \ + -e Grpc__EndPoint="$Grpc__Endpoint" \ -e Http__EndPoint="${Http__Endpoint:-$Grpc__Endpoint}" \ ${Grpc__SSLValidation:+-e Grpc__SSLValidation} \ ${Grpc__CaCert:+-e Grpc__CaCert="/app/source/certs/server1-ca.pem"} \ diff --git a/scripts/mock_test.sh b/scripts/mock_test.sh index c4b67f445..06e32817c 100755 --- a/scripts/mock_test.sh +++ b/scripts/mock_test.sh @@ -1,4 +1,6 @@ -#!/bin/bash +#! /bin/sh + +set -x if [ $# -lt 2 ]; then echo "Usage: $0 " @@ -8,34 +10,26 @@ fi script_path="$(dirname "${BASH_SOURCE:-$0}")" working_dir="$(realpath "$script_path/../packages" )" -TEST_DIR=$1 -TEST_COMMAND=$2 +TEST_DIR="${1:?Test dir is not set}" +TEST_COMMAND="${2:?Command is not set}" if [ -n "$Grpc__CaCert" ]; then - export Grpc__CaCert=$working_dir/csharp/$Grpc__CaCert + export Grpc__CaCert="$working_dir/csharp/$Grpc__CaCert" fi if [ -n "$Grpc__ClientCert" ]; then - export Grpc__ClientCert=$working_dir/csharp/$Grpc__ClientCert + export Grpc__ClientCert="$working_dir/csharp/$Grpc__ClientCert" fi if [ -n "$Grpc__ClientKey" ]; then - export Grpc__ClientKey=$working_dir/csharp/$Grpc__ClientKey + export Grpc__ClientKey="$working_dir/csharp/$Grpc__ClientKey" fi -cd $working_dir/csharp -set +e -set -x -./out/ArmoniK.Api.Mock.exe & +"$working_dir/csharp/out/ArmoniK.Api.Mock.exe" & server_pid=$! - sleep 5 - -set -e +sleep 5 -cd $working_dir -cd $TEST_DIR +cd "$working_dir/$TEST_DIR" -$TEST_COMMAND -ret=$? -set +e +$TEST_COMMAND || ret=$? echo $server_pid kill $server_pid From ac97add67afd13d7f162f4c4f213c5c2c8d2f724 Mon Sep 17 00:00:00 2001 From: Daouda Diakite Date: Wed, 6 Nov 2024 16:03:39 +0100 Subject: [PATCH 17/17] PR comments --- packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs index d3566dfb4..e60f8b101 100644 --- a/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs +++ b/packages/csharp/ArmoniK.Api.Client.Test/ConnectivityTest.cs @@ -47,7 +47,7 @@ public void SetUp() var builder = new ConfigurationBuilder().AddEnvironmentVariables(); var configuration = builder.Build(); options_ = configuration.GetRequiredSection(GrpcClient.SettingSection) - .Get(); + .Get()!; if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") || options_.HttpMessageHandler.ToLower() .Contains("web")) {