Merge pull request #45 from Crypto-TII/performance-tests

Performance tests

.github/workflows/generate-and-submit-documentation.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         persist-credentials: false
         fetch-depth: 0

.github/workflows/run-benchmark-tests.yaml

@@ -0,0 +1,42 @@
+name: Run benchmark tests
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened, edited ]
+    branches:
+      - develop
+      - main
+
+concurrency:
+  group: run-benchmark-tests_${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-benchmark-tests:
+    runs-on: self-hosted
+    timeout-minutes: 3600
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+    - name: Print GITHUB_WORKSPACE
+      run: |
+          echo ${GITHUB_WORKSPACE}
+          pwd
+          ls -lah
+    - name: Fix Directory Structure
+      run: |
+          mkdir /home/runner/_work/claasp_backup
+          mv -f /home/runner/_work/claasp/claasp/* /home/runner/_work/claasp_backup
+          rm -rf /home/runner/_work/claasp/
+          mkdir /home/runner/_work/claasp
+          mv -f /home/runner/_work/claasp_backup/* /home/runner/_work/claasp
+          chmod g+w /home/runner/_work/claasp/ -R
+          rm -rf /home/runner/_work/claasp_backup
+
+    - name: Run pytest-benchmark
+      run: |
+            cd /home/runner/_work/claasp
+            echo Running Benchmark tests
+            make benchmark-tests

.github/workflows/run-pytest-and-sonarcloud-scan.yaml

@@ -11,7 +11,7 @@ on:
       - main
 
 concurrency:
-  group: ${{ github.ref }}
+  group: run-pytest-tests_${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -49,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         persist-credentials: false
         fetch-depth: 0

.github/workflows/update-changelog.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
           persist-credentials: false

.gitignore

@@ -47,6 +47,9 @@ coverage.xml
 *,cover
 .hypothesis/
 
+# Benchmark results
+.benchmarks/
+
 # Translations
 *.mo
 *.pot

Makefile

@@ -41,13 +41,16 @@ develop:
 	$(SAGE_BIN) -pip install --upgrade -e .
 
 remote-pytest:
-	pytest -v -n=auto --dist loadfile --cov-report xml:coverage.xml --cov=$(PACKAGE) tests/
+	pytest -v -n=auto --dist loadfile --cov-report xml:coverage.xml --cov=$(PACKAGE) tests/unit/
 
 pytest:
-	pytest -v -n=auto --dist loadfile tests/
+	pytest -v -n=auto --dist loadfile tests/unit/
 
 pytest-coverage:
-	pytest -v -n=2 --dist loadfile --cov-report term-missing --cov=$(PACKAGE) tests/
+	pytest -v -n=2 --dist loadfile --cov-report term-missing --cov=$(PACKAGE) tests/unit/
+
+benchmark-tests:
+	pytest -v tests/benchmark/
 
 testfast:
 	$(SAGE_BIN) setup.py testfast

README.md

@@ -38,7 +38,7 @@ These instructions allow to generate:
 
 ## Source code
 
-All source code is stored in the folder ``claasp/``.
+All source code is stored in the folder `claasp/`.
 
 ## Contributing
 We want you to help us grow this library, so, please, feel free to submit your Pull Request following the 

claasp/cipher_modules/code_generator.py

@@ -779,7 +779,7 @@ def generate_word_based_c_code(cipher, word_size, intermediate_output, verbosity
         code.append('\tchar *str;')
     code.extend(get_rounds_word_based_c_code(cipher, intermediate_output, verbosity, word_size))
     code.append('}')
-    code.append('int main(int argc, char *argv[]) {{')
+    code.append('int main(int argc, char *argv[]) {')
     evaluate_args = []
     for i in range(len(cipher.inputs)):
         evaluate_args.append(cipher.inputs[i])

docker/Dockerfile

@@ -83,7 +83,8 @@ RUN sage -pip install bitstring==4.0.1 \
   tensorflow==2.11.0 \
   pytest==7.2.1 \
   pytest-cov==4.0.0 \
-  pytest-xdist==3.2.0
+  pytest-xdist==3.2.0 \
+  pytest-benchmark==4.0.0
 
 # Installing nist sts
 RUN curl -O -s https://csrc.nist.gov/CSRC/media/Projects/Random-Bit-Generation/documents/sts-2_1_2.zip \

sonar-project.properties

@@ -6,7 +6,7 @@ sonar.organization=crypto-tii
 ## Project config
 sonar.sources=claasp
 sonar.exclusions=claasp/ciphers/toys/**
-sonar.tests=tests
+sonar.tests=tests/unit
 
 ## C/C++ config
 sonar.c.file.suffixes=-

tests/benchmark/cipher_test.py

@@ -0,0 +1,60 @@
+import pytest
+import numpy as np
+
+from claasp.ciphers.block_ciphers.aes_block_cipher import AESBlockCipher
+from claasp.ciphers.block_ciphers.speck_block_cipher import SpeckBlockCipher
+
+speck = SpeckBlockCipher()
+aes = AESBlockCipher()
+
+
+@pytest.mark.parametrize("number_of_samples", [10, 100, 1000, 10000])
+def test_diffusion_tests_with_speck_cipher(benchmark, number_of_samples):
+    benchmark(speck.diffusion_tests, number_of_samples=number_of_samples)
+
+
+@pytest.mark.parametrize("number_of_samples", [10, 100, 1000, 10000])
+def test_diffusion_tests_with_aes_cipher(benchmark, number_of_samples):
+    benchmark(aes.diffusion_tests, number_of_samples=number_of_samples)
+
+
+def test_evaluate_with_speck_cipher(benchmark):
+    benchmark(speck.evaluate, [0x01234567, 0x89ABCDEF])
+
+
+def test_evaluate_with_aes_cipher(benchmark):
+    benchmark(aes.evaluate, [0x01234567, 0x89ABCDEF])
+
+
+def test_evaluate_using_c_with_speck_cipher(benchmark):
+    benchmark(speck.evaluate_using_c, [0x012345, 0x89ABCD], True)
+
+
+def test_evaluate_using_c_with_aes_cipher(benchmark):
+    benchmark(aes.evaluate_using_c, [0x012345, 0x89ABCD], True)
+
+
+cipher_inputs_parameter_values = [[np.random.randint(256, size=(8, 2), dtype=np.uint8) for _ in range(10)],
+                                  [np.random.randint(256, size=(8, 2), dtype=np.uint8) for _ in range(100)],
+                                  [np.random.randint(256, size=(8, 2), dtype=np.uint8) for _ in range(10000)],
+                                  [np.random.randint(256, size=(8, 2), dtype=np.uint8) for _ in range(1000000)]]
+
+
+@pytest.mark.parametrize("cipher_input", cipher_inputs_parameter_values)
+def test_evaluate_vectorized_with_speck_cipher(benchmark, cipher_input):
+    benchmark(speck.evaluate_vectorized, cipher_input)
+
+
+@pytest.mark.parametrize("cipher_input", cipher_inputs_parameter_values)
+def test_evaluate_vectorized_with_aes_cipher(benchmark, cipher_input):
+    benchmark(aes.evaluate_vectorized, cipher_input)
+
+
+def test_neural_network_blackbox_distinguisher_tests_with_speck_cipher(benchmark):
+    benchmark(speck.neural_network_blackbox_distinguisher_tests, nb_samples=10, hidden_layers=[32, 32, 32],
+              number_of_epochs=[1, 10, 100])
+
+
+def test_neural_network_blackbox_distinguisher_tests_with_aes_cipher(benchmark):
+    benchmark(aes.neural_network_blackbox_distinguisher_tests, nb_samples=10, hidden_layers=[32, 32, 32],
+              number_of_epochs=[1, 10, 100])

tests/benchmark/sat_xor_differential_model_test.py

@@ -0,0 +1,83 @@
+from claasp.ciphers.block_ciphers.aes_block_cipher import AESBlockCipher
+from claasp.ciphers.block_ciphers.speck_block_cipher import SpeckBlockCipher
+from claasp.cipher_modules.models.utils import set_fixed_variables, integer_to_bit_list
+from claasp.cipher_modules.models.sat.sat_models.sat_xor_differential_model import SatXorDifferentialModel
+
+speck = SpeckBlockCipher()
+aes = AESBlockCipher()
+
+
+def test_build_xor_differential_trail_model_with_speck_cipher(benchmark):
+    sat = SatXorDifferentialModel(speck)
+    plaintext = set_fixed_variables(
+        component_id='plaintext',
+        constraint_type='not_equal',
+        bit_positions=range(32),
+        bit_values=integer_to_bit_list(0, 32, 'big'))
+    key = set_fixed_variables(
+        component_id='key',
+        constraint_type='equal',
+        bit_positions=range(64),
+        bit_values=(0,) * 64)
+    benchmark(sat.build_xor_differential_trail_model, 3, fixed_variables=[plaintext, key])
+
+
+def test_build_xor_differential_trail_model_with_aes_cipher(benchmark):
+    sat = SatXorDifferentialModel(aes)
+    plaintext = set_fixed_variables(
+        component_id='plaintext',
+        constraint_type='not_equal',
+        bit_positions=range(32),
+        bit_values=integer_to_bit_list(0, 32, 'big'))
+    key = set_fixed_variables(
+        component_id='key',
+        constraint_type='equal',
+        bit_positions=range(64),
+        bit_values=(0,) * 64)
+    benchmark(sat.build_xor_differential_trail_model, 3, fixed_variables=[plaintext, key])
+
+
+def test_find_all_xor_differential_trails_with_fixed_weight_with_speck_cipher(benchmark):
+    sat = SatXorDifferentialModel(speck, window_size_weight_pr_vars=1)
+    plaintext = set_fixed_variables(component_id='plaintext', constraint_type='not_equal',
+                                    bit_positions=range(32), bit_values=(0,) * 32)
+    key = set_fixed_variables(component_id='key', constraint_type='equal',
+                              bit_positions=range(64), bit_values=(0,) * 64)
+    benchmark(sat.find_all_xor_differential_trails_with_fixed_weight, 9, fixed_values=[plaintext, key])
+
+
+def test_find_all_xor_differential_trails_with_fixed_weight_with_aes_cipher(benchmark):
+    sat = SatXorDifferentialModel(aes, window_size_weight_pr_vars=1)
+    plaintext = set_fixed_variables(component_id='plaintext', constraint_type='not_equal',
+                                    bit_positions=range(32), bit_values=(0,) * 32)
+    key = set_fixed_variables(component_id='key', constraint_type='equal',
+                              bit_positions=range(64), bit_values=(0,) * 64)
+    benchmark(sat.find_all_xor_differential_trails_with_fixed_weight, 9, fixed_values=[plaintext, key])
+
+
+def test_find_lowest_weight_xor_differential_trail_with_speck_cipher(benchmark):
+    speck = SpeckBlockCipher(number_of_rounds=7)
+    sat = SatXorDifferentialModel(speck)
+    plaintext = set_fixed_variables(component_id='plaintext', constraint_type='not_equal',
+                                    bit_positions=range(32), bit_values=(0,) * 32)
+    key = set_fixed_variables(component_id='key', constraint_type='equal',
+                              bit_positions=range(64), bit_values=(0,) * 64)
+    benchmark(sat.find_lowest_weight_xor_differential_trail, fixed_values=[plaintext, key])
+
+
+def test_find_one_xor_differential_trail_with_fixed_weight(benchmark):
+    sat = SatXorDifferentialModel(speck, window_size=0)
+    plaintext = set_fixed_variables(component_id='plaintext', constraint_type='not_equal',
+                                    bit_positions=range(32), bit_values=(0,) * 32)
+    key = set_fixed_variables(component_id='key', constraint_type='equal',
+                              bit_positions=range(64), bit_values=(0,) * 64)
+    benchmark(sat.find_one_xor_differential_trail_with_fixed_weight, 3, fixed_values=[plaintext, key])
+
+
+def test_find_one_xor_differential_trail_with_fixed_weight_with_aes_cipher(benchmark):
+    sat = SatXorDifferentialModel(aes, window_size=0)
+    plaintext = set_fixed_variables(component_id='plaintext', constraint_type='not_equal',
+                                    bit_positions=range(32), bit_values=(0,) * 32)
+    key = set_fixed_variables(component_id='key', constraint_type='equal',
+                              bit_positions=range(64), bit_values=(0,) * 64)
+    benchmark(sat.find_one_xor_differential_trail_with_fixed_weight, 3, fixed_values=[plaintext, key])

tests/benchmark/statistical_tests_test.py

@@ -0,0 +1,18 @@
+import pytest
+
+from claasp.ciphers.block_ciphers.aes_block_cipher import AESBlockCipher
+from claasp.ciphers.block_ciphers.speck_block_cipher import SpeckBlockCipher
+from claasp.cipher_modules.statistical_tests.nist_statistical_tests import StatisticalTests
+
+speck = SpeckBlockCipher()
+aes = AESBlockCipher()
+
+
+def test_run_avalanche_nist_statistics_test_with_speck_cipher(benchmark):
+    tests = StatisticalTests(speck)
+    benchmark(tests.run_avalanche_nist_statistics_test, 0, 10, 10)
+
+
+def test_run_avalanche_nist_statistics_test_with_aes_cipher(benchmark):
+    tests = StatisticalTests(aes)
+    benchmark(tests.run_avalanche_nist_statistics_test, 0, 10, 10)