From a259cf7aa0bc503cc7bd0ca52962affde155cf37 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 12:49:31 -0700 Subject: [PATCH 01/11] chore: fix typo --- test/functional/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/test_client.py b/test/functional/test_client.py index ebe7e14d1..02ae5d949 100644 --- a/test/functional/test_client.py +++ b/test/functional/test_client.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Functional test suite for aws_encryption_sdk.kms_thick_client""" +"""Functional test suite for aws_encryption_sdk""" from __future__ import division import io From db1b270d2ce35a6d72d4159b9f3f87f4715bd5d6 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 12:49:55 -0700 Subject: [PATCH 02/11] chore: add end-to-end benchmark tests --- setup.cfg | 1 + test/integration/test_client_performance.py | 94 +++++++++++++++++++++ test/requirements.txt | 1 + tox.ini | 4 + 4 files changed, 100 insertions(+) create mode 100644 test/integration/test_client_performance.py diff --git a/setup.cfg b/setup.cfg index 0671c3c64..52443c83c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ markers = integ: mark a test as an integration test (requires network access) accept: mark a test as an acceptance test (requires network access) examples: mark a test as an examples test (requires network access) + benchmark: mark a test as a performance benchmark test # Flake8 Configuration [flake8] diff --git a/test/integration/test_client_performance.py b/test/integration/test_client_performance.py new file mode 100644 index 000000000..6aa7fdc95 --- /dev/null +++ b/test/integration/test_client_performance.py @@ -0,0 +1,94 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Functional performance test suite for ``aws_encryption_sdk``.""" +import copy + +import pytest + +import aws_encryption_sdk +from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + +from ..unit.unit_test_utils import ( + ephemeral_raw_aes_keyring, + ephemeral_raw_aes_master_key, + ephemeral_raw_rsa_keyring, + ephemeral_raw_rsa_master_key, +) +from .integration_test_utils import build_aws_kms_keyring, setup_kms_master_key_provider + +pytestmark = [pytest.mark.benchmark] +ENCRYPTION_CONTEXT = { + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", +} + + +def _cycle(**kwargs): + encrypt_kwargs = copy.copy(kwargs) + decrypt_kwargs = copy.copy(kwargs) + for param in ("encryption_context", "frame_length", "source"): + try: + del decrypt_kwargs[param] + except KeyError: + pass + + encrypted = aws_encryption_sdk.encrypt(**encrypt_kwargs) + decrypt_kwargs["source"] = encrypted.result + aws_encryption_sdk.decrypt(**decrypt_kwargs) + + +@pytest.mark.parametrize( + "provider_param, provider_builder", + ( + pytest.param("keyring", ephemeral_raw_aes_keyring, id="Raw AES keyring"), + pytest.param("key_provider", ephemeral_raw_aes_master_key, id="Raw AES master key"), + pytest.param("keyring", ephemeral_raw_rsa_keyring, id="Raw RSA keyring"), + pytest.param("key_provider", ephemeral_raw_rsa_master_key, id="Raw RSA master key"), + pytest.param("keyring", build_aws_kms_keyring, id="AWS KMS keyring"), + pytest.param("key_provider", setup_kms_master_key_provider, id="AWS KMS master key provider"), + ), +) +@pytest.mark.parametrize( + "cache_messages", + ( + pytest.param(0, id="no cache"), + pytest.param(1000000, id="cache and only miss once"), + pytest.param(10, id="cache and only hit every 10"), + ), +) +@pytest.mark.parametrize("plaintext, frame_length", (pytest.param("foo", 1024, id="single frame"),)) +@pytest.mark.parametrize( + "operation", + ( + pytest.param(aws_encryption_sdk.encrypt, id="encrypt only"), + pytest.param(aws_encryption_sdk.decrypt, id="decrypt only"), + pytest.param(_cycle, id="encrypt decrypt cycle"), + ), +) +def test_end2end_performance( + benchmark, provider_param, provider_builder, cache_messages, plaintext, frame_length, operation +): + provider = provider_builder() + if cache_messages == 0: + cmm = DefaultCryptoMaterialsManager(**{provider_param: provider}) + else: + cmm = CachingCryptoMaterialsManager( + max_age=6000.0, + max_messages_encrypted=cache_messages, + cache=LocalCryptoMaterialsCache(capacity=10), + **{provider_param: provider} + ) + kwargs = dict( + source=plaintext, + materials_provider=cmm, + encryption_context=copy.copy(ENCRYPTION_CONTEXT), + frame_length=frame_length, + ) + if operation is aws_encryption_sdk.decrypt: + kwargs = dict(source=aws_encryption_sdk.encrypt(**kwargs).result, materails_provider=cmm,) + benchmark.pedantic(target=operation, kwargs=kwargs, iterations=100, rounds=10) diff --git a/test/requirements.txt b/test/requirements.txt index ff9311dc4..c82fdaace 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -3,3 +3,4 @@ pytest>=3.3.1 pytest-cov pytest-mock moto>=1.3.14 +pytest-benchmark>=3.2.3 diff --git a/tox.ini b/tox.ini index 0785c62f7..2b4d6c9d1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,5 @@ [tox] +minversion = 3.4.0 envlist = py{27,35,36,37,38}-{local,integ,accept,examples}, nocmk, bandit, doc8, readme, docs, @@ -49,7 +50,10 @@ passenv = AWS_PROFILE sitepackages = False deps = -rtest/requirements.txt +# 'download' forces tox to always upgrade pip to the latest +download = true commands = + benchmark: {[testenv:base-command]commands} test/ -m benchmark local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept From e2037f26ea331b36cc3969d7f571be7333f5c439 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 12:51:20 -0700 Subject: [PATCH 03/11] chore: prettify yaml --- .travis.yml | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index a8ca00f68..3c3bf8270 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,92 +54,77 @@ matrix: ######################## # CPython 2.7 - python: 2.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py27-awses_1.3.3 stage: Test Vector Handler Tests - python: 2.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py27-awses_1.3.max stage: Test Vector Handler Tests - python: 2.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py27-awses_latest stage: Test Vector Handler Tests # CPython 3.5 - python: 3.5 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py35-awses_1.3.3 stage: Test Vector Handler Tests - python: 3.5 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py35-awses_1.3.max stage: Test Vector Handler Tests - python: 3.5 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py35-awses_latest stage: Test Vector Handler Tests # CPython 3.6 - python: 3.6 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py36-awses_1.3.3 stage: Test Vector Handler Tests - python: 3.6 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py36-awses_1.3.max stage: Test Vector Handler Tests - python: 3.6 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py36-awses_latest stage: Test Vector Handler Tests # CPython 3.7 - python: 3.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py37-awses_1.3.3 dist: xenial sudo: true stage: Test Vector Handler Tests - python: 3.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py37-awses_1.3.max dist: xenial sudo: true stage: Test Vector Handler Tests - python: 3.7 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py37-awses_latest dist: xenial sudo: true stage: Test Vector Handler Tests # CPython 3.8 - python: 3.8 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py38-awses_1.3.3 dist: xenial sudo: true stage: Test Vector Handler Tests - python: 3.8 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py38-awses_1.3.max dist: xenial sudo: true stage: Test Vector Handler Tests - python: 3.8 - env: - TEST_VECTOR_HANDLERS=1 + env: TEST_VECTOR_HANDLERS=1 TOXENV=py38-awses_latest dist: xenial sudo: true From f0c0234f845b29f424a905d5da2e4bb373a9e652 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 12:52:46 -0700 Subject: [PATCH 04/11] chore: add CPython 3.8 benchmark run to Travis --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3c3bf8270..c15e80d3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,11 @@ matrix: dist: xenial sudo: true stage: Client Tests + - python: 3.8 + env: TOXENV=py38-benchmark + dist: xenial + sudo: true + stage: Client Tests ######################## # Test Vector Handlers # ######################## From 1de1733f1af34f2c65ecea9b8248e5a00268910b Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 13:37:11 -0700 Subject: [PATCH 05/11] chore: simplify test configuration --- test/integration/test_client_performance.py | 31 ++++++++++++--------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/test/integration/test_client_performance.py b/test/integration/test_client_performance.py index 6aa7fdc95..fab440909 100644 --- a/test/integration/test_client_performance.py +++ b/test/integration/test_client_performance.py @@ -7,6 +7,7 @@ import aws_encryption_sdk from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.keyrings.base import Keyring from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager @@ -43,14 +44,14 @@ def _cycle(**kwargs): @pytest.mark.parametrize( - "provider_param, provider_builder", + "provider_builder", ( - pytest.param("keyring", ephemeral_raw_aes_keyring, id="Raw AES keyring"), - pytest.param("key_provider", ephemeral_raw_aes_master_key, id="Raw AES master key"), - pytest.param("keyring", ephemeral_raw_rsa_keyring, id="Raw RSA keyring"), - pytest.param("key_provider", ephemeral_raw_rsa_master_key, id="Raw RSA master key"), - pytest.param("keyring", build_aws_kms_keyring, id="AWS KMS keyring"), - pytest.param("key_provider", setup_kms_master_key_provider, id="AWS KMS master key provider"), + pytest.param(ephemeral_raw_aes_keyring, id="Raw AES keyring"), + pytest.param(ephemeral_raw_aes_master_key, id="Raw AES master key"), + pytest.param(ephemeral_raw_rsa_keyring, id="Raw RSA keyring"), + pytest.param(ephemeral_raw_rsa_master_key, id="Raw RSA master key"), + pytest.param(build_aws_kms_keyring, id="AWS KMS keyring"), + pytest.param(setup_kms_master_key_provider, id="AWS KMS master key provider"), ), ) @pytest.mark.parametrize( @@ -58,7 +59,7 @@ def _cycle(**kwargs): ( pytest.param(0, id="no cache"), pytest.param(1000000, id="cache and only miss once"), - pytest.param(10, id="cache and only hit every 10"), + pytest.param(10, id="cache and miss every 10"), ), ) @pytest.mark.parametrize("plaintext, frame_length", (pytest.param("foo", 1024, id="single frame"),)) @@ -70,10 +71,13 @@ def _cycle(**kwargs): pytest.param(_cycle, id="encrypt decrypt cycle"), ), ) -def test_end2end_performance( - benchmark, provider_param, provider_builder, cache_messages, plaintext, frame_length, operation -): +def test_end2end_performance(benchmark, provider_builder, cache_messages, plaintext, frame_length, operation): provider = provider_builder() + if isinstance(provider, Keyring): + provider_param = "keyring" + else: + provider_param = "master_key_provider" + if cache_messages == 0: cmm = DefaultCryptoMaterialsManager(**{provider_param: provider}) else: @@ -83,12 +87,13 @@ def test_end2end_performance( cache=LocalCryptoMaterialsCache(capacity=10), **{provider_param: provider} ) + kwargs = dict( source=plaintext, - materials_provider=cmm, + materials_manager=cmm, encryption_context=copy.copy(ENCRYPTION_CONTEXT), frame_length=frame_length, ) if operation is aws_encryption_sdk.decrypt: - kwargs = dict(source=aws_encryption_sdk.encrypt(**kwargs).result, materails_provider=cmm,) + kwargs = dict(source=aws_encryption_sdk.encrypt(**kwargs).result, materials_manager=cmm,) benchmark.pedantic(target=operation, kwargs=kwargs, iterations=100, rounds=10) From bcf47237957a7c79370ce78b3e91ebec5e477010 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 16:19:54 -0700 Subject: [PATCH 06/11] chore: reorganize benchmark files into their own directory and specifically only mark KMS tests as integ --- test/benchmark/__init__.py | 3 + .../benchmark_test_utils.py} | 74 +++++------ test/benchmark/test_client_performance.py | 117 ++++++++++++++++++ 3 files changed, 152 insertions(+), 42 deletions(-) create mode 100644 test/benchmark/__init__.py rename test/{integration/test_client_performance.py => benchmark/benchmark_test_utils.py} (55%) create mode 100644 test/benchmark/test_client_performance.py diff --git a/test/benchmark/__init__.py b/test/benchmark/__init__.py new file mode 100644 index 000000000..db168aa04 --- /dev/null +++ b/test/benchmark/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance benchmarking tests.""" diff --git a/test/integration/test_client_performance.py b/test/benchmark/benchmark_test_utils.py similarity index 55% rename from test/integration/test_client_performance.py rename to test/benchmark/benchmark_test_utils.py index fab440909..770e4c874 100644 --- a/test/integration/test_client_performance.py +++ b/test/benchmark/benchmark_test_utils.py @@ -1,25 +1,17 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Functional performance test suite for ``aws_encryption_sdk``.""" +"""Helper utilities for benchmark tests.""" import copy import pytest import aws_encryption_sdk from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.identifiers import AlgorithmSuite from aws_encryption_sdk.keyrings.base import Keyring from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager -from ..unit.unit_test_utils import ( - ephemeral_raw_aes_keyring, - ephemeral_raw_aes_master_key, - ephemeral_raw_rsa_keyring, - ephemeral_raw_rsa_master_key, -) -from .integration_test_utils import build_aws_kms_keyring, setup_kms_master_key_provider - -pytestmark = [pytest.mark.benchmark] ENCRYPTION_CONTEXT = { "encryption": "context", "is not": "secret", @@ -29,10 +21,21 @@ } -def _cycle(**kwargs): +def all_operations(): + return pytest.mark.parametrize( + "operation", + ( + pytest.param(aws_encryption_sdk.encrypt, id="encrypt only"), + pytest.param(aws_encryption_sdk.decrypt, id="decrypt only"), + pytest.param(encrypt_decrypt_cycle, id="encrypt decrypt cycle"), + ), + ) + + +def encrypt_decrypt_cycle(**kwargs): encrypt_kwargs = copy.copy(kwargs) decrypt_kwargs = copy.copy(kwargs) - for param in ("encryption_context", "frame_length", "source"): + for param in ("encryption_context", "frame_length", "source", "algorithm"): try: del decrypt_kwargs[param] except KeyError: @@ -43,35 +46,7 @@ def _cycle(**kwargs): aws_encryption_sdk.decrypt(**decrypt_kwargs) -@pytest.mark.parametrize( - "provider_builder", - ( - pytest.param(ephemeral_raw_aes_keyring, id="Raw AES keyring"), - pytest.param(ephemeral_raw_aes_master_key, id="Raw AES master key"), - pytest.param(ephemeral_raw_rsa_keyring, id="Raw RSA keyring"), - pytest.param(ephemeral_raw_rsa_master_key, id="Raw RSA master key"), - pytest.param(build_aws_kms_keyring, id="AWS KMS keyring"), - pytest.param(setup_kms_master_key_provider, id="AWS KMS master key provider"), - ), -) -@pytest.mark.parametrize( - "cache_messages", - ( - pytest.param(0, id="no cache"), - pytest.param(1000000, id="cache and only miss once"), - pytest.param(10, id="cache and miss every 10"), - ), -) -@pytest.mark.parametrize("plaintext, frame_length", (pytest.param("foo", 1024, id="single frame"),)) -@pytest.mark.parametrize( - "operation", - ( - pytest.param(aws_encryption_sdk.encrypt, id="encrypt only"), - pytest.param(aws_encryption_sdk.decrypt, id="decrypt only"), - pytest.param(_cycle, id="encrypt decrypt cycle"), - ), -) -def test_end2end_performance(benchmark, provider_builder, cache_messages, plaintext, frame_length, operation): +def build_cmm(provider_builder, cache_messages): provider = provider_builder() if isinstance(provider, Keyring): provider_param = "keyring" @@ -88,12 +63,27 @@ def test_end2end_performance(benchmark, provider_builder, cache_messages, plaint **{provider_param: provider} ) + return cmm + + +def run_benchmark( + benchmark, + provider_builder, + operation, + cache_messages=0, + plaintext="foo", + frame_length=1024, + algorithm=AlgorithmSuite.AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384, +): + cmm = build_cmm(provider_builder, cache_messages) + kwargs = dict( source=plaintext, materials_manager=cmm, encryption_context=copy.copy(ENCRYPTION_CONTEXT), frame_length=frame_length, + algorithm=algorithm, ) if operation is aws_encryption_sdk.decrypt: kwargs = dict(source=aws_encryption_sdk.encrypt(**kwargs).result, materials_manager=cmm,) - benchmark.pedantic(target=operation, kwargs=kwargs, iterations=100, rounds=10) + benchmark.pedantic(target=operation, kwargs=kwargs, iterations=10, rounds=10) diff --git a/test/benchmark/test_client_performance.py b/test/benchmark/test_client_performance.py new file mode 100644 index 000000000..efaea8a95 --- /dev/null +++ b/test/benchmark/test_client_performance.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Functional performance test suite for ``aws_encryption_sdk``.""" +import os + +import pytest + +from aws_encryption_sdk.identifiers import AlgorithmSuite + +from ..integration.integration_test_utils import build_aws_kms_keyring, setup_kms_master_key_provider +from ..unit.unit_test_utils import ( + ephemeral_raw_aes_keyring, + ephemeral_raw_aes_master_key, + ephemeral_raw_rsa_keyring, + ephemeral_raw_rsa_master_key, +) +from .benchmark_test_utils import all_operations, run_benchmark + +pytestmark = [pytest.mark.benchmark] +SMALL_PLAINTEXT = os.urandom(32) # 32B +LARGE_PLAINTEXT = os.urandom(1024 * 1024) # 1MiB +VERY_LARGE_PLAINTEXT = os.urandom(10 * 1024 * 1024) # 10MiB + + +@pytest.mark.parametrize("algorithm_suite", AlgorithmSuite) +@all_operations() +def test_compare_algorithm_suite_performance(benchmark, algorithm_suite, operation): + """Compare the affect of algorithm suite on performance. + Use the Raw AES keyring as a baseline keyring. + """ + run_benchmark(benchmark=benchmark, provider_builder=ephemeral_raw_aes_keyring, operation=operation) + + +@pytest.mark.parametrize( + "cache_messages", + ( + pytest.param(0, id="no cache"), + pytest.param(1000000, id="cache and only miss once"), + pytest.param(10, id="cache and miss every 10"), + ), +) +@all_operations() +def test_compare_caching_performance(benchmark, operation, cache_messages): + """Compare the affect of caching on performance. + Use the Raw AES keyring as a baseline keyring. + """ + run_benchmark( + benchmark=benchmark, + provider_builder=ephemeral_raw_aes_keyring, + operation=operation, + cache_messages=cache_messages, + ) + + +@pytest.mark.parametrize( + "plaintext, frame_length", + ( + pytest.param(SMALL_PLAINTEXT, 0, id="small message, unframed"), + pytest.param(SMALL_PLAINTEXT, 128, id="small message, single frame"), + pytest.param(LARGE_PLAINTEXT, 1024 * 1024 * 1024, id="large message, single frame"), + pytest.param(LARGE_PLAINTEXT, 102400, id="large message, few large frames"), + pytest.param(LARGE_PLAINTEXT, 1024, id="large message, many small frames"), + ), +) +@all_operations() +def test_compare_framing_performance(benchmark, operation, plaintext, frame_length): + """Compare the affect of framing and on performance. + Use the Raw AES keyring as a baseline keyring. + """ + run_benchmark( + benchmark=benchmark, + provider_builder=ephemeral_raw_aes_keyring, + operation=operation, + plaintext=plaintext, + frame_length=frame_length, + ) + + +def _frame_sizes(): + for frame_kb in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 10240): + yield pytest.param(frame_kb * 1024, id="{} kiB frame".format(frame_kb)) + + +@pytest.mark.parametrize( + "plaintext", + (pytest.param(LARGE_PLAINTEXT, id="1MiB plaintext"), pytest.param(VERY_LARGE_PLAINTEXT, id="10MiB plaintext"),), +) +@pytest.mark.parametrize("frame_length", _frame_sizes()) +@all_operations() +def test_compare_frame_size_performance(benchmark, operation, plaintext, frame_length): + """Compare the affect of framing and on performance. + Use the Raw AES keyring as a baseline keyring. + """ + run_benchmark( + benchmark=benchmark, + provider_builder=ephemeral_raw_aes_keyring, + operation=operation, + plaintext=plaintext, + frame_length=frame_length, + ) + + +@pytest.mark.parametrize( + "provider_builder", + ( + pytest.param(ephemeral_raw_aes_keyring, id="Raw AES keyring"), + pytest.param(ephemeral_raw_aes_master_key, id="Raw AES master key"), + pytest.param(ephemeral_raw_rsa_keyring, id="Raw RSA keyring"), + pytest.param(ephemeral_raw_rsa_master_key, id="Raw RSA master key"), + pytest.param(build_aws_kms_keyring, id="AWS KMS keyring", marks=pytest.mark.integ), + pytest.param(setup_kms_master_key_provider, id="AWS KMS master key provider", marks=pytest.mark.integ), + ), +) +@all_operations() +def test_compare_keyring_performance(benchmark, provider_builder, operation): + """Compare the performance of different keyrings and master key providers.""" + run_benchmark(benchmark=benchmark, provider_builder=provider_builder, operation=operation) From 30ad9c7141f0041999b097fe9fa941085f5f93ee Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Thu, 11 Jun 2020 16:32:54 -0700 Subject: [PATCH 07/11] chore: change parameterized tests to indirect references to large values and remove 1kiB-frame benchmark test --- test/benchmark/test_client_performance.py | 28 ++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/benchmark/test_client_performance.py b/test/benchmark/test_client_performance.py index efaea8a95..4ed2562ee 100644 --- a/test/benchmark/test_client_performance.py +++ b/test/benchmark/test_client_performance.py @@ -17,9 +17,12 @@ from .benchmark_test_utils import all_operations, run_benchmark pytestmark = [pytest.mark.benchmark] -SMALL_PLAINTEXT = os.urandom(32) # 32B -LARGE_PLAINTEXT = os.urandom(1024 * 1024) # 1MiB -VERY_LARGE_PLAINTEXT = os.urandom(10 * 1024 * 1024) # 10MiB + +PLAINTEXTS = { + "SMALL": os.urandom(32), # 32B + "LARGE": os.urandom(1024 * 1024), # 1MiB + "VERY_LARGE": os.urandom(10 * 1024 * 1024), # 10MiB +} @pytest.mark.parametrize("algorithm_suite", AlgorithmSuite) @@ -55,11 +58,11 @@ def test_compare_caching_performance(benchmark, operation, cache_messages): @pytest.mark.parametrize( "plaintext, frame_length", ( - pytest.param(SMALL_PLAINTEXT, 0, id="small message, unframed"), - pytest.param(SMALL_PLAINTEXT, 128, id="small message, single frame"), - pytest.param(LARGE_PLAINTEXT, 1024 * 1024 * 1024, id="large message, single frame"), - pytest.param(LARGE_PLAINTEXT, 102400, id="large message, few large frames"), - pytest.param(LARGE_PLAINTEXT, 1024, id="large message, many small frames"), + pytest.param("SMALL", 0, id="small message, unframed"), + pytest.param("SMALL", 128, id="small message, single frame"), + pytest.param("LARGE", 1024 * 1024 * 1024, id="large message, single frame"), + pytest.param("LARGE", 102400, id="large message, few large frames"), + pytest.param("LARGE", 1024, id="large message, many small frames"), ), ) @all_operations() @@ -71,19 +74,18 @@ def test_compare_framing_performance(benchmark, operation, plaintext, frame_leng benchmark=benchmark, provider_builder=ephemeral_raw_aes_keyring, operation=operation, - plaintext=plaintext, + plaintext=PLAINTEXTS[plaintext], frame_length=frame_length, ) def _frame_sizes(): - for frame_kb in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 10240): + for frame_kb in (2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 10240): yield pytest.param(frame_kb * 1024, id="{} kiB frame".format(frame_kb)) @pytest.mark.parametrize( - "plaintext", - (pytest.param(LARGE_PLAINTEXT, id="1MiB plaintext"), pytest.param(VERY_LARGE_PLAINTEXT, id="10MiB plaintext"),), + "plaintext", (pytest.param("LARGE", id="1MiB plaintext"), pytest.param("VERY_LARGE", id="10MiB plaintext"),), ) @pytest.mark.parametrize("frame_length", _frame_sizes()) @all_operations() @@ -95,7 +97,7 @@ def test_compare_frame_size_performance(benchmark, operation, plaintext, frame_l benchmark=benchmark, provider_builder=ephemeral_raw_aes_keyring, operation=operation, - plaintext=plaintext, + plaintext=PLAINTEXTS[plaintext], frame_length=frame_length, ) From 877057d1df313624dd67ef397750166f921fb6ee Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 12 Jun 2020 11:28:41 -0700 Subject: [PATCH 08/11] chore: split local and integ benchmarks --- .github/workflows/ci_tests.yaml | 20 ++++++++++++++++++++ .travis.yml | 2 +- tox.ini | 4 +++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index c2f297ea2..d3b20def9 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -59,6 +59,26 @@ jobs: env: TOXENV: ${{ matrix.category }} run: tox -- -vv + benchmarks: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + category: + - nocmk + - test-upstream-requirements-py37 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: 3.x + - run: | + python -m pip install --upgrade pip + pip install --upgrade -r ci-requirements.txt + - name: run test + env: + TOXENV: benchmark-local + run: tox -- -vv upstream-py3: runs-on: ubuntu-latest strategy: diff --git a/.travis.yml b/.travis.yml index c15e80d3f..42538f56e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ matrix: sudo: true stage: Client Tests - python: 3.8 - env: TOXENV=py38-benchmark + env: TOXENV=py38-benchmark-integ dist: xenial sudo: true stage: Client Tests diff --git a/tox.ini b/tox.ini index 2b4d6c9d1..7d48e00cc 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,9 @@ deps = -rtest/requirements.txt # 'download' forces tox to always upgrade pip to the latest download = true commands = - benchmark: {[testenv:base-command]commands} test/ -m benchmark + benchmark-all: {[testenv:base-command]commands} test/ -m benchmark + benchmark-integ: {[testenv:base-command]commands} test/ -m "benchmark and integ" + benchmark-local: {[testenv:base-command]commands} test/ -m "benchmark and not integ" local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept From b404f2afbb17fd501f8955afddbf17dc5efeeee3 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 12 Jun 2020 12:09:27 -0700 Subject: [PATCH 09/11] chore: fix algorithms benchmark to actually change algorithms --- test/benchmark/test_client_performance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/benchmark/test_client_performance.py b/test/benchmark/test_client_performance.py index 4ed2562ee..e8a3460f4 100644 --- a/test/benchmark/test_client_performance.py +++ b/test/benchmark/test_client_performance.py @@ -31,7 +31,9 @@ def test_compare_algorithm_suite_performance(benchmark, algorithm_suite, operati """Compare the affect of algorithm suite on performance. Use the Raw AES keyring as a baseline keyring. """ - run_benchmark(benchmark=benchmark, provider_builder=ephemeral_raw_aes_keyring, operation=operation) + run_benchmark( + benchmark=benchmark, provider_builder=ephemeral_raw_aes_keyring, operation=operation, algorithm=algorithm_suite + ) @pytest.mark.parametrize( From a4352b0600816bcaa786172991c1e386e0f109c8 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 12 Jun 2020 12:10:39 -0700 Subject: [PATCH 10/11] chore: remove unnecessary strategy section for benchmarks job --- .github/workflows/ci_tests.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d3b20def9..8baf28054 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -61,12 +61,6 @@ jobs: run: tox -- -vv benchmarks: runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - category: - - nocmk - - test-upstream-requirements-py37 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 From 873218cf1f06b18ccdf503dbf4d6ca4e75eaae70 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 12 Jun 2020 14:05:38 -0700 Subject: [PATCH 11/11] chore: rename tox benchmark environments to avoid conflicts --- .github/workflows/ci_tests.yaml | 2 +- .travis.yml | 2 +- tox.ini | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 8baf28054..45cfeb380 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -71,7 +71,7 @@ jobs: pip install --upgrade -r ci-requirements.txt - name: run test env: - TOXENV: benchmark-local + TOXENV: benchmark-nokms run: tox -- -vv upstream-py3: runs-on: ubuntu-latest diff --git a/.travis.yml b/.travis.yml index 42538f56e..0a1abc76b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ matrix: sudo: true stage: Client Tests - python: 3.8 - env: TOXENV=py38-benchmark-integ + env: TOXENV=py38-benchmark-kms dist: xenial sudo: true stage: Client Tests diff --git a/tox.ini b/tox.ini index 7d48e00cc..7071e4c61 100644 --- a/tox.ini +++ b/tox.ini @@ -53,9 +53,9 @@ deps = -rtest/requirements.txt # 'download' forces tox to always upgrade pip to the latest download = true commands = - benchmark-all: {[testenv:base-command]commands} test/ -m benchmark - benchmark-integ: {[testenv:base-command]commands} test/ -m "benchmark and integ" - benchmark-local: {[testenv:base-command]commands} test/ -m "benchmark and not integ" + benchmark-full: {[testenv:base-command]commands} test/ -m benchmark + benchmark-kms: {[testenv:base-command]commands} test/ -m "benchmark and integ" + benchmark-nokms: {[testenv:base-command]commands} test/ -m "benchmark and not integ" local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept