From b6d16f68ffc74dd60d7f9dbbd9dd835f67442f9d Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 3 Jul 2024 10:13:47 -0500 Subject: [PATCH 01/25] run emulators for blob tests --- eng/ci/emulator-tests.yml | 43 +++++++++++++++++++ eng/templates/jobs/ci-emulator-tests.yml | 53 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 eng/ci/emulator-tests.yml create mode 100644 eng/templates/jobs/ci-emulator-tests.yml diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml new file mode 100644 index 000000000..88b4a9e3f --- /dev/null +++ b/eng/ci/emulator-tests.yml @@ -0,0 +1,43 @@ +trigger: none # ensure this is not ran as a CI build + +pr: + branches: + include: + - dev + - release/* + +schedules: + - cron: "0 8 * * 1,2,3,4,5" + displayName: Monday to Friday 3 AM CST build + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: /ci/variables/build.yml@eng + - template: /ci/variables/cfs.yml@eng + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: RunEmulatorTests + jobs: + - template: /eng/templates/jobs/ci-emulator-tests.yml@self diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml new file mode 100644 index 000000000..d10e3a592 --- /dev/null +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -0,0 +1,53 @@ +jobs: + - job: "TestPython" + displayName: "Run Python Emulator Tests" + + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + strategy: + matrix: + Python37: + PYTHON_VERSION: '3.7' + Python38: + PYTHON_VERSION: '3.8' + Python39: + PYTHON_VERSION: '3.9' + Python310: + PYTHON_VERSION: '3.10' + Python311: + PYTHON_VERSION: '3.11' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + - task: UseDotNet@2 + displayName: 'Install .NET 8' + inputs: + version: 8.0.x + - bash: | + python -m pip install --upgrade pip + python -m pip install -U azure-functions --pre + python -m pip install -U -e .[dev] + + if [[ $(PYTHON_VERSION) != "3.7" ]]; then + python -m pip install --pre -U -e .[test-http-v2] + fi + if [[ $(PYTHON_VERSION) != "3.7" && $(PYTHON_VERSION) != "3.8" ]]; then + python -m pip install --pre -U -e .[test-deferred-bindings] + fi + + python setup.py build + displayName: 'Install dependencies and the worker' + - bash: | + sudo npm install -g azurite + sudo mkdir azurite + sudo azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + - bash: | + python -m pytest -q -n auto --dist loadfile --reruns 4 tests/endtoend/test_blob_functions.py + env: + AzureWebJobsStorage: "UseDevelopmentStorage=true" + displayName: "Running $(PYTHON_VERSION) Python E2E Tests" From 76197a2405d4b070e82cb8168acc6aa034c5a88f Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 16 Aug 2024 16:12:07 -0500 Subject: [PATCH 02/25] blob, table, queue working. large refactor --- eng/ci/emulator-tests.yml | 43 - eng/ci/integration-tests.yml | 4 + eng/ci/official-build.yml | 4 + eng/ci/public-build.yml | 6 +- eng/templates/jobs/ci-emulator-tests.yml | 61 +- .../blob_functions_stein/function_app.py | 786 +++++++++--------- .../generic/function_app.py | 0 .../blob_functions/blob_trigger/function.json | 0 .../blob_functions/blob_trigger/main.py | 0 .../get_blob_as_bytes/function.json | 0 .../blob_functions/get_blob_as_bytes/main.py | 0 .../function.json | 0 .../main.py | 0 .../function.json | 0 .../main.py | 0 .../get_blob_as_str/function.json | 0 .../blob_functions/get_blob_as_str/main.py | 0 .../function.json | 0 .../main.py | 0 .../get_blob_bytes/function.json | 0 .../blob_functions/get_blob_bytes/main.py | 0 .../get_blob_filelike/function.json | 0 .../blob_functions/get_blob_filelike/main.py | 0 .../get_blob_return/function.json | 0 .../blob_functions/get_blob_return/main.py | 0 .../blob_functions/get_blob_str/function.json | 0 .../blob_functions/get_blob_str/main.py | 0 .../get_blob_triggered/function.json | 0 .../blob_functions/get_blob_triggered/main.py | 0 .../function.json | 0 .../main.py | 0 .../function.json | 0 .../main.py | 0 .../put_blob_bytes/function.json | 0 .../blob_functions/put_blob_bytes/main.py | 0 .../put_blob_filelike/function.json | 0 .../blob_functions/put_blob_filelike/main.py | 0 .../put_blob_return/function.json | 0 .../blob_functions/put_blob_return/main.py | 0 .../blob_functions/put_blob_str/function.json | 0 .../blob_functions/put_blob_str/main.py | 0 .../put_blob_trigger/function.json | 0 .../blob_functions/put_blob_trigger/main.py | 0 .../function.json | 0 .../main.py | 0 .../function_app.py | 0 .../eventhub_multiple/__init__.py | 0 .../eventhub_multiple/function.json | 0 .../eventhub_output_batch/__init__.py | 0 .../eventhub_output_batch/function.json | 0 .../get_eventhub_batch_triggered/__init__.py | 0 .../function.json | 50 +- .../get_metadata_batch_triggered/__init__.py | 0 .../function.json | 0 .../metadata_multiple/__init__.py | 0 .../metadata_multiple/function.json | 0 .../metadata_output_batch/__init__.py | 0 .../metadata_output_batch/function.json | 0 .../eventhub_functions_stein/function_app.py | 214 ++--- .../generic/function_app.py | 0 .../eventhub_output/__init__.py | 0 .../eventhub_output/function.json | 0 .../eventhub_trigger/__init__.py | 0 .../eventhub_trigger/function.json | 0 .../get_eventhub_triggered/function.json | 0 .../get_eventhub_triggered/main.py | 0 .../get_metadata_triggered/__init__.py | 0 .../get_metadata_triggered/function.json | 0 .../metadata_output/__init__.py | 0 .../metadata_output/function.json | 0 .../metadata_trigger/__init__.py | 0 .../metadata_trigger/function.json | 0 .../generic_functions_stein/function_app.py | 0 .../return_bool/function.json | 0 .../generic_functions/return_bool/main.py | 0 .../return_bytes/function.json | 0 .../generic_functions/return_bytes/main.py | 0 .../return_dict/function.json | 0 .../generic_functions/return_dict/main.py | 0 .../return_double/function.json | 0 .../generic_functions/return_double/main.py | 0 .../return_int/function.json | 0 .../generic_functions/return_int/main.py | 0 .../return_list/function.json | 0 .../generic_functions/return_list/main.py | 0 .../return_none/function.json | 0 .../generic_functions/return_none/main.py | 0 .../return_none_no_type_hint/function.json | 0 .../return_none_no_type_hint/main.py | 0 .../return_not_processed_last/__init__.py | 0 .../return_not_processed_last/function.json | 0 .../return_processed_last/__init__.py | 0 .../return_processed_last/function.json | 0 .../return_string/function.json | 0 .../generic_functions/return_string/main.py | 0 .../get_queue_blob/function.json | 0 .../queue_functions/get_queue_blob/main.py | 0 .../function.json | 0 .../get_queue_blob_message_return/main.py | 0 .../get_queue_blob_return/function.json | 0 .../get_queue_blob_return/main.py | 0 .../function.json | 0 .../get_queue_untyped_blob_return/main.py | 0 .../queue_functions/put_queue/function.json | 0 .../queue_functions/put_queue/main.py | 0 .../put_queue_message_return/function.json | 0 .../put_queue_message_return/main.py | 0 .../put_queue_multiple_out/function.json | 0 .../put_queue_multiple_out/main.py | 0 .../put_queue_return/function.json | 0 .../queue_functions/put_queue_return/main.py | 0 .../put_queue_return_multiple/function.json | 0 .../put_queue_return_multiple/main.py | 0 .../put_queue_untyped_return/function.json | 0 .../put_queue_untyped_return/main.py | 0 .../queue_functions_stein/function_app.py | 370 ++++----- .../generic/function_app.py | 0 .../queue_trigger/function.json | 0 .../queue_functions/queue_trigger/main.py | 0 .../function.json | 0 .../queue_trigger_message_return/main.py | 0 .../queue_trigger_return/function.json | 0 .../queue_trigger_return/main.py | 0 .../function.json | 0 .../queue_trigger_return_multiple/main.py | 0 .../queue_trigger_untyped/function.json | 0 .../queue_trigger_untyped/main.py | 0 .../table_functions_stein/function_app.py | 0 .../generic/function_app.py | 0 .../table_in_binding/__init__.py | 0 .../table_in_binding/function.json | 0 .../table_out_binding/__init__.py | 0 .../table_out_binding/function.json | 0 .../test_blob_functions.py | 6 +- .../test_eventhub_batch_functions.py | 4 +- .../test_eventhub_functions.py | 6 +- .../test_generic_functions.py | 4 +- .../test_queue_functions.py | 6 +- .../test_table_functions.py | 6 +- 139 files changed, 787 insertions(+), 783 deletions(-) delete mode 100644 eng/ci/emulator-tests.yml rename tests/{endtoend => emulator_tests}/blob_functions/blob_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/blob_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_as_str_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_filelike/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_filelike/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/get_blob_triggered/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_str_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_as_str_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_filelike/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_filelike/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_str/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_str/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_blob_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json (100%) rename tests/{endtoend => emulator_tests}/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_multiple/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_output_batch/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/eventhub_output_batch/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_eventhub_batch_triggered/function.json (95%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/get_metadata_batch_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_multiple/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_output_batch/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_batch_functions/metadata_output_batch/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_output/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_output/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_trigger/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/eventhub_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_eventhub_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_eventhub_triggered/main.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_metadata_triggered/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/get_metadata_triggered/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_output/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_output/function.json (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_trigger/__init__.py (100%) rename tests/{endtoend => emulator_tests}/eventhub_functions/metadata_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/generic_functions_stein/function_app.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bool/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bool/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bytes/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_bytes/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_dict/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_dict/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_double/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_double/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_int/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_int/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_list/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_list/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none_no_type_hint/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_none_no_type_hint/main.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_not_processed_last/__init__.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_not_processed_last/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_processed_last/__init__.py (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_processed_last/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_string/function.json (100%) rename tests/{endtoend => emulator_tests}/generic_functions/return_string/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_untyped_blob_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/get_queue_untyped_blob_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_multiple_out/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_multiple_out/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_return_multiple/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_untyped_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/put_queue_untyped_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_functions_stein/function_app.py (97%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_message_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_message_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return_multiple/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_return_multiple/main.py (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_untyped/function.json (100%) rename tests/{endtoend => emulator_tests}/queue_functions/queue_trigger_untyped/main.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_functions_stein/function_app.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_functions_stein/generic/function_app.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_in_binding/__init__.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_in_binding/function.json (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_out_binding/__init__.py (100%) rename tests/{endtoend => emulator_tests}/table_functions/table_out_binding/function.json (100%) rename tests/{endtoend => emulator_tests}/test_blob_functions.py (96%) rename tests/{endtoend => emulator_tests}/test_eventhub_batch_functions.py (98%) rename tests/{endtoend => emulator_tests}/test_eventhub_functions.py (95%) rename tests/{endtoend => emulator_tests}/test_generic_functions.py (95%) rename tests/{endtoend => emulator_tests}/test_queue_functions.py (94%) rename tests/{endtoend => emulator_tests}/test_table_functions.py (93%) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml deleted file mode 100644 index 88b4a9e3f..000000000 --- a/eng/ci/emulator-tests.yml +++ /dev/null @@ -1,43 +0,0 @@ -trigger: none # ensure this is not ran as a CI build - -pr: - branches: - include: - - dev - - release/* - -schedules: - - cron: "0 8 * * 1,2,3,4,5" - displayName: Monday to Friday 3 AM CST build - branches: - include: - - dev - always: true - -resources: - repositories: - - repository: 1es - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - - repository: eng - type: git - name: engineering - ref: refs/tags/release - -variables: - - template: /ci/variables/build.yml@eng - - template: /ci/variables/cfs.yml@eng - -extends: - template: v1/1ES.Unofficial.PipelineTemplate.yml@1es - parameters: - pool: - name: 1es-pool-azfunc - image: 1es-windows-2022 - os: windows - - stages: - - stage: RunEmulatorTests - jobs: - - template: /eng/templates/jobs/ci-emulator-tests.yml@self diff --git a/eng/ci/integration-tests.yml b/eng/ci/integration-tests.yml index 6f8f69d9e..b4c1e4733 100644 --- a/eng/ci/integration-tests.yml +++ b/eng/ci/integration-tests.yml @@ -51,3 +51,7 @@ extends: dependsOn: [] jobs: - template: /eng/templates/official/jobs/ci-lc-tests.yml@self + - stage: RunEmulatorTests + dependsOn: [] + jobs: + - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 8ac6231d2..1b41610c1 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -48,6 +48,10 @@ extends: dependsOn: Build jobs: - template: /eng/templates/official/jobs/ci-e2e-tests.yml@self + - stage: RunEmulatorTests + dependsOn: Build + jobs: + - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self - stage: RunUnitTests dependsOn: Build jobs: diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 675597448..82251caf3 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -50,4 +50,8 @@ extends: - stage: RunUnitTests dependsOn: Build jobs: - - template: /eng/templates/jobs/ci-unit-tests.yml@self \ No newline at end of file + - template: /eng/templates/jobs/ci-unit-tests.yml@self + - stage: RunEmulatorTests + dependsOn: Build + jobs: + - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self \ No newline at end of file diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index d10e3a592..f78224bd0 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -28,26 +28,61 @@ jobs: inputs: version: 8.0.x - bash: | - python -m pip install --upgrade pip - python -m pip install -U azure-functions --pre - python -m pip install -U -e .[dev] + chmod +x eng/scripts/install-dependencies.sh + chmod +x eng/scripts/test-setup.sh - if [[ $(PYTHON_VERSION) != "3.7" ]]; then - python -m pip install --pre -U -e .[test-http-v2] - fi - if [[ $(PYTHON_VERSION) != "3.7" && $(PYTHON_VERSION) != "3.8" ]]; then - python -m pip install --pre -U -e .[test-deferred-bindings] - fi - - python setup.py build + eng/scripts/install-dependencies.sh $(PYTHON_VERSION) + eng/scripts/test-setup.sh displayName: 'Install dependencies and the worker' + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python SDK Artifact' + inputs: + buildType: specific + artifactName: 'azure-functions' + project: 'internal' + definition: 679 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonSdkArtifact' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - bash: | + chmod +x eng/scripts/test-sdk.sh + chmod +x eng/scripts/test-setup.sh + + eng/scripts/test-sdk.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python sdk, dependencies and the worker' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python Extension Artifact' + inputs: + buildType: specific + artifactName: $(PYTHONEXTENSIONNAME) + project: 'internal' + definition: 798 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonExtensionArtifact' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + chmod +x eng/scripts/test-setup.sh + chmod +x eng/scripts/test-extensions.sh + + eng/scripts/test-extensions.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python extension, dependencies and the worker' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + yes | git clone https://github.com/Azure/azure-event-hubs-emulator-installer.git + yes | azure-event-hubs-emulator-installer/EventHub-Emulator/Scripts/Linux/LaunchEmulator.sh + displayName: 'Install and Run Azure Event Hubs Emulator' - bash: | sudo npm install -g azurite sudo mkdir azurite sudo azurite --silent --location azurite --debug azurite\debug.log & displayName: 'Install and Run Azurite' - bash: | - python -m pytest -q -n auto --dist loadfile --reruns 4 tests/endtoend/test_blob_functions.py + python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests env: AzureWebJobsStorage: "UseDevelopmentStorage=true" - displayName: "Running $(PYTHON_VERSION) Python E2E Tests" + AzureWebJobsEventHubConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" + displayName: "Running $(PYTHON_VERSION) Python Emulator Tests" diff --git a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py b/tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/blob_functions/blob_functions_stein/function_app.py rename to tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py index 2621a3360..51349a631 100644 --- a/tests/endtoend/blob_functions/blob_functions_stein/function_app.py +++ b/tests/emulator_tests/blob_functions/blob_functions_stein/function_app.py @@ -1,393 +1,393 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import hashlib -import io -import json -import random -import string - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="blob_trigger") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") -def blob_trigger(file: func.InputStream) -> str: - return json.dumps({ - 'name': file.name, - 'length': file.length, - 'content': file.read().decode('utf-8') - }) - - -@app.function_name(name="get_blob_as_bytes") -@app.route(route="get_blob_as_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes(req: func.HttpRequest, file: bytes) -> str: - assert isinstance(file, bytes) - return file.decode('utf-8') - - -@app.function_name(name="get_blob_as_bytes_return_http_response") -@app.route(route="get_blob_as_bytes_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ - -> func.HttpResponse: - """ - Read a blob (bytes) and respond back (in HTTP response) with the number of - bytes read and the MD5 digest of the content. - """ - assert isinstance(file, bytes) - - content_size = len(file) - content_md5 = hashlib.md5(file).hexdigest() - - response_dict = { - 'content_size': content_size, - 'content_md5': content_md5 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_as_bytes_stream_return_http_response") -@app.route(route="get_blob_as_bytes_stream_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, - file: func.InputStream) \ - -> func.HttpResponse: - """ - Read a blob (as azf.InputStream) and respond back (in HTTP response) with - the number of bytes read and the MD5 digest of the content. - """ - file_bytes = file.read() - - content_size = len(file_bytes) - content_md5 = hashlib.md5(file_bytes).hexdigest() - - response_dict = { - 'content_size': content_size, - 'content_md5': content_md5 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_as_str") -@app.route(route="get_blob_as_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -def get_blob_as_str(req: func.HttpRequest, file: str) -> str: - assert isinstance(file, str) - return file - - -@app.function_name(name="get_blob_as_str_return_http_response") -@app.route(route="get_blob_as_str_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -def get_blob_as_str_return_http_response(req: func.HttpRequest, - file: str) -> func.HttpResponse: - """ - Read a blob (string) and respond back (in HTTP response) with the number of - characters read and the MD5 digest of the utf-8 encoded content. - """ - assert isinstance(file, str) - - num_chars = len(file) - content_bytes = file.encode('utf-8') - content_md5 = hashlib.md5(content_bytes).hexdigest() - - response_dict = { - 'num_chars': num_chars, - 'content_md5': content_md5 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="get_blob_bytes") -@app.route(route="get_blob_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") -def get_blob_bytes(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_filelike") -@app.route(route="get_blob_filelike") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") -def get_blob_filelike(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_return") -@app.route(route="get_blob_return") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") -def get_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_str") -@app.route(route="get_blob_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") -def get_blob_str(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_blob_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") -@app.route(route="get_blob_triggered") -def get_blob_triggered(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="put_blob_as_bytes_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-bytes-out.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_bytes_return_http_response") -def put_blob_as_bytes_return_http_response(req: func.HttpRequest, - file: func.Out[ - bytes]) -> func.HttpResponse: - """ - Write a blob (bytes) and respond back (in HTTP response) with the number of - bytes written and the MD5 digest of the content. - The number of bytes to write are specified in the input HTTP request. - """ - content_size = int(req.params['content_size']) - - # When this is set, then 0x01 byte is repeated content_size number of - # times to use as input. - # This is to avoid generating random input for large size which can be - # slow. - if 'no_random_input' in req.params: - content = b'\x01' * content_size - else: - content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() - - file.set(content) - - response_dict = { - 'content_size': content_size, - 'content_md5': content_md5 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="put_blob_as_str_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-str-out.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_str_return_http_response") -def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ - str]) -> func.HttpResponse: - """ - Write a blob (string) and respond back (in HTTP response) with the number of - characters written and the MD5 digest of the utf-8 encoded content. - The number of characters to write are specified in the input HTTP request. - """ - num_chars = int(req.params['num_chars']) - - content = ''.join(random.choices(string.ascii_uppercase + string.digits, - k=num_chars)) - content_bytes = content.encode('utf-8') - content_size = len(content_bytes) - content_md5 = hashlib.md5(content_bytes).hexdigest() - - file.set(content) - - response_dict = { - 'num_chars': num_chars, - 'content_size': content_size, - 'content_md5': content_md5 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) - - -@app.function_name(name="put_blob_bytes") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_bytes") -def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: - file.set(req.get_body()) - return 'OK' - - -@app.function_name(name="put_blob_filelike") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_filelike") -def put_blob_filelike(req: func.HttpRequest, - file: func.Out[io.StringIO]) -> str: - file.set(io.StringIO('filelike')) - return 'OK' - - -@app.function_name(name="put_blob_return") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_return", binding_arg_name="resp") -def put_blob_return(req: func.HttpRequest, - resp: func.Out[func.HttpResponse]) -> str: - return 'FROM RETURN' - - -@app.function_name(name="put_blob_str") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_str") -def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: - file.set(req.get_body()) - return 'OK' - - -@app.function_name(name="put_blob_trigger") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_trigger") -def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: - file.set(req.get_body()) - return 'OK' - - -def _generate_content_and_digest(content_size): - content = bytearray(random.getrandbits(8) for _ in range(content_size)) - content_md5 = hashlib.md5(content).hexdigest() - return content, content_md5 - - -@app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") -@app.blob_input(arg_name="inputfile1", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-1.txt", - connection="AzureWebJobsStorage") -@app.blob_input(arg_name="inputfile2", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-2.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile1", - path="python-worker-tests/shmem-test-bytes-out-1.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile2", - path="python-worker-tests/shmem-test-bytes-out-2.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_get_multiple_blobs_as_bytes_return_http_response") -def put_get_multiple_blobs_as_bytes_return_http_response( - req: func.HttpRequest, - inputfile1: bytes, - inputfile2: bytes, - outputfile1: func.Out[bytes], - outputfile2: func.Out[bytes]) -> func.HttpResponse: - """ - Read two blobs (bytes) and respond back (in HTTP response) with the number - of bytes read from each blob and the MD5 digest of the content of each. - Write two blobs (bytes) and respond back (in HTTP response) with the number - bytes written in each blob and the MD5 digest of the content of each. - The number of bytes to write are specified in the input HTTP request. - """ - input_content_size_1 = len(inputfile1) - input_content_size_2 = len(inputfile2) - - input_content_md5_1 = hashlib.md5(inputfile1).hexdigest() - input_content_md5_2 = hashlib.md5(inputfile2).hexdigest() - - output_content_size_1 = int(req.params['output_content_size_1']) - output_content_size_2 = int(req.params['output_content_size_2']) - - output_content_1, output_content_md5_1 = \ - _generate_content_and_digest(output_content_size_1) - output_content_2, output_content_md5_2 = \ - _generate_content_and_digest(output_content_size_2) - - outputfile1.set(output_content_1) - outputfile2.set(output_content_2) - - response_dict = { - 'input_content_size_1': input_content_size_1, - 'input_content_size_2': input_content_size_2, - 'input_content_md5_1': input_content_md5_1, - 'input_content_md5_2': input_content_md5_2, - 'output_content_size_1': output_content_size_1, - 'output_content_size_2': output_content_size_2, - 'output_content_md5_1': output_content_md5_1, - 'output_content_md5_2': output_content_md5_2 - } - - response_body = json.dumps(response_dict, indent=2) - - return func.HttpResponse( - body=response_body, - mimetype="application/json", - status_code=200 - ) +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import hashlib +import io +import json +import random +import string + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="blob_trigger") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +def blob_trigger(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) + + +@app.function_name(name="get_blob_as_bytes") +@app.route(route="get_blob_as_bytes") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes(req: func.HttpRequest, file: bytes) -> str: + assert isinstance(file, bytes) + return file.decode('utf-8') + + +@app.function_name(name="get_blob_as_bytes_return_http_response") +@app.route(route="get_blob_as_bytes_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ + -> func.HttpResponse: + """ + Read a blob (bytes) and respond back (in HTTP response) with the number of + bytes read and the MD5 digest of the content. + """ + assert isinstance(file, bytes) + + content_size = len(file) + content_md5 = hashlib.md5(file).hexdigest() + + response_dict = { + 'content_size': content_size, + 'content_md5': content_md5 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_as_bytes_stream_return_http_response") +@app.route(route="get_blob_as_bytes_stream_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, + file: func.InputStream) \ + -> func.HttpResponse: + """ + Read a blob (as azf.InputStream) and respond back (in HTTP response) with + the number of bytes read and the MD5 digest of the content. + """ + file_bytes = file.read() + + content_size = len(file_bytes) + content_md5 = hashlib.md5(file_bytes).hexdigest() + + response_dict = { + 'content_size': content_size, + 'content_md5': content_md5 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_as_str") +@app.route(route="get_blob_as_str") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-str.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def get_blob_as_str(req: func.HttpRequest, file: str) -> str: + assert isinstance(file, str) + return file + + +@app.function_name(name="get_blob_as_str_return_http_response") +@app.route(route="get_blob_as_str_return_http_response") +@app.blob_input(arg_name="file", + path="python-worker-tests/shmem-test-bytes.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def get_blob_as_str_return_http_response(req: func.HttpRequest, + file: str) -> func.HttpResponse: + """ + Read a blob (string) and respond back (in HTTP response) with the number of + characters read and the MD5 digest of the utf-8 encoded content. + """ + assert isinstance(file, str) + + num_chars = len(file) + content_bytes = file.encode('utf-8') + content_md5 = hashlib.md5(content_bytes).hexdigest() + + response_dict = { + 'num_chars': num_chars, + 'content_md5': content_md5 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="get_blob_bytes") +@app.route(route="get_blob_bytes") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-bytes.txt", + connection="AzureWebJobsStorage") +def get_blob_bytes(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_filelike") +@app.route(route="get_blob_filelike") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-filelike.txt", + connection="AzureWebJobsStorage") +def get_blob_filelike(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_return") +@app.route(route="get_blob_return") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-return.txt", + connection="AzureWebJobsStorage") +def get_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_str") +@app.route(route="get_blob_str") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-str.txt", + connection="AzureWebJobsStorage") +def get_blob_str(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_blob_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_blob_triggered") +def get_blob_triggered(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="put_blob_as_bytes_return_http_response") +@app.blob_output(arg_name="file", + path="python-worker-tests/shmem-test-bytes-out.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_as_bytes_return_http_response") +def put_blob_as_bytes_return_http_response(req: func.HttpRequest, + file: func.Out[ + bytes]) -> func.HttpResponse: + """ + Write a blob (bytes) and respond back (in HTTP response) with the number of + bytes written and the MD5 digest of the content. + The number of bytes to write are specified in the input HTTP request. + """ + content_size = int(req.params['content_size']) + + # When this is set, then 0x01 byte is repeated content_size number of + # times to use as input. + # This is to avoid generating random input for large size which can be + # slow. + if 'no_random_input' in req.params: + content = b'\x01' * content_size + else: + content = bytearray(random.getrandbits(8) for _ in range(content_size)) + content_md5 = hashlib.md5(content).hexdigest() + + file.set(content) + + response_dict = { + 'content_size': content_size, + 'content_md5': content_md5 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="put_blob_as_str_return_http_response") +@app.blob_output(arg_name="file", + path="python-worker-tests/shmem-test-str-out.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_as_str_return_http_response") +def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ + str]) -> func.HttpResponse: + """ + Write a blob (string) and respond back (in HTTP response) with the number of + characters written and the MD5 digest of the utf-8 encoded content. + The number of characters to write are specified in the input HTTP request. + """ + num_chars = int(req.params['num_chars']) + + content = ''.join(random.choices(string.ascii_uppercase + string.digits, + k=num_chars)) + content_bytes = content.encode('utf-8') + content_size = len(content_bytes) + content_md5 = hashlib.md5(content_bytes).hexdigest() + + file.set(content) + + response_dict = { + 'num_chars': num_chars, + 'content_size': content_size, + 'content_md5': content_md5 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) + + +@app.function_name(name="put_blob_bytes") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-bytes.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_bytes") +def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="put_blob_filelike") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-filelike.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_filelike") +def put_blob_filelike(req: func.HttpRequest, + file: func.Out[io.StringIO]) -> str: + file.set(io.StringIO('filelike')) + return 'OK' + + +@app.function_name(name="put_blob_return") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-return.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_return", binding_arg_name="resp") +def put_blob_return(req: func.HttpRequest, + resp: func.Out[func.HttpResponse]) -> str: + return 'FROM RETURN' + + +@app.function_name(name="put_blob_str") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-str.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_str") +def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="put_blob_trigger") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_trigger") +def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +def _generate_content_and_digest(content_size): + content = bytearray(random.getrandbits(8) for _ in range(content_size)) + content_md5 = hashlib.md5(content).hexdigest() + return content, content_md5 + + +@app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") +@app.blob_input(arg_name="inputfile1", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-1.txt", + connection="AzureWebJobsStorage") +@app.blob_input(arg_name="inputfile2", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-2.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="outputfile1", + path="python-worker-tests/shmem-test-bytes-out-1.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="outputfile2", + path="python-worker-tests/shmem-test-bytes-out-2.txt", + data_type="BINARY", + connection="AzureWebJobsStorage") +@app.route(route="put_get_multiple_blobs_as_bytes_return_http_response") +def put_get_multiple_blobs_as_bytes_return_http_response( + req: func.HttpRequest, + inputfile1: bytes, + inputfile2: bytes, + outputfile1: func.Out[bytes], + outputfile2: func.Out[bytes]) -> func.HttpResponse: + """ + Read two blobs (bytes) and respond back (in HTTP response) with the number + of bytes read from each blob and the MD5 digest of the content of each. + Write two blobs (bytes) and respond back (in HTTP response) with the number + bytes written in each blob and the MD5 digest of the content of each. + The number of bytes to write are specified in the input HTTP request. + """ + input_content_size_1 = len(inputfile1) + input_content_size_2 = len(inputfile2) + + input_content_md5_1 = hashlib.md5(inputfile1).hexdigest() + input_content_md5_2 = hashlib.md5(inputfile2).hexdigest() + + output_content_size_1 = int(req.params['output_content_size_1']) + output_content_size_2 = int(req.params['output_content_size_2']) + + output_content_1, output_content_md5_1 = \ + _generate_content_and_digest(output_content_size_1) + output_content_2, output_content_md5_2 = \ + _generate_content_and_digest(output_content_size_2) + + outputfile1.set(output_content_1) + outputfile2.set(output_content_2) + + response_dict = { + 'input_content_size_1': input_content_size_1, + 'input_content_size_2': input_content_size_2, + 'input_content_md5_1': input_content_md5_1, + 'input_content_md5_2': input_content_md5_2, + 'output_content_size_1': output_content_size_1, + 'output_content_size_2': output_content_size_2, + 'output_content_md5_1': output_content_md5_1, + 'output_content_md5_2': output_content_md5_2 + } + + response_body = json.dumps(response_dict, indent=2) + + return func.HttpResponse( + body=response_body, + mimetype="application/json", + status_code=200 + ) diff --git a/tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py b/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/blob_functions/blob_functions_stein/generic/function_app.py rename to tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py diff --git a/tests/endtoend/blob_functions/blob_trigger/function.json b/tests/emulator_tests/blob_functions/blob_trigger/function.json similarity index 100% rename from tests/endtoend/blob_functions/blob_trigger/function.json rename to tests/emulator_tests/blob_functions/blob_trigger/function.json diff --git a/tests/endtoend/blob_functions/blob_trigger/main.py b/tests/emulator_tests/blob_functions/blob_trigger/main.py similarity index 100% rename from tests/endtoend/blob_functions/blob_trigger/main.py rename to tests/emulator_tests/blob_functions/blob_trigger/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_bytes_stream_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_str/function.json b/tests/emulator_tests/blob_functions/get_blob_as_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_str/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_str/main.py b/tests/emulator_tests/blob_functions/get_blob_as_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_str/main.py diff --git a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/function.json b/tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str_return_http_response/function.json rename to tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py b/tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_as_str_return_http_response/main.py rename to tests/emulator_tests/blob_functions/get_blob_as_str_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/get_blob_bytes/function.json b/tests/emulator_tests/blob_functions/get_blob_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_bytes/function.json rename to tests/emulator_tests/blob_functions/get_blob_bytes/function.json diff --git a/tests/endtoend/blob_functions/get_blob_bytes/main.py b/tests/emulator_tests/blob_functions/get_blob_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_bytes/main.py rename to tests/emulator_tests/blob_functions/get_blob_bytes/main.py diff --git a/tests/endtoend/blob_functions/get_blob_filelike/function.json b/tests/emulator_tests/blob_functions/get_blob_filelike/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_filelike/function.json rename to tests/emulator_tests/blob_functions/get_blob_filelike/function.json diff --git a/tests/endtoend/blob_functions/get_blob_filelike/main.py b/tests/emulator_tests/blob_functions/get_blob_filelike/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_filelike/main.py rename to tests/emulator_tests/blob_functions/get_blob_filelike/main.py diff --git a/tests/endtoend/blob_functions/get_blob_return/function.json b/tests/emulator_tests/blob_functions/get_blob_return/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_return/function.json rename to tests/emulator_tests/blob_functions/get_blob_return/function.json diff --git a/tests/endtoend/blob_functions/get_blob_return/main.py b/tests/emulator_tests/blob_functions/get_blob_return/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_return/main.py rename to tests/emulator_tests/blob_functions/get_blob_return/main.py diff --git a/tests/endtoend/blob_functions/get_blob_str/function.json b/tests/emulator_tests/blob_functions/get_blob_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_str/function.json rename to tests/emulator_tests/blob_functions/get_blob_str/function.json diff --git a/tests/endtoend/blob_functions/get_blob_str/main.py b/tests/emulator_tests/blob_functions/get_blob_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_str/main.py rename to tests/emulator_tests/blob_functions/get_blob_str/main.py diff --git a/tests/endtoend/blob_functions/get_blob_triggered/function.json b/tests/emulator_tests/blob_functions/get_blob_triggered/function.json similarity index 100% rename from tests/endtoend/blob_functions/get_blob_triggered/function.json rename to tests/emulator_tests/blob_functions/get_blob_triggered/function.json diff --git a/tests/endtoend/blob_functions/get_blob_triggered/main.py b/tests/emulator_tests/blob_functions/get_blob_triggered/main.py similarity index 100% rename from tests/endtoend/blob_functions/get_blob_triggered/main.py rename to tests/emulator_tests/blob_functions/get_blob_triggered/main.py diff --git a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_blob_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_str_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_as_str_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_blob_as_str_return_http_response/main.py diff --git a/tests/endtoend/blob_functions/put_blob_bytes/function.json b/tests/emulator_tests/blob_functions/put_blob_bytes/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_bytes/function.json rename to tests/emulator_tests/blob_functions/put_blob_bytes/function.json diff --git a/tests/endtoend/blob_functions/put_blob_bytes/main.py b/tests/emulator_tests/blob_functions/put_blob_bytes/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_bytes/main.py rename to tests/emulator_tests/blob_functions/put_blob_bytes/main.py diff --git a/tests/endtoend/blob_functions/put_blob_filelike/function.json b/tests/emulator_tests/blob_functions/put_blob_filelike/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_filelike/function.json rename to tests/emulator_tests/blob_functions/put_blob_filelike/function.json diff --git a/tests/endtoend/blob_functions/put_blob_filelike/main.py b/tests/emulator_tests/blob_functions/put_blob_filelike/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_filelike/main.py rename to tests/emulator_tests/blob_functions/put_blob_filelike/main.py diff --git a/tests/endtoend/blob_functions/put_blob_return/function.json b/tests/emulator_tests/blob_functions/put_blob_return/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_return/function.json rename to tests/emulator_tests/blob_functions/put_blob_return/function.json diff --git a/tests/endtoend/blob_functions/put_blob_return/main.py b/tests/emulator_tests/blob_functions/put_blob_return/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_return/main.py rename to tests/emulator_tests/blob_functions/put_blob_return/main.py diff --git a/tests/endtoend/blob_functions/put_blob_str/function.json b/tests/emulator_tests/blob_functions/put_blob_str/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_str/function.json rename to tests/emulator_tests/blob_functions/put_blob_str/function.json diff --git a/tests/endtoend/blob_functions/put_blob_str/main.py b/tests/emulator_tests/blob_functions/put_blob_str/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_str/main.py rename to tests/emulator_tests/blob_functions/put_blob_str/main.py diff --git a/tests/endtoend/blob_functions/put_blob_trigger/function.json b/tests/emulator_tests/blob_functions/put_blob_trigger/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_blob_trigger/function.json rename to tests/emulator_tests/blob_functions/put_blob_trigger/function.json diff --git a/tests/endtoend/blob_functions/put_blob_trigger/main.py b/tests/emulator_tests/blob_functions/put_blob_trigger/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_blob_trigger/main.py rename to tests/emulator_tests/blob_functions/put_blob_trigger/main.py diff --git a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json b/tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json similarity index 100% rename from tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json rename to tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/function.json diff --git a/tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py b/tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py similarity index 100% rename from tests/endtoend/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py rename to tests/emulator_tests/blob_functions/put_get_multiple_blobs_as_bytes_return_http_response/main.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json b/tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json rename to tests/emulator_tests/eventhub_batch_functions/eventhub_multiple/function.json diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_output_batch/__init__.py b/tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_output_batch/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/eventhub_output_batch/function.json b/tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/eventhub_output_batch/function.json rename to tests/emulator_tests/eventhub_batch_functions/eventhub_output_batch/function.json diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json similarity index 95% rename from tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json rename to tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json index 3e8a6995d..8ec2e9d65 100644 --- a/tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json +++ b/tests/emulator_tests/eventhub_batch_functions/get_eventhub_batch_triggered/function.json @@ -1,26 +1,26 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "type": "httpTrigger", - "direction": "in", - "authLevel": "anonymous", - "methods": [ - "get" - ], - "name": "req" - }, - { - "direction": "in", - "type": "blob", - "name": "testEntities", - "path": "python-worker-tests/test-eventhub-batch-triggered.txt", - "connection": "AzureWebJobsStorage" - }, - { - "type": "http", - "direction": "out", - "name": "$return" - } - ] +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": [ + "get" + ], + "name": "req" + }, + { + "direction": "in", + "type": "blob", + "name": "testEntities", + "path": "python-worker-tests/test-eventhub-batch-triggered.txt", + "connection": "AzureWebJobsStorage" + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] } \ No newline at end of file diff --git a/tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py b/tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/function.json b/tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/get_metadata_batch_triggered/function.json rename to tests/emulator_tests/eventhub_batch_functions/get_metadata_batch_triggered/function.json diff --git a/tests/endtoend/eventhub_batch_functions/metadata_multiple/__init__.py b/tests/emulator_tests/eventhub_batch_functions/metadata_multiple/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_multiple/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/metadata_multiple/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/metadata_multiple/function.json b/tests/emulator_tests/eventhub_batch_functions/metadata_multiple/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_multiple/function.json rename to tests/emulator_tests/eventhub_batch_functions/metadata_multiple/function.json diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py b/tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/__init__.py similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_output_batch/__init__.py rename to tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/__init__.py diff --git a/tests/endtoend/eventhub_batch_functions/metadata_output_batch/function.json b/tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/function.json similarity index 100% rename from tests/endtoend/eventhub_batch_functions/metadata_output_batch/function.json rename to tests/emulator_tests/eventhub_batch_functions/metadata_output_batch/function.json diff --git a/tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py rename to tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py index 1091f8dd5..1481f7b55 100644 --- a/tests/endtoend/eventhub_functions/eventhub_functions_stein/function_app.py +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/function_app.py @@ -1,107 +1,107 @@ -import json -import os -import typing - -import azure.functions as func -from azure.eventhub import EventData -from azure.eventhub.aio import EventHubProducerClient - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -# An HttpTrigger to generating EventHub event from EventHub Output Binding -@app.function_name(name="eventhub_output") -@app.route(route="eventhub_output") -@app.event_hub_output(arg_name="event", - event_hub_name="python-worker-ci-eventhub-one", - connection="AzureWebJobsEventHubConnectionString") -def eventhub_output(req: func.HttpRequest, event: func.Out[str]): - event.set(req.get_body().decode('utf-8')) - return 'OK' - - -# This is an actual EventHub trigger which will convert the event data -# into a storage blob. -@app.function_name(name="eventhub_trigger") -@app.event_hub_message_trigger(arg_name="event", - event_hub_name="python-worker-ci-eventhub-one", - connection="AzureWebJobsEventHubConnectionString" - ) -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-eventhub-triggered.txt", - connection="AzureWebJobsStorage") -def eventhub_trigger(event: func.EventHubEvent) -> bytes: - return event.get_body() - - -# Retrieve the event data from storage blob and return it as Http response -@app.function_name(name="get_eventhub_triggered") -@app.route(route="get_eventhub_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-eventhub-triggered.txt", - connection="AzureWebJobsStorage") -def get_eventhub_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -# Retrieve the event data from storage blob and return it as Http response -@app.function_name(name="get_metadata_triggered") -@app.route(route="get_metadata_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-metadata-triggered.txt", - connection="AzureWebJobsStorage") -async def get_metadata_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse(body=file.read().decode('utf-8'), - status_code=200, - mimetype='application/json') - - -# An HttpTrigger to generating EventHub event from azure-eventhub SDK. -# Events generated from azure-eventhub contain the full metadata. -@app.function_name(name="metadata_output") -@app.route(route="metadata_output") -async def metadata_output(req: func.HttpRequest): - # Parse event metadata from http request - json_string = req.get_body().decode('utf-8') - event_dict = json.loads(json_string) - - # Create an EventHub Client and event batch - client = EventHubProducerClient.from_connection_string( - os.getenv('AzureWebJobsEventHubConnectionString'), - eventhub_name='python-worker-ci-eventhub-one-metadata') - - # Generate new event based on http request with full metadata - event_data_batch = await client.create_batch() - event_data_batch.add(EventData(event_dict.get('body'))) - - # Send out event into event hub - try: - await client.send_batch(event_data_batch) - finally: - await client.close() - - return 'OK' - - -@app.function_name(name="metadata_trigger") -@app.event_hub_message_trigger( - arg_name="event", - event_hub_name="python-worker-ci-eventhub-one-metadata", - connection="AzureWebJobsEventHubConnectionString") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-metadata-triggered.txt", - connection="AzureWebJobsStorage") -async def metadata_trigger(event: func.EventHubEvent) -> bytes: - event_dict: typing.Mapping[str, typing.Any] = { - 'body': event.get_body().decode('utf-8'), - # Uncomment this when the EnqueuedTimeUtc is fixed in azure-functions - # 'enqueued_time': event.enqueued_time.isoformat(), - 'partition_key': event.partition_key, - 'sequence_number': event.sequence_number, - 'offset': event.offset, - 'metadata': event.metadata - } - - return json.dumps(event_dict) +import json +import os +import typing + +import azure.functions as func +from azure.eventhub import EventData +from azure.eventhub.aio import EventHubProducerClient + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +# An HttpTrigger to generating EventHub event from EventHub Output Binding +@app.function_name(name="eventhub_output") +@app.route(route="eventhub_output") +@app.event_hub_output(arg_name="event", + event_hub_name="python-worker-ci-eventhub-one", + connection="AzureWebJobsEventHubConnectionString") +def eventhub_output(req: func.HttpRequest, event: func.Out[str]): + event.set(req.get_body().decode('utf-8')) + return 'OK' + + +# This is an actual EventHub trigger which will convert the event data +# into a storage blob. +@app.function_name(name="eventhub_trigger") +@app.event_hub_message_trigger(arg_name="event", + event_hub_name="python-worker-ci-eventhub-one", + connection="AzureWebJobsEventHubConnectionString" + ) +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-eventhub-triggered.txt", + connection="AzureWebJobsStorage") +def eventhub_trigger(event: func.EventHubEvent) -> bytes: + return event.get_body() + + +# Retrieve the event data from storage blob and return it as Http response +@app.function_name(name="get_eventhub_triggered") +@app.route(route="get_eventhub_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-eventhub-triggered.txt", + connection="AzureWebJobsStorage") +def get_eventhub_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +# Retrieve the event data from storage blob and return it as Http response +@app.function_name(name="get_metadata_triggered") +@app.route(route="get_metadata_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-metadata-triggered.txt", + connection="AzureWebJobsStorage") +async def get_metadata_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse(body=file.read().decode('utf-8'), + status_code=200, + mimetype='application/json') + + +# An HttpTrigger to generating EventHub event from azure-eventhub SDK. +# Events generated from azure-eventhub contain the full metadata. +@app.function_name(name="metadata_output") +@app.route(route="metadata_output") +async def metadata_output(req: func.HttpRequest): + # Parse event metadata from http request + json_string = req.get_body().decode('utf-8') + event_dict = json.loads(json_string) + + # Create an EventHub Client and event batch + client = EventHubProducerClient.from_connection_string( + os.getenv('AzureWebJobsEventHubConnectionString'), + eventhub_name='python-worker-ci-eventhub-one-metadata') + + # Generate new event based on http request with full metadata + event_data_batch = await client.create_batch() + event_data_batch.add(EventData(event_dict.get('body'))) + + # Send out event into event hub + try: + await client.send_batch(event_data_batch) + finally: + await client.close() + + return 'OK' + + +@app.function_name(name="metadata_trigger") +@app.event_hub_message_trigger( + arg_name="event", + event_hub_name="python-worker-ci-eventhub-one-metadata", + connection="AzureWebJobsEventHubConnectionString") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-metadata-triggered.txt", + connection="AzureWebJobsStorage") +async def metadata_trigger(event: func.EventHubEvent) -> bytes: + event_dict: typing.Mapping[str, typing.Any] = { + 'body': event.get_body().decode('utf-8'), + # Uncomment this when the EnqueuedTimeUtc is fixed in azure-functions + # 'enqueued_time': event.enqueued_time.isoformat(), + 'partition_key': event.partition_key, + 'sequence_number': event.sequence_number, + 'offset': event.offset, + 'metadata': event.metadata + } + + return json.dumps(event_dict) diff --git a/tests/endtoend/eventhub_functions/eventhub_functions_stein/generic/function_app.py b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_functions_stein/generic/function_app.py rename to tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/function_app.py diff --git a/tests/endtoend/eventhub_functions/eventhub_output/__init__.py b/tests/emulator_tests/eventhub_functions/eventhub_output/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_output/__init__.py rename to tests/emulator_tests/eventhub_functions/eventhub_output/__init__.py diff --git a/tests/endtoend/eventhub_functions/eventhub_output/function.json b/tests/emulator_tests/eventhub_functions/eventhub_output/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_output/function.json rename to tests/emulator_tests/eventhub_functions/eventhub_output/function.json diff --git a/tests/endtoend/eventhub_functions/eventhub_trigger/__init__.py b/tests/emulator_tests/eventhub_functions/eventhub_trigger/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_trigger/__init__.py rename to tests/emulator_tests/eventhub_functions/eventhub_trigger/__init__.py diff --git a/tests/endtoend/eventhub_functions/eventhub_trigger/function.json b/tests/emulator_tests/eventhub_functions/eventhub_trigger/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/eventhub_trigger/function.json rename to tests/emulator_tests/eventhub_functions/eventhub_trigger/function.json diff --git a/tests/endtoend/eventhub_functions/get_eventhub_triggered/function.json b/tests/emulator_tests/eventhub_functions/get_eventhub_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/get_eventhub_triggered/function.json rename to tests/emulator_tests/eventhub_functions/get_eventhub_triggered/function.json diff --git a/tests/endtoend/eventhub_functions/get_eventhub_triggered/main.py b/tests/emulator_tests/eventhub_functions/get_eventhub_triggered/main.py similarity index 100% rename from tests/endtoend/eventhub_functions/get_eventhub_triggered/main.py rename to tests/emulator_tests/eventhub_functions/get_eventhub_triggered/main.py diff --git a/tests/endtoend/eventhub_functions/get_metadata_triggered/__init__.py b/tests/emulator_tests/eventhub_functions/get_metadata_triggered/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/get_metadata_triggered/__init__.py rename to tests/emulator_tests/eventhub_functions/get_metadata_triggered/__init__.py diff --git a/tests/endtoend/eventhub_functions/get_metadata_triggered/function.json b/tests/emulator_tests/eventhub_functions/get_metadata_triggered/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/get_metadata_triggered/function.json rename to tests/emulator_tests/eventhub_functions/get_metadata_triggered/function.json diff --git a/tests/endtoend/eventhub_functions/metadata_output/__init__.py b/tests/emulator_tests/eventhub_functions/metadata_output/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_output/__init__.py rename to tests/emulator_tests/eventhub_functions/metadata_output/__init__.py diff --git a/tests/endtoend/eventhub_functions/metadata_output/function.json b/tests/emulator_tests/eventhub_functions/metadata_output/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_output/function.json rename to tests/emulator_tests/eventhub_functions/metadata_output/function.json diff --git a/tests/endtoend/eventhub_functions/metadata_trigger/__init__.py b/tests/emulator_tests/eventhub_functions/metadata_trigger/__init__.py similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_trigger/__init__.py rename to tests/emulator_tests/eventhub_functions/metadata_trigger/__init__.py diff --git a/tests/endtoend/eventhub_functions/metadata_trigger/function.json b/tests/emulator_tests/eventhub_functions/metadata_trigger/function.json similarity index 100% rename from tests/endtoend/eventhub_functions/metadata_trigger/function.json rename to tests/emulator_tests/eventhub_functions/metadata_trigger/function.json diff --git a/tests/endtoend/generic_functions/generic_functions_stein/function_app.py b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py similarity index 100% rename from tests/endtoend/generic_functions/generic_functions_stein/function_app.py rename to tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py diff --git a/tests/endtoend/generic_functions/return_bool/function.json b/tests/emulator_tests/generic_functions/return_bool/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_bool/function.json rename to tests/emulator_tests/generic_functions/return_bool/function.json diff --git a/tests/endtoend/generic_functions/return_bool/main.py b/tests/emulator_tests/generic_functions/return_bool/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_bool/main.py rename to tests/emulator_tests/generic_functions/return_bool/main.py diff --git a/tests/endtoend/generic_functions/return_bytes/function.json b/tests/emulator_tests/generic_functions/return_bytes/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_bytes/function.json rename to tests/emulator_tests/generic_functions/return_bytes/function.json diff --git a/tests/endtoend/generic_functions/return_bytes/main.py b/tests/emulator_tests/generic_functions/return_bytes/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_bytes/main.py rename to tests/emulator_tests/generic_functions/return_bytes/main.py diff --git a/tests/endtoend/generic_functions/return_dict/function.json b/tests/emulator_tests/generic_functions/return_dict/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_dict/function.json rename to tests/emulator_tests/generic_functions/return_dict/function.json diff --git a/tests/endtoend/generic_functions/return_dict/main.py b/tests/emulator_tests/generic_functions/return_dict/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_dict/main.py rename to tests/emulator_tests/generic_functions/return_dict/main.py diff --git a/tests/endtoend/generic_functions/return_double/function.json b/tests/emulator_tests/generic_functions/return_double/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_double/function.json rename to tests/emulator_tests/generic_functions/return_double/function.json diff --git a/tests/endtoend/generic_functions/return_double/main.py b/tests/emulator_tests/generic_functions/return_double/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_double/main.py rename to tests/emulator_tests/generic_functions/return_double/main.py diff --git a/tests/endtoend/generic_functions/return_int/function.json b/tests/emulator_tests/generic_functions/return_int/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_int/function.json rename to tests/emulator_tests/generic_functions/return_int/function.json diff --git a/tests/endtoend/generic_functions/return_int/main.py b/tests/emulator_tests/generic_functions/return_int/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_int/main.py rename to tests/emulator_tests/generic_functions/return_int/main.py diff --git a/tests/endtoend/generic_functions/return_list/function.json b/tests/emulator_tests/generic_functions/return_list/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_list/function.json rename to tests/emulator_tests/generic_functions/return_list/function.json diff --git a/tests/endtoend/generic_functions/return_list/main.py b/tests/emulator_tests/generic_functions/return_list/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_list/main.py rename to tests/emulator_tests/generic_functions/return_list/main.py diff --git a/tests/endtoend/generic_functions/return_none/function.json b/tests/emulator_tests/generic_functions/return_none/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_none/function.json rename to tests/emulator_tests/generic_functions/return_none/function.json diff --git a/tests/endtoend/generic_functions/return_none/main.py b/tests/emulator_tests/generic_functions/return_none/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_none/main.py rename to tests/emulator_tests/generic_functions/return_none/main.py diff --git a/tests/endtoend/generic_functions/return_none_no_type_hint/function.json b/tests/emulator_tests/generic_functions/return_none_no_type_hint/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_none_no_type_hint/function.json rename to tests/emulator_tests/generic_functions/return_none_no_type_hint/function.json diff --git a/tests/endtoend/generic_functions/return_none_no_type_hint/main.py b/tests/emulator_tests/generic_functions/return_none_no_type_hint/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_none_no_type_hint/main.py rename to tests/emulator_tests/generic_functions/return_none_no_type_hint/main.py diff --git a/tests/endtoend/generic_functions/return_not_processed_last/__init__.py b/tests/emulator_tests/generic_functions/return_not_processed_last/__init__.py similarity index 100% rename from tests/endtoend/generic_functions/return_not_processed_last/__init__.py rename to tests/emulator_tests/generic_functions/return_not_processed_last/__init__.py diff --git a/tests/endtoend/generic_functions/return_not_processed_last/function.json b/tests/emulator_tests/generic_functions/return_not_processed_last/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_not_processed_last/function.json rename to tests/emulator_tests/generic_functions/return_not_processed_last/function.json diff --git a/tests/endtoend/generic_functions/return_processed_last/__init__.py b/tests/emulator_tests/generic_functions/return_processed_last/__init__.py similarity index 100% rename from tests/endtoend/generic_functions/return_processed_last/__init__.py rename to tests/emulator_tests/generic_functions/return_processed_last/__init__.py diff --git a/tests/endtoend/generic_functions/return_processed_last/function.json b/tests/emulator_tests/generic_functions/return_processed_last/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_processed_last/function.json rename to tests/emulator_tests/generic_functions/return_processed_last/function.json diff --git a/tests/endtoend/generic_functions/return_string/function.json b/tests/emulator_tests/generic_functions/return_string/function.json similarity index 100% rename from tests/endtoend/generic_functions/return_string/function.json rename to tests/emulator_tests/generic_functions/return_string/function.json diff --git a/tests/endtoend/generic_functions/return_string/main.py b/tests/emulator_tests/generic_functions/return_string/main.py similarity index 100% rename from tests/endtoend/generic_functions/return_string/main.py rename to tests/emulator_tests/generic_functions/return_string/main.py diff --git a/tests/endtoend/queue_functions/get_queue_blob/function.json b/tests/emulator_tests/queue_functions/get_queue_blob/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob/main.py b/tests/emulator_tests/queue_functions/get_queue_blob/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob/main.py diff --git a/tests/endtoend/queue_functions/get_queue_blob_message_return/function.json b/tests/emulator_tests/queue_functions/get_queue_blob_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_message_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob_message_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob_message_return/main.py b/tests/emulator_tests/queue_functions/get_queue_blob_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_message_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob_message_return/main.py diff --git a/tests/endtoend/queue_functions/get_queue_blob_return/function.json b/tests/emulator_tests/queue_functions/get_queue_blob_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_blob_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_blob_return/main.py b/tests/emulator_tests/queue_functions/get_queue_blob_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_blob_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_blob_return/main.py diff --git a/tests/endtoend/queue_functions/get_queue_untyped_blob_return/function.json b/tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/get_queue_untyped_blob_return/function.json rename to tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/function.json diff --git a/tests/endtoend/queue_functions/get_queue_untyped_blob_return/main.py b/tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/get_queue_untyped_blob_return/main.py rename to tests/emulator_tests/queue_functions/get_queue_untyped_blob_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue/function.json b/tests/emulator_tests/queue_functions/put_queue/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue/function.json rename to tests/emulator_tests/queue_functions/put_queue/function.json diff --git a/tests/endtoend/queue_functions/put_queue/main.py b/tests/emulator_tests/queue_functions/put_queue/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue/main.py rename to tests/emulator_tests/queue_functions/put_queue/main.py diff --git a/tests/endtoend/queue_functions/put_queue_message_return/function.json b/tests/emulator_tests/queue_functions/put_queue_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_message_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_message_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_message_return/main.py b/tests/emulator_tests/queue_functions/put_queue_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_message_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_message_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue_multiple_out/function.json b/tests/emulator_tests/queue_functions/put_queue_multiple_out/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_multiple_out/function.json rename to tests/emulator_tests/queue_functions/put_queue_multiple_out/function.json diff --git a/tests/endtoend/queue_functions/put_queue_multiple_out/main.py b/tests/emulator_tests/queue_functions/put_queue_multiple_out/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_multiple_out/main.py rename to tests/emulator_tests/queue_functions/put_queue_multiple_out/main.py diff --git a/tests/endtoend/queue_functions/put_queue_return/function.json b/tests/emulator_tests/queue_functions/put_queue_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_return/main.py b/tests/emulator_tests/queue_functions/put_queue_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_return/main.py diff --git a/tests/endtoend/queue_functions/put_queue_return_multiple/function.json b/tests/emulator_tests/queue_functions/put_queue_return_multiple/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return_multiple/function.json rename to tests/emulator_tests/queue_functions/put_queue_return_multiple/function.json diff --git a/tests/endtoend/queue_functions/put_queue_return_multiple/main.py b/tests/emulator_tests/queue_functions/put_queue_return_multiple/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_return_multiple/main.py rename to tests/emulator_tests/queue_functions/put_queue_return_multiple/main.py diff --git a/tests/endtoend/queue_functions/put_queue_untyped_return/function.json b/tests/emulator_tests/queue_functions/put_queue_untyped_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/put_queue_untyped_return/function.json rename to tests/emulator_tests/queue_functions/put_queue_untyped_return/function.json diff --git a/tests/endtoend/queue_functions/put_queue_untyped_return/main.py b/tests/emulator_tests/queue_functions/put_queue_untyped_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/put_queue_untyped_return/main.py rename to tests/emulator_tests/queue_functions/put_queue_untyped_return/main.py diff --git a/tests/endtoend/queue_functions/queue_functions_stein/function_app.py b/tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py similarity index 97% rename from tests/endtoend/queue_functions/queue_functions_stein/function_app.py rename to tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py index 0b883fb12..087cf4592 100644 --- a/tests/endtoend/queue_functions/queue_functions_stein/function_app.py +++ b/tests/emulator_tests/queue_functions/queue_functions_stein/function_app.py @@ -1,185 +1,185 @@ -import json -import logging -import typing - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="get_queue_blob") -@app.route(route="get_queue_blob") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob.txt") -def get_queue_blob(req: func.HttpRequest, file: func.InputStream) -> str: - return json.dumps({ - 'queue': json.loads(file.read().decode('utf-8')) - }) - - -@app.function_name(name="get_queue_blob_message_return") -@app.route(route="get_queue_blob_message_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-message-return.txt") -def get_queue_blob_message_return(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_queue_blob_return") -@app.route(route="get_queue_blob_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-return.txt") -def get_queue_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="get_queue_untyped_blob_return") -@app.route(route="get_queue_untyped_blob_return") -@app.blob_input(arg_name="file", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-untyped-blob-return.txt") -def get_queue_untyped_blob_return(req: func.HttpRequest, - file: func.InputStream) -> str: - return file.read().decode('utf-8') - - -@app.function_name(name="put_queue") -@app.route(route="put_queue") -@app.queue_output(arg_name="msg", - connection="AzureWebJobsStorage", - queue_name="testqueue") -def put_queue(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body()) - - return 'OK' - - -@app.function_name(name="put_queue_message_return") -@app.route(route="put_queue_message_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-message-return") -def main(req: func.HttpRequest, resp: func.Out[str]) -> bytes: - return func.QueueMessage(body=req.get_body()) - - -@app.function_name("put_queue_multiple_out") -@app.route(route="put_queue_multiple_out", binding_arg_name="resp") -@app.queue_output(arg_name="msg", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple-outparam") -def put_queue_multiple_out(req: func.HttpRequest, - resp: func.Out[func.HttpResponse], - msg: func.Out[func.QueueMessage]) -> None: - data = req.get_body().decode() - msg.set(func.QueueMessage(body=data)) - resp.set(func.HttpResponse(body='HTTP response: {}'.format(data))) - - -@app.function_name("put_queue_return") -@app.route(route="put_queue_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-return") -def put_queue_return(req: func.HttpRequest, resp: func.Out[str]) -> bytes: - return req.get_body() - - -@app.function_name(name="put_queue_multiple_return") -@app.route(route="put_queue_multiple_return") -@app.queue_output(arg_name="msgs", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple") -def put_queue_multiple_return(req: func.HttpRequest, - msgs: func.Out[typing.List[str]]): - msgs.set(['one', 'two']) - - -@app.function_name(name="put_queue_untyped_return") -@app.route(route="put_queue_untyped_return", binding_arg_name="resp") -@app.queue_output(arg_name="$return", - connection="AzureWebJobsStorage", - queue_name="testqueue-untyped-return") -def put_queue_untyped_return(req: func.HttpRequest, - resp: func.Out[str]) -> bytes: - return func.QueueMessage(body=req.get_body()) - - -@app.function_name(name="queue_trigger") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob.txt") -def queue_trigger(msg: func.QueueMessage) -> str: - result = json.dumps({ - 'id': msg.id, - 'body': msg.get_body().decode('utf-8'), - 'expiration_time': (msg.expiration_time.isoformat() - if msg.expiration_time else None), - 'insertion_time': (msg.insertion_time.isoformat() - if msg.insertion_time else None), - 'time_next_visible': (msg.time_next_visible.isoformat() - if msg.time_next_visible else None), - 'pop_receipt': msg.pop_receipt, - 'dequeue_count': msg.dequeue_count - }) - - return result - - -@app.function_name(name="queue_trigger_message_return") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-message-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-message-return.txt") -def queue_trigger_message_return(msg: func.QueueMessage) -> bytes: - return msg.get_body() - - -@app.function_name(name="queue_trigger_return") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-blob-return.txt") -def queue_trigger_return(msg: func.QueueMessage) -> bytes: - return msg.get_body() - - -@app.function_name(name="queue_trigger_return_multiple") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-return-multiple", - connection="AzureWebJobsStorage") -def queue_trigger_return_multiple(msg: func.QueueMessage) -> None: - logging.info('trigger on message: %s', msg.get_body().decode('utf-8')) - - -@app.function_name(name="queue_trigger_untyped") -@app.queue_trigger(arg_name="msg", - queue_name="testqueue-untyped-return", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - connection="AzureWebJobsStorage", - path="python-worker-tests/test-queue-untyped-blob-return.txt") -def queue_trigger_untyped(msg: str) -> str: - return msg - - -@app.function_name(name="put_queue_return_multiple") -@app.route(route="put_queue_return_multiple", binding_arg_name="resp") -@app.queue_output(arg_name="msgs", - connection="AzureWebJobsStorage", - queue_name="testqueue-return-multiple") -def put_queue_return_multiple(req: func.HttpRequest, - resp: func.Out[str], - msgs: func.Out[typing.List[str]]): - msgs.set(['one', 'two']) +import json +import logging +import typing + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="get_queue_blob") +@app.route(route="get_queue_blob") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob.txt") +def get_queue_blob(req: func.HttpRequest, file: func.InputStream) -> str: + return json.dumps({ + 'queue': json.loads(file.read().decode('utf-8')) + }) + + +@app.function_name(name="get_queue_blob_message_return") +@app.route(route="get_queue_blob_message_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-message-return.txt") +def get_queue_blob_message_return(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_queue_blob_return") +@app.route(route="get_queue_blob_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-return.txt") +def get_queue_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="get_queue_untyped_blob_return") +@app.route(route="get_queue_untyped_blob_return") +@app.blob_input(arg_name="file", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-untyped-blob-return.txt") +def get_queue_untyped_blob_return(req: func.HttpRequest, + file: func.InputStream) -> str: + return file.read().decode('utf-8') + + +@app.function_name(name="put_queue") +@app.route(route="put_queue") +@app.queue_output(arg_name="msg", + connection="AzureWebJobsStorage", + queue_name="testqueue") +def put_queue(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body()) + + return 'OK' + + +@app.function_name(name="put_queue_message_return") +@app.route(route="put_queue_message_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-message-return") +def main(req: func.HttpRequest, resp: func.Out[str]) -> bytes: + return func.QueueMessage(body=req.get_body()) + + +@app.function_name("put_queue_multiple_out") +@app.route(route="put_queue_multiple_out", binding_arg_name="resp") +@app.queue_output(arg_name="msg", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple-outparam") +def put_queue_multiple_out(req: func.HttpRequest, + resp: func.Out[func.HttpResponse], + msg: func.Out[func.QueueMessage]) -> None: + data = req.get_body().decode() + msg.set(func.QueueMessage(body=data)) + resp.set(func.HttpResponse(body='HTTP response: {}'.format(data))) + + +@app.function_name("put_queue_return") +@app.route(route="put_queue_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-return") +def put_queue_return(req: func.HttpRequest, resp: func.Out[str]) -> bytes: + return req.get_body() + + +@app.function_name(name="put_queue_multiple_return") +@app.route(route="put_queue_multiple_return") +@app.queue_output(arg_name="msgs", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple") +def put_queue_multiple_return(req: func.HttpRequest, + msgs: func.Out[typing.List[str]]): + msgs.set(['one', 'two']) + + +@app.function_name(name="put_queue_untyped_return") +@app.route(route="put_queue_untyped_return", binding_arg_name="resp") +@app.queue_output(arg_name="$return", + connection="AzureWebJobsStorage", + queue_name="testqueue-untyped-return") +def put_queue_untyped_return(req: func.HttpRequest, + resp: func.Out[str]) -> bytes: + return func.QueueMessage(body=req.get_body()) + + +@app.function_name(name="queue_trigger") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob.txt") +def queue_trigger(msg: func.QueueMessage) -> str: + result = json.dumps({ + 'id': msg.id, + 'body': msg.get_body().decode('utf-8'), + 'expiration_time': (msg.expiration_time.isoformat() + if msg.expiration_time else None), + 'insertion_time': (msg.insertion_time.isoformat() + if msg.insertion_time else None), + 'time_next_visible': (msg.time_next_visible.isoformat() + if msg.time_next_visible else None), + 'pop_receipt': msg.pop_receipt, + 'dequeue_count': msg.dequeue_count + }) + + return result + + +@app.function_name(name="queue_trigger_message_return") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-message-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-message-return.txt") +def queue_trigger_message_return(msg: func.QueueMessage) -> bytes: + return msg.get_body() + + +@app.function_name(name="queue_trigger_return") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-blob-return.txt") +def queue_trigger_return(msg: func.QueueMessage) -> bytes: + return msg.get_body() + + +@app.function_name(name="queue_trigger_return_multiple") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-return-multiple", + connection="AzureWebJobsStorage") +def queue_trigger_return_multiple(msg: func.QueueMessage) -> None: + logging.info('trigger on message: %s', msg.get_body().decode('utf-8')) + + +@app.function_name(name="queue_trigger_untyped") +@app.queue_trigger(arg_name="msg", + queue_name="testqueue-untyped-return", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-queue-untyped-blob-return.txt") +def queue_trigger_untyped(msg: str) -> str: + return msg + + +@app.function_name(name="put_queue_return_multiple") +@app.route(route="put_queue_return_multiple", binding_arg_name="resp") +@app.queue_output(arg_name="msgs", + connection="AzureWebJobsStorage", + queue_name="testqueue-return-multiple") +def put_queue_return_multiple(req: func.HttpRequest, + resp: func.Out[str], + msgs: func.Out[typing.List[str]]): + msgs.set(['one', 'two']) diff --git a/tests/endtoend/queue_functions/queue_functions_stein/generic/function_app.py b/tests/emulator_tests/queue_functions/queue_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/queue_functions/queue_functions_stein/generic/function_app.py rename to tests/emulator_tests/queue_functions/queue_functions_stein/generic/function_app.py diff --git a/tests/endtoend/queue_functions/queue_trigger/function.json b/tests/emulator_tests/queue_functions/queue_trigger/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger/function.json rename to tests/emulator_tests/queue_functions/queue_trigger/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger/main.py b/tests/emulator_tests/queue_functions/queue_trigger/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger/main.py rename to tests/emulator_tests/queue_functions/queue_trigger/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_message_return/function.json b/tests/emulator_tests/queue_functions/queue_trigger_message_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_message_return/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_message_return/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_message_return/main.py b/tests/emulator_tests/queue_functions/queue_trigger_message_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_message_return/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_message_return/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_return/function.json b/tests/emulator_tests/queue_functions/queue_trigger_return/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_return/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_return/main.py b/tests/emulator_tests/queue_functions/queue_trigger_return/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_return/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_return_multiple/function.json b/tests/emulator_tests/queue_functions/queue_trigger_return_multiple/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return_multiple/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_return_multiple/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_return_multiple/main.py b/tests/emulator_tests/queue_functions/queue_trigger_return_multiple/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_return_multiple/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_return_multiple/main.py diff --git a/tests/endtoend/queue_functions/queue_trigger_untyped/function.json b/tests/emulator_tests/queue_functions/queue_trigger_untyped/function.json similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_untyped/function.json rename to tests/emulator_tests/queue_functions/queue_trigger_untyped/function.json diff --git a/tests/endtoend/queue_functions/queue_trigger_untyped/main.py b/tests/emulator_tests/queue_functions/queue_trigger_untyped/main.py similarity index 100% rename from tests/endtoend/queue_functions/queue_trigger_untyped/main.py rename to tests/emulator_tests/queue_functions/queue_trigger_untyped/main.py diff --git a/tests/endtoend/table_functions/table_functions_stein/function_app.py b/tests/emulator_tests/table_functions/table_functions_stein/function_app.py similarity index 100% rename from tests/endtoend/table_functions/table_functions_stein/function_app.py rename to tests/emulator_tests/table_functions/table_functions_stein/function_app.py diff --git a/tests/endtoend/table_functions/table_functions_stein/generic/function_app.py b/tests/emulator_tests/table_functions/table_functions_stein/generic/function_app.py similarity index 100% rename from tests/endtoend/table_functions/table_functions_stein/generic/function_app.py rename to tests/emulator_tests/table_functions/table_functions_stein/generic/function_app.py diff --git a/tests/endtoend/table_functions/table_in_binding/__init__.py b/tests/emulator_tests/table_functions/table_in_binding/__init__.py similarity index 100% rename from tests/endtoend/table_functions/table_in_binding/__init__.py rename to tests/emulator_tests/table_functions/table_in_binding/__init__.py diff --git a/tests/endtoend/table_functions/table_in_binding/function.json b/tests/emulator_tests/table_functions/table_in_binding/function.json similarity index 100% rename from tests/endtoend/table_functions/table_in_binding/function.json rename to tests/emulator_tests/table_functions/table_in_binding/function.json diff --git a/tests/endtoend/table_functions/table_out_binding/__init__.py b/tests/emulator_tests/table_functions/table_out_binding/__init__.py similarity index 100% rename from tests/endtoend/table_functions/table_out_binding/__init__.py rename to tests/emulator_tests/table_functions/table_out_binding/__init__.py diff --git a/tests/endtoend/table_functions/table_out_binding/function.json b/tests/emulator_tests/table_functions/table_out_binding/function.json similarity index 100% rename from tests/endtoend/table_functions/table_out_binding/function.json rename to tests/emulator_tests/table_functions/table_out_binding/function.json diff --git a/tests/endtoend/test_blob_functions.py b/tests/emulator_tests/test_blob_functions.py similarity index 96% rename from tests/endtoend/test_blob_functions.py rename to tests/emulator_tests/test_blob_functions.py index 4d51693e9..8ec760b86 100644 --- a/tests/endtoend/test_blob_functions.py +++ b/tests/emulator_tests/test_blob_functions.py @@ -10,7 +10,7 @@ class TestBlobFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' @testutils.retryable_test(3, 5) def test_blob_io_str(self): @@ -154,7 +154,7 @@ class TestBlobFunctionsStein(TestBlobFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ 'blob_functions_stein' @@ -162,5 +162,5 @@ class TestBlobFunctionsSteinGeneric(TestBlobFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'blob_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ 'blob_functions_stein' / 'generic' diff --git a/tests/endtoend/test_eventhub_batch_functions.py b/tests/emulator_tests/test_eventhub_batch_functions.py similarity index 98% rename from tests/endtoend/test_eventhub_batch_functions.py rename to tests/emulator_tests/test_eventhub_batch_functions.py index 5785de398..e8450ff00 100644 --- a/tests/endtoend/test_eventhub_batch_functions.py +++ b/tests/emulator_tests/test_eventhub_batch_functions.py @@ -28,7 +28,7 @@ class TestEventHubFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_batch_functions' @classmethod def get_libraries_to_install(cls): @@ -144,7 +144,7 @@ class TestEventHubBatchFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_batch_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_batch_functions' / \ 'eventhub_batch_functions_stein' @classmethod diff --git a/tests/endtoend/test_eventhub_functions.py b/tests/emulator_tests/test_eventhub_functions.py similarity index 95% rename from tests/endtoend/test_eventhub_functions.py rename to tests/emulator_tests/test_eventhub_functions.py index c4c3d74ab..22b8ec775 100644 --- a/tests/endtoend/test_eventhub_functions.py +++ b/tests/emulator_tests/test_eventhub_functions.py @@ -17,7 +17,7 @@ class TestEventHubFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' @classmethod def get_libraries_to_install(cls): @@ -106,7 +106,7 @@ class TestEventHubFunctionsStein(TestEventHubFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ 'eventhub_functions_stein' @@ -114,5 +114,5 @@ class TestEventHubFunctionsSteinGeneric(TestEventHubFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'eventhub_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ 'eventhub_functions_stein' / 'generic' diff --git a/tests/endtoend/test_generic_functions.py b/tests/emulator_tests/test_generic_functions.py similarity index 95% rename from tests/endtoend/test_generic_functions.py rename to tests/emulator_tests/test_generic_functions.py index 46f839377..aa54db91f 100644 --- a/tests/endtoend/test_generic_functions.py +++ b/tests/emulator_tests/test_generic_functions.py @@ -26,7 +26,7 @@ class TestGenericFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'generic_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'generic_functions' def test_return_processed_last(self): # Tests the case where implicit and explicit return are true @@ -81,5 +81,5 @@ class TestGenericFunctionsStein(TestGenericFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'generic_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'generic_functions' / \ 'generic_functions_stein' diff --git a/tests/endtoend/test_queue_functions.py b/tests/emulator_tests/test_queue_functions.py similarity index 94% rename from tests/endtoend/test_queue_functions.py rename to tests/emulator_tests/test_queue_functions.py index 015e53ae4..fa6872d30 100644 --- a/tests/endtoend/test_queue_functions.py +++ b/tests/emulator_tests/test_queue_functions.py @@ -9,7 +9,7 @@ class TestQueueFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' def test_queue_basic(self): r = self.webhost.request('POST', 'put_queue', @@ -91,7 +91,7 @@ class TestQueueFunctionsStein(TestQueueFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ 'queue_functions_stein' @@ -99,5 +99,5 @@ class TestQueueFunctionsSteinGeneric(TestQueueFunctions): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'queue_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ 'queue_functions_stein' / 'generic' diff --git a/tests/endtoend/test_table_functions.py b/tests/emulator_tests/test_table_functions.py similarity index 93% rename from tests/endtoend/test_table_functions.py rename to tests/emulator_tests/test_table_functions.py index c53d6e8d6..bd992a7e6 100644 --- a/tests/endtoend/test_table_functions.py +++ b/tests/emulator_tests/test_table_functions.py @@ -19,7 +19,7 @@ class TestTableFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' def test_table_bindings(self): out_resp = self.webhost.request('POST', 'table_out_binding') @@ -58,7 +58,7 @@ class TestTableFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ 'table_functions_stein' def test_table_bindings(self): @@ -84,6 +84,6 @@ class TestTableFunctionsGeneric(TestTableFunctionsStein): @classmethod def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'table_functions' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ 'table_functions_stein' /\ 'generic' From 9899abfbd1bc45e4ad5ee6d51f1a637cd3e0e101 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 19 Aug 2024 14:57:58 -0500 Subject: [PATCH 03/25] directory changes --- eng/ci/integration-tests.yml | 2 +- eng/ci/official-build.yml | 2 +- eng/ci/public-build.yml | 2 +- tests/utils/testutils.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/ci/integration-tests.yml b/eng/ci/integration-tests.yml index b4c1e4733..5da502c46 100644 --- a/eng/ci/integration-tests.yml +++ b/eng/ci/integration-tests.yml @@ -54,4 +54,4 @@ extends: - stage: RunEmulatorTests dependsOn: [] jobs: - - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self + - template: /eng/templates/jobs/ci-emulator-tests.yml@self diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 1b41610c1..b1ac0aee8 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -51,7 +51,7 @@ extends: - stage: RunEmulatorTests dependsOn: Build jobs: - - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self + - template: /eng/templates/jobs/ci-emulator-tests.yml@self - stage: RunUnitTests dependsOn: Build jobs: diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index 82251caf3..e35516daa 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -54,4 +54,4 @@ extends: - stage: RunEmulatorTests dependsOn: Build jobs: - - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self \ No newline at end of file + - template: /eng/templates/jobs/ci-emulator-tests.yml@self \ No newline at end of file diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index aaeb40dec..c04b134c5 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -68,6 +68,7 @@ E2E_TESTS_ROOT = TESTS_ROOT / E2E_TESTS_FOLDER UNIT_TESTS_FOLDER = pathlib.Path('unittests') UNIT_TESTS_ROOT = TESTS_ROOT / UNIT_TESTS_FOLDER +EMULATOR_TESTS_FOLDER = pathlib.Path('emulator_tests') EXTENSION_TESTS_FOLDER = pathlib.Path('extension_tests') WEBHOST_DLL = "Microsoft.Azure.WebJobs.Script.WebHost.dll" DEFAULT_WEBHOST_DLL_PATH = ( From e6226f706db64301ea4b9c7e04d4a7999480f75d Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 20 Aug 2024 13:41:01 -0500 Subject: [PATCH 04/25] fix: public build, generic table creation, eh config --- eng/templates/jobs/ci-emulator-tests.yml | 5 --- .../eventhub_batch_functions/Config.json | 24 +++++++++++++ .../Config.json | 24 +++++++++++++ .../eventhub_functions/Config.json | 24 +++++++++++++ .../eventhub_functions_stein/Config.json | 24 +++++++++++++ .../generic/Config.json | 24 +++++++++++++ .../generic_functions_stein/function_app.py | 35 +++++++++++++------ .../table_out_binding/__init__.py | 11 ++++++ .../table_out_binding/function.json | 24 +++++++++++++ .../emulator_tests/test_generic_functions.py | 6 ++++ 10 files changed, 186 insertions(+), 15 deletions(-) create mode 100644 tests/emulator_tests/eventhub_batch_functions/Config.json create mode 100644 tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/Config.json create mode 100644 tests/emulator_tests/eventhub_functions/Config.json create mode 100644 tests/emulator_tests/eventhub_functions/eventhub_functions_stein/Config.json create mode 100644 tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/Config.json create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/__init__.py create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/function.json diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index f78224bd0..83736eea6 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -2,11 +2,6 @@ jobs: - job: "TestPython" displayName: "Run Python Emulator Tests" - pool: - name: 1es-pool-azfunc - image: 1es-ubuntu-22.04 - os: linux - strategy: matrix: Python37: diff --git a/tests/emulator_tests/eventhub_batch_functions/Config.json b/tests/emulator_tests/eventhub_batch_functions/Config.json new file mode 100644 index 000000000..6f520c88f --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/Config.json @@ -0,0 +1,24 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/Config.json b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/Config.json new file mode 100644 index 000000000..6f520c88f --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/Config.json @@ -0,0 +1,24 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/Config.json b/tests/emulator_tests/eventhub_functions/Config.json new file mode 100644 index 000000000..6f520c88f --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/Config.json @@ -0,0 +1,24 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/Config.json b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/Config.json new file mode 100644 index 000000000..6f520c88f --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/Config.json @@ -0,0 +1,24 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/Config.json b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/Config.json new file mode 100644 index 000000000..6f520c88f --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/Config.json @@ -0,0 +1,24 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py index 654148c93..336bcdfdd 100644 --- a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py +++ b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import json import logging +import uuid import azure.functions as func @@ -15,7 +17,7 @@ arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_processed_last(req: func.HttpRequest, testEntity): return func.HttpResponse(status_code=200) @@ -28,7 +30,7 @@ def return_processed_last(req: func.HttpRequest, testEntity): arg_name="testEntities", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_not_processed_last(req: func.HttpRequest, testEntities): return func.HttpResponse(status_code=200) @@ -41,7 +43,7 @@ def return_not_processed_last(req: func.HttpRequest, testEntities): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def mytimer(mytimer: func.TimerRequest, testEntity) -> None: logging.info("This timer trigger function executed successfully") @@ -54,7 +56,7 @@ def mytimer(mytimer: func.TimerRequest, testEntity) -> None: arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_string(mytimer: func.TimerRequest, testEntity): logging.info("Return string") return "hi!" @@ -68,7 +70,7 @@ def return_string(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bytes(mytimer: func.TimerRequest, testEntity): logging.info("Return bytes") return "test-dată" @@ -82,7 +84,7 @@ def return_bytes(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_dict(mytimer: func.TimerRequest, testEntity): logging.info("Return dict") return {"hello": "world"} @@ -96,7 +98,7 @@ def return_dict(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_list(mytimer: func.TimerRequest, testEntity): logging.info("Return list") return [1, 2, 3] @@ -110,7 +112,7 @@ def return_list(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_int(mytimer: func.TimerRequest, testEntity): logging.info("Return int") return 12 @@ -124,7 +126,7 @@ def return_int(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_double(mytimer: func.TimerRequest, testEntity): logging.info("Return double") return 12.34 @@ -138,7 +140,20 @@ def return_double(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bool(mytimer: func.TimerRequest, testEntity): logging.info("Return bool") return True + + +@app.function_name(name="table_out_binding") +@app.route(route="table_out_binding", binding_arg_name="resp") +@app.table_output(arg_name="$return", + connection="AzureWebJobsStorage", + table_name="BindingTestTable") +def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json \ No newline at end of file diff --git a/tests/emulator_tests/generic_functions/table_out_binding/__init__.py b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py new file mode 100644 index 000000000..9e14750ae --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import uuid +import azure.functions as func +def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json diff --git a/tests/emulator_tests/generic_functions/table_out_binding/function.json b/tests/emulator_tests/generic_functions/table_out_binding/function.json new file mode 100644 index 000000000..25537873a --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/function.json @@ -0,0 +1,24 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": ["post"], + "name": "req" + }, + { + "direction": "out", + "type": "table", + "name": "$return", + "tableName": "BindingTestTable", + "connection": "AzureWebJobsStorage" + }, + { + "name": "resp", + "type": "http", + "direction": "out" + } + ] + } \ No newline at end of file diff --git a/tests/emulator_tests/test_generic_functions.py b/tests/emulator_tests/test_generic_functions.py index aa54db91f..172b6c21c 100644 --- a/tests/emulator_tests/test_generic_functions.py +++ b/tests/emulator_tests/test_generic_functions.py @@ -32,6 +32,8 @@ def test_return_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and $return is processed before # the generic binding is + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_processed_last') self.assertEqual(r.status_code, 200) @@ -40,11 +42,15 @@ def test_return_not_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and the generic binding is processed # before $return + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_not_processed_last') self.assertEqual(r.status_code, 200) def test_return_types(self): + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) # Checking that the function app is okay time.sleep(10) # Checking webhost status. From 4aa4547ed504ad49c795149814a2cde5a164e9b0 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 20 Aug 2024 14:42:17 -0500 Subject: [PATCH 05/25] docker-compose --- eng/ci/integration-tests.yml | 4 --- .../docker-compose.yml | 32 +++++++++++++++++++ .../docker-compose.yml | 32 +++++++++++++++++++ .../eventhub_functions/docker-compose.yml | 32 +++++++++++++++++++ .../docker-compose.yml | 32 +++++++++++++++++++ .../generic/docker-compose.yml | 32 +++++++++++++++++++ .../generic_functions_stein/function_app.py | 2 +- .../table_out_binding/__init__.py | 2 ++ tests/emulator_tests/test_blob_functions.py | 2 +- .../emulator_tests/test_eventhub_functions.py | 2 +- tests/emulator_tests/test_queue_functions.py | 2 +- tests/emulator_tests/test_table_functions.py | 6 ++-- .../test_mock_blob_shared_memory_functions.py | 2 +- 13 files changed, 170 insertions(+), 12 deletions(-) create mode 100644 tests/emulator_tests/eventhub_batch_functions/docker-compose.yml create mode 100644 tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/docker-compose.yml create mode 100644 tests/emulator_tests/eventhub_functions/docker-compose.yml create mode 100644 tests/emulator_tests/eventhub_functions/eventhub_functions_stein/docker-compose.yml create mode 100644 tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/docker-compose.yml diff --git a/eng/ci/integration-tests.yml b/eng/ci/integration-tests.yml index 5da502c46..6f8f69d9e 100644 --- a/eng/ci/integration-tests.yml +++ b/eng/ci/integration-tests.yml @@ -51,7 +51,3 @@ extends: dependsOn: [] jobs: - template: /eng/templates/official/jobs/ci-lc-tests.yml@self - - stage: RunEmulatorTests - dependsOn: [] - jobs: - - template: /eng/templates/jobs/ci-emulator-tests.yml@self diff --git a/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml b/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/docker-compose.yml b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/eventhub_batch_functions_stein/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/docker-compose.yml b/tests/emulator_tests/eventhub_functions/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/docker-compose.yml b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/docker-compose.yml b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_functions/eventhub_functions_stein/generic/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py index 336bcdfdd..2da6d44ca 100644 --- a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py +++ b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py @@ -156,4 +156,4 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} table_json = json.dumps(table_dict) resp.set(table_json) - return table_json \ No newline at end of file + return table_json diff --git a/tests/emulator_tests/generic_functions/table_out_binding/__init__.py b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py index 9e14750ae..09c7058e9 100644 --- a/tests/emulator_tests/generic_functions/table_out_binding/__init__.py +++ b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py @@ -3,6 +3,8 @@ import json import uuid import azure.functions as func + + def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): row_key_uuid = str(uuid.uuid4()) table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} diff --git a/tests/emulator_tests/test_blob_functions.py b/tests/emulator_tests/test_blob_functions.py index 8ec760b86..d6a840a38 100644 --- a/tests/emulator_tests/test_blob_functions.py +++ b/tests/emulator_tests/test_blob_functions.py @@ -155,7 +155,7 @@ class TestBlobFunctionsStein(TestBlobFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ - 'blob_functions_stein' + 'blob_functions_stein' class TestBlobFunctionsSteinGeneric(TestBlobFunctions): diff --git a/tests/emulator_tests/test_eventhub_functions.py b/tests/emulator_tests/test_eventhub_functions.py index 22b8ec775..32a3c9378 100644 --- a/tests/emulator_tests/test_eventhub_functions.py +++ b/tests/emulator_tests/test_eventhub_functions.py @@ -107,7 +107,7 @@ class TestEventHubFunctionsStein(TestEventHubFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ - 'eventhub_functions_stein' + 'eventhub_functions_stein' class TestEventHubFunctionsSteinGeneric(TestEventHubFunctions): diff --git a/tests/emulator_tests/test_queue_functions.py b/tests/emulator_tests/test_queue_functions.py index fa6872d30..793628169 100644 --- a/tests/emulator_tests/test_queue_functions.py +++ b/tests/emulator_tests/test_queue_functions.py @@ -92,7 +92,7 @@ class TestQueueFunctionsStein(TestQueueFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ - 'queue_functions_stein' + 'queue_functions_stein' class TestQueueFunctionsSteinGeneric(TestQueueFunctions): diff --git a/tests/emulator_tests/test_table_functions.py b/tests/emulator_tests/test_table_functions.py index bd992a7e6..cc6aa5419 100644 --- a/tests/emulator_tests/test_table_functions.py +++ b/tests/emulator_tests/test_table_functions.py @@ -59,7 +59,7 @@ class TestTableFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' + 'table_functions_stein' def test_table_bindings(self): out_resp = self.webhost.request('POST', 'table_out_binding') @@ -85,5 +85,5 @@ class TestTableFunctionsGeneric(TestTableFunctionsStein): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' /\ - 'generic' + 'table_functions_stein' / \ + 'generic' diff --git a/tests/unittests/test_mock_blob_shared_memory_functions.py b/tests/unittests/test_mock_blob_shared_memory_functions.py index cdbbfc920..6c78561f8 100644 --- a/tests/unittests/test_mock_blob_shared_memory_functions.py +++ b/tests/unittests/test_mock_blob_shared_memory_functions.py @@ -27,7 +27,7 @@ class TestMockBlobSharedMemoryFunctions(testutils.SharedMemoryTestCase, """ def setUp(self): super().setUp() - self.blob_funcs_dir = testutils.E2E_TESTS_FOLDER / 'blob_functions' + self.blob_funcs_dir = testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' async def test_binary_blob_read_as_bytes_function(self): """ From 5db919a72f6afd7bccfe25a2861f6890189a1469 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 26 Aug 2024 12:57:07 -0500 Subject: [PATCH 06/25] emulator support --- eng/ci/official-build.yml | 2 +- eng/templates/jobs/ci-emulator-tests.yml | 5 +- .../official/jobs/ci-emulator-tests.yml | 89 +++++++++++++++++++ .../eventhub_batch_functions/Config.json | 51 +++++++++++ .../docker-compose.yml | 32 +++++++ .../generic_functions_stein/function_app.py | 35 +++++--- .../return_not_processed_last/function.json | 2 +- .../return_processed_last/function.json | 2 +- .../table_out_binding/__init__.py | 13 +++ .../table_out_binding/function.json | 24 +++++ tests/emulator_tests/test_blob_functions.py | 2 +- .../emulator_tests/test_eventhub_functions.py | 2 +- .../emulator_tests/test_generic_functions.py | 6 ++ tests/emulator_tests/test_queue_functions.py | 2 +- tests/emulator_tests/test_table_functions.py | 6 +- 15 files changed, 252 insertions(+), 21 deletions(-) create mode 100644 eng/templates/official/jobs/ci-emulator-tests.yml create mode 100644 tests/emulator_tests/eventhub_batch_functions/Config.json create mode 100644 tests/emulator_tests/eventhub_batch_functions/docker-compose.yml create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/__init__.py create mode 100644 tests/emulator_tests/generic_functions/table_out_binding/function.json diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index b1ac0aee8..1b41610c1 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -51,7 +51,7 @@ extends: - stage: RunEmulatorTests dependsOn: Build jobs: - - template: /eng/templates/jobs/ci-emulator-tests.yml@self + - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self - stage: RunUnitTests dependsOn: Build jobs: diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index f78224bd0..4f3b80d2a 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -3,7 +3,7 @@ jobs: displayName: "Run Python Emulator Tests" pool: - name: 1es-pool-azfunc + name: 1es-pool-azfunc-public image: 1es-ubuntu-22.04 os: linux @@ -73,7 +73,8 @@ jobs: condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | yes | git clone https://github.com/Azure/azure-event-hubs-emulator-installer.git - yes | azure-event-hubs-emulator-installer/EventHub-Emulator/Scripts/Linux/LaunchEmulator.sh + docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull + docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d displayName: 'Install and Run Azure Event Hubs Emulator' - bash: | sudo npm install -g azurite diff --git a/eng/templates/official/jobs/ci-emulator-tests.yml b/eng/templates/official/jobs/ci-emulator-tests.yml new file mode 100644 index 000000000..2a83033be --- /dev/null +++ b/eng/templates/official/jobs/ci-emulator-tests.yml @@ -0,0 +1,89 @@ +jobs: + - job: "TestPython" + displayName: "Run Python Emulator Tests" + + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + strategy: + matrix: + Python37: + PYTHON_VERSION: '3.7' + Python38: + PYTHON_VERSION: '3.8' + Python39: + PYTHON_VERSION: '3.9' + Python310: + PYTHON_VERSION: '3.10' + Python311: + PYTHON_VERSION: '3.11' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + - task: UseDotNet@2 + displayName: 'Install .NET 8' + inputs: + version: 8.0.x + - bash: | + chmod +x eng/scripts/install-dependencies.sh + chmod +x eng/scripts/test-setup.sh + + eng/scripts/install-dependencies.sh $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install dependencies and the worker' + condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python SDK Artifact' + inputs: + buildType: specific + artifactName: 'azure-functions' + project: 'internal' + definition: 679 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonSdkArtifact' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - bash: | + chmod +x eng/scripts/test-sdk.sh + chmod +x eng/scripts/test-setup.sh + + eng/scripts/test-sdk.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python sdk, dependencies and the worker' + condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python Extension Artifact' + inputs: + buildType: specific + artifactName: $(PYTHONEXTENSIONNAME) + project: 'internal' + definition: 798 + buildVersionToDownload: latest + targetPath: '$(Pipeline.Workspace)/PythonExtensionArtifact' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + chmod +x eng/scripts/test-setup.sh + chmod +x eng/scripts/test-extensions.sh + + eng/scripts/test-extensions.sh $(Pipeline.Workspace) $(PYTHON_VERSION) + eng/scripts/test-setup.sh + displayName: 'Install test python extension, dependencies and the worker' + condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) + - bash: | + yes | git clone https://github.com/Azure/azure-event-hubs-emulator-installer.git + docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull + docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d + displayName: 'Install and Run Azure Event Hubs Emulator' + - bash: | + sudo npm install -g azurite + sudo mkdir azurite + sudo azurite --silent --location azurite --debug azurite\debug.log & + displayName: 'Install and Run Azurite' + - bash: | + python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests + env: + AzureWebJobsStorage: "UseDevelopmentStorage=true" + AzureWebJobsEventHubConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" + displayName: "Running $(PYTHON_VERSION) Python Emulator Tests" diff --git a/tests/emulator_tests/eventhub_batch_functions/Config.json b/tests/emulator_tests/eventhub_batch_functions/Config.json new file mode 100644 index 000000000..710935c14 --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/Config.json @@ -0,0 +1,51 @@ +{ + "UserConfig": { + "NamespaceConfig": [ + { + "Type": "EventHub", + "Name": "emulatorNs1", + "Entities": [ + { + "Name": "python-worker-ci-eventhub-batch", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-batch-metadata", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-one", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + }, + { + "Name": "python-worker-ci-eventhub-one-metadata", + "PartitionCount": 2, + "ConsumerGroups": [ + { + "Name": "cg1" + } + ] + } + ] + } + ], + "LoggingConfig": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml b/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml new file mode 100644 index 000000000..b52c73146 --- /dev/null +++ b/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml @@ -0,0 +1,32 @@ +name: microsoft-azure-eventhubs +services: + emulator: + container_name: "eventhubs-emulator" + image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" + volumes: + - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + BLOB_SERVER: azurite + METADATA_SERVER: azurite + ACCEPT_EULA: Y + depends_on: + - azurite + networks: + eh-emulator: + aliases: + - "eventhubs-emulator" + azurite: + container_name: "azurite" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + networks: + eh-emulator: + aliases: + - "azurite" +networks: + eh-emulator: \ No newline at end of file diff --git a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py index 654148c93..2da6d44ca 100644 --- a/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py +++ b/tests/emulator_tests/generic_functions/generic_functions_stein/function_app.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import json import logging +import uuid import azure.functions as func @@ -15,7 +17,7 @@ arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_processed_last(req: func.HttpRequest, testEntity): return func.HttpResponse(status_code=200) @@ -28,7 +30,7 @@ def return_processed_last(req: func.HttpRequest, testEntity): arg_name="testEntities", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_not_processed_last(req: func.HttpRequest, testEntities): return func.HttpResponse(status_code=200) @@ -41,7 +43,7 @@ def return_not_processed_last(req: func.HttpRequest, testEntities): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def mytimer(mytimer: func.TimerRequest, testEntity) -> None: logging.info("This timer trigger function executed successfully") @@ -54,7 +56,7 @@ def mytimer(mytimer: func.TimerRequest, testEntity) -> None: arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_string(mytimer: func.TimerRequest, testEntity): logging.info("Return string") return "hi!" @@ -68,7 +70,7 @@ def return_string(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bytes(mytimer: func.TimerRequest, testEntity): logging.info("Return bytes") return "test-dată" @@ -82,7 +84,7 @@ def return_bytes(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_dict(mytimer: func.TimerRequest, testEntity): logging.info("Return dict") return {"hello": "world"} @@ -96,7 +98,7 @@ def return_dict(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_list(mytimer: func.TimerRequest, testEntity): logging.info("Return list") return [1, 2, 3] @@ -110,7 +112,7 @@ def return_list(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_int(mytimer: func.TimerRequest, testEntity): logging.info("Return int") return 12 @@ -124,7 +126,7 @@ def return_int(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_double(mytimer: func.TimerRequest, testEntity): logging.info("Return double") return 12.34 @@ -138,7 +140,20 @@ def return_double(mytimer: func.TimerRequest, testEntity): arg_name="testEntity", type="table", connection="AzureWebJobsStorage", - table_name="EventHubBatchTest") + table_name="BindingTestTable") def return_bool(mytimer: func.TimerRequest, testEntity): logging.info("Return bool") return True + + +@app.function_name(name="table_out_binding") +@app.route(route="table_out_binding", binding_arg_name="resp") +@app.table_output(arg_name="$return", + connection="AzureWebJobsStorage", + table_name="BindingTestTable") +def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json diff --git a/tests/emulator_tests/generic_functions/return_not_processed_last/function.json b/tests/emulator_tests/generic_functions/return_not_processed_last/function.json index 66d1e80e1..e02ae4d15 100644 --- a/tests/emulator_tests/generic_functions/return_not_processed_last/function.json +++ b/tests/emulator_tests/generic_functions/return_not_processed_last/function.json @@ -14,7 +14,7 @@ "direction": "in", "type": "table", "name": "testEntities", - "tableName": "EventHubBatchTest", + "tableName": "BindingTestTable", "connection": "AzureWebJobsStorage" }, { diff --git a/tests/emulator_tests/generic_functions/return_processed_last/function.json b/tests/emulator_tests/generic_functions/return_processed_last/function.json index 82ac266a6..d23f01a86 100644 --- a/tests/emulator_tests/generic_functions/return_processed_last/function.json +++ b/tests/emulator_tests/generic_functions/return_processed_last/function.json @@ -14,7 +14,7 @@ "direction": "in", "type": "table", "name": "testEntity", - "tableName": "EventHubBatchTest", + "tableName": "BindingTestTable", "connection": "AzureWebJobsStorage" }, { diff --git a/tests/emulator_tests/generic_functions/table_out_binding/__init__.py b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py new file mode 100644 index 000000000..09c7058e9 --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import uuid +import azure.functions as func + + +def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]): + row_key_uuid = str(uuid.uuid4()) + table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid} + table_json = json.dumps(table_dict) + resp.set(table_json) + return table_json diff --git a/tests/emulator_tests/generic_functions/table_out_binding/function.json b/tests/emulator_tests/generic_functions/table_out_binding/function.json new file mode 100644 index 000000000..25537873a --- /dev/null +++ b/tests/emulator_tests/generic_functions/table_out_binding/function.json @@ -0,0 +1,24 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "authLevel": "anonymous", + "methods": ["post"], + "name": "req" + }, + { + "direction": "out", + "type": "table", + "name": "$return", + "tableName": "BindingTestTable", + "connection": "AzureWebJobsStorage" + }, + { + "name": "resp", + "type": "http", + "direction": "out" + } + ] + } \ No newline at end of file diff --git a/tests/emulator_tests/test_blob_functions.py b/tests/emulator_tests/test_blob_functions.py index 8ec760b86..d6a840a38 100644 --- a/tests/emulator_tests/test_blob_functions.py +++ b/tests/emulator_tests/test_blob_functions.py @@ -155,7 +155,7 @@ class TestBlobFunctionsStein(TestBlobFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'blob_functions' / \ - 'blob_functions_stein' + 'blob_functions_stein' class TestBlobFunctionsSteinGeneric(TestBlobFunctions): diff --git a/tests/emulator_tests/test_eventhub_functions.py b/tests/emulator_tests/test_eventhub_functions.py index 22b8ec775..32a3c9378 100644 --- a/tests/emulator_tests/test_eventhub_functions.py +++ b/tests/emulator_tests/test_eventhub_functions.py @@ -107,7 +107,7 @@ class TestEventHubFunctionsStein(TestEventHubFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'eventhub_functions' / \ - 'eventhub_functions_stein' + 'eventhub_functions_stein' class TestEventHubFunctionsSteinGeneric(TestEventHubFunctions): diff --git a/tests/emulator_tests/test_generic_functions.py b/tests/emulator_tests/test_generic_functions.py index aa54db91f..172b6c21c 100644 --- a/tests/emulator_tests/test_generic_functions.py +++ b/tests/emulator_tests/test_generic_functions.py @@ -32,6 +32,8 @@ def test_return_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and $return is processed before # the generic binding is + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_processed_last') self.assertEqual(r.status_code, 200) @@ -40,11 +42,15 @@ def test_return_not_processed_last(self): # Tests the case where implicit and explicit return are true # in the same function and the generic binding is processed # before $return + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) r = self.webhost.request('GET', 'return_not_processed_last') self.assertEqual(r.status_code, 200) def test_return_types(self): + out_resp = self.webhost.request('POST', 'table_out_binding') + self.assertEqual(out_resp.status_code, 200) # Checking that the function app is okay time.sleep(10) # Checking webhost status. diff --git a/tests/emulator_tests/test_queue_functions.py b/tests/emulator_tests/test_queue_functions.py index fa6872d30..793628169 100644 --- a/tests/emulator_tests/test_queue_functions.py +++ b/tests/emulator_tests/test_queue_functions.py @@ -92,7 +92,7 @@ class TestQueueFunctionsStein(TestQueueFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'queue_functions' / \ - 'queue_functions_stein' + 'queue_functions_stein' class TestQueueFunctionsSteinGeneric(TestQueueFunctions): diff --git a/tests/emulator_tests/test_table_functions.py b/tests/emulator_tests/test_table_functions.py index bd992a7e6..cc6aa5419 100644 --- a/tests/emulator_tests/test_table_functions.py +++ b/tests/emulator_tests/test_table_functions.py @@ -59,7 +59,7 @@ class TestTableFunctionsStein(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' + 'table_functions_stein' def test_table_bindings(self): out_resp = self.webhost.request('POST', 'table_out_binding') @@ -85,5 +85,5 @@ class TestTableFunctionsGeneric(TestTableFunctionsStein): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'table_functions' / \ - 'table_functions_stein' /\ - 'generic' + 'table_functions_stein' / \ + 'generic' From a4bdeb27fbb3643ccb23c04a9b14c4e1d3eb5641 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 28 Aug 2024 10:28:43 -0500 Subject: [PATCH 07/25] skip eventhub metadata tests for 37 --- tests/emulator_tests/test_eventhub_batch_functions.py | 7 +++++++ tests/emulator_tests/test_eventhub_functions.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tests/emulator_tests/test_eventhub_batch_functions.py b/tests/emulator_tests/test_eventhub_batch_functions.py index e8450ff00..23d129a89 100644 --- a/tests/emulator_tests/test_eventhub_batch_functions.py +++ b/tests/emulator_tests/test_eventhub_batch_functions.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import json +import sys import time from datetime import datetime from unittest import skipIf @@ -73,6 +74,9 @@ def test_eventhub_multiple(self): self.assertDictEqual(all_row_keys_seen, row_keys_seen) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_multiple_with_metadata(self): # Generate a unique event body for EventHub event @@ -185,6 +189,9 @@ def test_eventhub_multiple(self): self.assertDictEqual(all_row_keys_seen, row_keys_seen) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_multiple_with_metadata(self): # Generate a unique event body for EventHub event diff --git a/tests/emulator_tests/test_eventhub_functions.py b/tests/emulator_tests/test_eventhub_functions.py index 32a3c9378..03088c731 100644 --- a/tests/emulator_tests/test_eventhub_functions.py +++ b/tests/emulator_tests/test_eventhub_functions.py @@ -1,8 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import json +import sys import time +from unittest import skipIf + from tests.utils import testutils @@ -52,6 +55,9 @@ def test_eventhub_trigger(self): # Check if the event body matches the initial data self.assertEqual(response, doc) + @skipIf(sys.version_info.minor == 7, + "Using azure-eventhub SDK with the EventHub Emulator" + "requires Python 3.8+") @testutils.retryable_test(3, 5) def test_eventhub_trigger_with_metadata(self): # Generate a unique event body for EventHub event From 5d2c24a6b6ec8410406f2183c77a6505646cb470 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 30 Aug 2024 10:35:49 -0500 Subject: [PATCH 08/25] run db blob tests in emulator --- .../deferred_bindings_blob_functions/function_app.py | 0 .../test_deferred_bindings_blob_functions.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{extension_tests/deferred_bindings_tests => emulator_tests}/deferred_bindings_blob_functions/function_app.py (100%) rename tests/{extension_tests/deferred_bindings_tests => emulator_tests}/test_deferred_bindings_blob_functions.py (100%) diff --git a/tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py b/tests/emulator_tests/deferred_bindings_blob_functions/function_app.py similarity index 100% rename from tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py rename to tests/emulator_tests/deferred_bindings_blob_functions/function_app.py diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py b/tests/emulator_tests/test_deferred_bindings_blob_functions.py similarity index 100% rename from tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py rename to tests/emulator_tests/test_deferred_bindings_blob_functions.py From 90ebd53550c7d609c839327cce22d5e2e2322beb Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 30 Aug 2024 12:45:38 -0500 Subject: [PATCH 09/25] db blob tests dir change --- tests/emulator_tests/test_deferred_bindings_blob_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/emulator_tests/test_deferred_bindings_blob_functions.py b/tests/emulator_tests/test_deferred_bindings_blob_functions.py index ed441a077..7bba9d00a 100644 --- a/tests/emulator_tests/test_deferred_bindings_blob_functions.py +++ b/tests/emulator_tests/test_deferred_bindings_blob_functions.py @@ -13,7 +13,7 @@ class TestDeferredBindingsBlobFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.EXTENSION_TESTS_FOLDER / 'deferred_bindings_tests' / \ + return testutils.EMULATOR_TESTS_FOLDER / 'deferred_bindings_tests' / \ 'deferred_bindings_blob_functions' @classmethod From f82778c66cade25b3812b8f47250e2998ddf82fd Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 30 Aug 2024 15:22:15 -0500 Subject: [PATCH 10/25] fix db test directory --- tests/emulator_tests/test_deferred_bindings_blob_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/emulator_tests/test_deferred_bindings_blob_functions.py b/tests/emulator_tests/test_deferred_bindings_blob_functions.py index 7bba9d00a..476532af9 100644 --- a/tests/emulator_tests/test_deferred_bindings_blob_functions.py +++ b/tests/emulator_tests/test_deferred_bindings_blob_functions.py @@ -13,7 +13,7 @@ class TestDeferredBindingsBlobFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.EMULATOR_TESTS_FOLDER / 'deferred_bindings_tests' / \ + return testutils.EMULATOR_TESTS_FOLDER / \ 'deferred_bindings_blob_functions' @classmethod From 9775a97c798a2c980b70bc0de8bf7e28ed6b03c6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 3 Sep 2024 10:29:20 -0500 Subject: [PATCH 11/25] merge --- tests/emulator_tests/test_eventhub_batch_functions.py | 1 + .../deferred_bindings_blob_functions/function_app.py | 0 .../test_deferred_bindings_blob_functions.py | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) rename tests/{emulator_tests => extension_tests/deferred_bindings_tests}/deferred_bindings_blob_functions/function_app.py (100%) rename tests/{emulator_tests => extension_tests/deferred_bindings_tests}/test_deferred_bindings_blob_functions.py (99%) diff --git a/tests/emulator_tests/test_eventhub_batch_functions.py b/tests/emulator_tests/test_eventhub_batch_functions.py index 75ed64e04..1a8ae2a9e 100644 --- a/tests/emulator_tests/test_eventhub_batch_functions.py +++ b/tests/emulator_tests/test_eventhub_batch_functions.py @@ -4,6 +4,7 @@ import sys import time from datetime import datetime +from unittest.case import skipIf from dateutil import parser from tests.utils import testutils diff --git a/tests/emulator_tests/deferred_bindings_blob_functions/function_app.py b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py similarity index 100% rename from tests/emulator_tests/deferred_bindings_blob_functions/function_app.py rename to tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py diff --git a/tests/emulator_tests/test_deferred_bindings_blob_functions.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py similarity index 99% rename from tests/emulator_tests/test_deferred_bindings_blob_functions.py rename to tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py index 476532af9..ed441a077 100644 --- a/tests/emulator_tests/test_deferred_bindings_blob_functions.py +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py @@ -13,7 +13,7 @@ class TestDeferredBindingsBlobFunctions(testutils.WebHostTestCase): @classmethod def get_script_dir(cls): - return testutils.EMULATOR_TESTS_FOLDER / \ + return testutils.EXTENSION_TESTS_FOLDER / 'deferred_bindings_tests' / \ 'deferred_bindings_blob_functions' @classmethod From 9675607309df5ed7a56671c2b903d6edf36b2847 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 3 Sep 2024 12:06:36 -0500 Subject: [PATCH 12/25] run emulator tests for 312 --- eng/templates/jobs/ci-emulator-tests.yml | 2 ++ eng/templates/official/jobs/ci-emulator-tests.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index 4f3b80d2a..b525140e8 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -19,6 +19,8 @@ jobs: PYTHON_VERSION: '3.10' Python311: PYTHON_VERSION: '3.11' + Python312: + PYTHON_VERSION: '3.12' steps: - task: UsePythonVersion@0 inputs: diff --git a/eng/templates/official/jobs/ci-emulator-tests.yml b/eng/templates/official/jobs/ci-emulator-tests.yml index 2a83033be..e65d6aa55 100644 --- a/eng/templates/official/jobs/ci-emulator-tests.yml +++ b/eng/templates/official/jobs/ci-emulator-tests.yml @@ -19,6 +19,8 @@ jobs: PYTHON_VERSION: '3.10' Python311: PYTHON_VERSION: '3.11' + Python312: + PYTHON_VERSION: '3.12' steps: - task: UsePythonVersion@0 inputs: From cd2bc311c9aa2cf4f0fc8092e88d97be2f5b75b8 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 4 Sep 2024 09:43:28 -0500 Subject: [PATCH 13/25] remove git repo clone --- eng/templates/jobs/ci-emulator-tests.yml | 1 - eng/templates/official/jobs/ci-emulator-tests.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index b525140e8..4b5012542 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -74,7 +74,6 @@ jobs: displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | - yes | git clone https://github.com/Azure/azure-event-hubs-emulator-installer.git docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d displayName: 'Install and Run Azure Event Hubs Emulator' diff --git a/eng/templates/official/jobs/ci-emulator-tests.yml b/eng/templates/official/jobs/ci-emulator-tests.yml index e65d6aa55..93051f258 100644 --- a/eng/templates/official/jobs/ci-emulator-tests.yml +++ b/eng/templates/official/jobs/ci-emulator-tests.yml @@ -74,7 +74,6 @@ jobs: displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | - yes | git clone https://github.com/Azure/azure-event-hubs-emulator-installer.git docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d displayName: 'Install and Run Azure Event Hubs Emulator' From 984f819c1a951aeeeee40b39f65914f944bbe600 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 12 Sep 2024 11:03:35 -0500 Subject: [PATCH 14/25] refactor docker-compose, codeowners --- CODEOWNERS | 2 +- eng/templates/jobs/ci-emulator-tests.yml | 11 +++-------- eng/templates/official/jobs/ci-emulator-tests.yml | 11 +++-------- .../{eventhub_batch_functions => utils}/Config.json | 0 .../docker-compose.yml | 2 ++ 5 files changed, 9 insertions(+), 17 deletions(-) rename tests/emulator_tests/{eventhub_batch_functions => utils}/Config.json (100%) rename tests/emulator_tests/{eventhub_batch_functions => utils}/docker-compose.yml (90%) diff --git a/CODEOWNERS b/CODEOWNERS index f93501102..b99c3e6aa 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,4 +10,4 @@ # For all file changes, github would automatically # include the following people in the PRs. -* @vrdmr @gavin-aguiar @YunchuWang @pdthummar @hallvictoria +* @vrdmr @gavin-aguiar @YunchuWang @hallvictoria @ejizba diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index 4b5012542..f4ebd63fc 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -74,14 +74,9 @@ jobs: displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | - docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull - docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d - displayName: 'Install and Run Azure Event Hubs Emulator' - - bash: | - sudo npm install -g azurite - sudo mkdir azurite - sudo azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' + docker compose -f tests/emulator_tests/utils/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/docker-compose.yml up -d + displayName: 'Install Azurite and Start Emulators' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests env: diff --git a/eng/templates/official/jobs/ci-emulator-tests.yml b/eng/templates/official/jobs/ci-emulator-tests.yml index 93051f258..f9683af69 100644 --- a/eng/templates/official/jobs/ci-emulator-tests.yml +++ b/eng/templates/official/jobs/ci-emulator-tests.yml @@ -74,14 +74,9 @@ jobs: displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | - docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml pull - docker compose -f tests/emulator_tests/eventhub_batch_functions/docker-compose.yml up -d - displayName: 'Install and Run Azure Event Hubs Emulator' - - bash: | - sudo npm install -g azurite - sudo mkdir azurite - sudo azurite --silent --location azurite --debug azurite\debug.log & - displayName: 'Install and Run Azurite' + docker compose -f tests/emulator_tests/utils/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/docker-compose.yml up -d + displayName: 'Install Azurite and Start Emulators' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests env: diff --git a/tests/emulator_tests/eventhub_batch_functions/Config.json b/tests/emulator_tests/utils/Config.json similarity index 100% rename from tests/emulator_tests/eventhub_batch_functions/Config.json rename to tests/emulator_tests/utils/Config.json diff --git a/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml b/tests/emulator_tests/utils/docker-compose.yml similarity index 90% rename from tests/emulator_tests/eventhub_batch_functions/docker-compose.yml rename to tests/emulator_tests/utils/docker-compose.yml index b52c73146..71b4ba38f 100644 --- a/tests/emulator_tests/eventhub_batch_functions/docker-compose.yml +++ b/tests/emulator_tests/utils/docker-compose.yml @@ -1,5 +1,6 @@ name: microsoft-azure-eventhubs services: + # Service for the Event Hubs Emulator emulator: container_name: "eventhubs-emulator" image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" @@ -17,6 +18,7 @@ services: eh-emulator: aliases: - "eventhubs-emulator" + # Service for the Azurite Storage Emulator azurite: container_name: "azurite" image: "mcr.microsoft.com/azure-storage/azurite:latest" From 9cfb04f5d92ea731d42eb0cbde105348145ca0f6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 12 Sep 2024 16:35:38 -0500 Subject: [PATCH 15/25] retries for flaky tests --- tests/unittests/test_http_functions_v2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittests/test_http_functions_v2.py b/tests/unittests/test_http_functions_v2.py index 17af822c1..157fa0aab 100644 --- a/tests/unittests/test_http_functions_v2.py +++ b/tests/unittests/test_http_functions_v2.py @@ -204,6 +204,7 @@ def test_accept_json(self): self.assertEqual(r_json, {'a': 'abc', 'd': 42}) self.assertEqual(r.headers['content-type'], 'application/json') + @testutils.retryable_test(3, 5) def test_unhandled_error(self): r = self.webhost.request('GET', 'unhandled_error') self.assertEqual(r.status_code, 500) @@ -331,6 +332,7 @@ def check_log_import_module_troubleshooting_url(self, passed = True self.assertTrue(passed) + @testutils.retryable_test(3, 5) def test_print_logging_no_flush(self): r = self.webhost.request('GET', 'print_logging?message=Secret42') self.assertEqual(r.status_code, 200) From 68f1a922104d714b3c94a9c45c6fab5d014eb0ee Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 16 Oct 2024 10:41:40 -0500 Subject: [PATCH 16/25] update codeowners --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index b99c3e6aa..e5f28aee3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,4 +10,4 @@ # For all file changes, github would automatically # include the following people in the PRs. -* @vrdmr @gavin-aguiar @YunchuWang @hallvictoria @ejizba +* @vrdmr @gavin-aguiar @hallvictoria From f73ce518a65ef936a98946ac53939e2d969c5338 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 4 Nov 2024 13:59:22 -0600 Subject: [PATCH 17/25] run emulator tests on schedule, lowercase config --- eng/ci/emulator-tests.yml | 44 +++++++++++++++++++ .../utils/{Config.json => config.json} | 0 tests/emulator_tests/utils/docker-compose.yml | 2 +- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 eng/ci/emulator-tests.yml rename tests/emulator_tests/utils/{Config.json => config.json} (100%) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml new file mode 100644 index 000000000..21744c317 --- /dev/null +++ b/eng/ci/emulator-tests.yml @@ -0,0 +1,44 @@ +trigger: none # ensure this is not ran as a CI build + +pr: + branches: + include: + - dev + - release/* + +schedules: + - cron: "0 8 * * 1,2,3,4,5" + displayName: Monday to Friday 3 AM CST build + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: /ci/variables/build.yml@eng + - template: /ci/variables/cfs.yml@eng + - template: /eng/templates/utils/variables.yml@self + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: RunEmulatorTests + jobs: + - template: /eng/templates/jobs/ci-emulator-tests.yml@self \ No newline at end of file diff --git a/tests/emulator_tests/utils/Config.json b/tests/emulator_tests/utils/config.json similarity index 100% rename from tests/emulator_tests/utils/Config.json rename to tests/emulator_tests/utils/config.json diff --git a/tests/emulator_tests/utils/docker-compose.yml b/tests/emulator_tests/utils/docker-compose.yml index 71b4ba38f..2c40aa042 100644 --- a/tests/emulator_tests/utils/docker-compose.yml +++ b/tests/emulator_tests/utils/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: "eventhubs-emulator" image: "mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest" volumes: - - "./Config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" + - "./config.json:/Eventhubs_Emulator/ConfigFiles/Config.json" ports: - "5672:5672" environment: From f3117c2344ded16f7d9649ed6dadddb9551f1ec9 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 4 Dec 2024 14:37:50 -0600 Subject: [PATCH 18/25] merge miss --- .../generic/function_app.py | 282 +++++++++++------- 1 file changed, 180 insertions(+), 102 deletions(-) diff --git a/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py b/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py index f49b554e2..77e9dc596 100644 --- a/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py +++ b/tests/emulator_tests/blob_functions/blob_functions_stein/generic/function_app.py @@ -12,12 +12,16 @@ @app.function_name(name="blob_trigger") -@app.blob_trigger(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") +@app.generic_trigger( + arg_name="file", + type="blobTrigger", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.generic_output_binding( + arg_name="$return", + type="blob", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") def blob_trigger(file: func.InputStream) -> str: return json.dumps({ 'name': file.name, @@ -27,22 +31,30 @@ def blob_trigger(file: func.InputStream) -> str: @app.function_name(name="get_blob_as_bytes") -@app.route(route="get_blob_as_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_as_bytes") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/test-bytes.txt") def get_blob_as_bytes(req: func.HttpRequest, file: bytes) -> str: assert isinstance(file, bytes) return file.decode('utf-8') @app.function_name(name="get_blob_as_bytes_return_http_response") -@app.route(route="get_blob_as_bytes_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_as_bytes_return_http_response") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes.txt") def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ -> func.HttpResponse: """ @@ -69,11 +81,15 @@ def get_blob_as_bytes_return_http_response(req: func.HttpRequest, file: bytes) \ @app.function_name(name="get_blob_as_bytes_stream_return_http_response") -@app.route(route="get_blob_as_bytes_stream_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_as_bytes_stream_return_http_response") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes.txt") def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, file: func.InputStream) \ -> func.HttpResponse: @@ -101,22 +117,30 @@ def get_blob_as_bytes_stream_return_http_response(req: func.HttpRequest, @app.function_name(name="get_blob_as_str") -@app.route(route="get_blob_as_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - data_type="STRING", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_as_str") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + data_type="STRING", + path="python-worker-tests/test-str.txt") def get_blob_as_str(req: func.HttpRequest, file: str) -> str: assert isinstance(file, str) return file @app.function_name(name="get_blob_as_str_return_http_response") -@app.route(route="get_blob_as_str_return_http_response") -@app.blob_input(arg_name="file", - path="python-worker-tests/shmem-test-bytes.txt", - data_type="STRING", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_as_str_return_http_response") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + data_type="STRING", + path="python-worker-tests/shmem-test-bytes.txt") def get_blob_as_str_return_http_response(req: func.HttpRequest, file: str) -> func.HttpResponse: """ @@ -144,56 +168,79 @@ def get_blob_as_str_return_http_response(req: func.HttpRequest, @app.function_name(name="get_blob_bytes") -@app.route(route="get_blob_bytes") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_bytes") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + path="python-worker-tests/test-bytes.txt") def get_blob_bytes(req: func.HttpRequest, file: func.InputStream) -> str: return file.read().decode('utf-8') @app.function_name(name="get_blob_filelike") -@app.route(route="get_blob_filelike") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_filelike") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + path="python-worker-tests/test-filelike.txt") def get_blob_filelike(req: func.HttpRequest, file: func.InputStream) -> str: return file.read().decode('utf-8') @app.function_name(name="get_blob_return") -@app.route(route="get_blob_return") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_return") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + path="python-worker-tests/test-return.txt") def get_blob_return(req: func.HttpRequest, file: func.InputStream) -> str: return file.read().decode('utf-8') @app.function_name(name="get_blob_str") -@app.route(route="get_blob_str") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_str") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + path="python-worker-tests/test-str.txt") def get_blob_str(req: func.HttpRequest, file: func.InputStream) -> str: return file.read().decode('utf-8') @app.function_name(name="get_blob_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-blob-triggered.txt", - connection="AzureWebJobsStorage") -@app.route(route="get_blob_triggered") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_blob_triggered") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_input_binding( + arg_name="file", + connection="AzureWebJobsStorage", + type="blob", + path="python-worker-tests/test-blob-triggered.txt") def get_blob_triggered(req: func.HttpRequest, file: func.InputStream) -> str: return file.read().decode('utf-8') @app.function_name(name="put_blob_as_bytes_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-bytes-out.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_bytes_return_http_response") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_as_bytes_return_http_response") +@app.generic_output_binding( + arg_name="file", + type="blob", + data_type="BINARY", + connection="AzureWebJobsStorage", + path="python-worker-tests/shmem-test-bytes-out.txt") def put_blob_as_bytes_return_http_response(req: func.HttpRequest, file: func.Out[ bytes]) -> func.HttpResponse: @@ -231,13 +278,16 @@ def put_blob_as_bytes_return_http_response(req: func.HttpRequest, @app.function_name(name="put_blob_as_str_return_http_response") -@app.blob_output(arg_name="file", - path="python-worker-tests/shmem-test-str-out.txt", - data_type="STRING", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_as_str_return_http_response") -def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ - str]) -> func.HttpResponse: +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_as_str_return_http_response") +@app.generic_output_binding( + arg_name="file", + type="blob", + data_type="STRING", + connection="AzureWebJobsStorage", + path="python-worker-tests/shmem-test-str-out.txt") +def put_blob_as_str_return_http_response( + req: func.HttpRequest, file: func.Out[str]) -> func.HttpResponse: """ Write a blob (string) and respond back (in HTTP response) with the number of characters written and the MD5 digest of the utf-8 encoded content. @@ -269,20 +319,28 @@ def put_blob_as_str_return_http_response(req: func.HttpRequest, file: func.Out[ @app.function_name(name="put_blob_bytes") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-bytes.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_bytes") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_bytes") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_output_binding( + arg_name="file", + type="blob", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-bytes.txt") def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: file.set(req.get_body()) return 'OK' @app.function_name(name="put_blob_filelike") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-filelike.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_filelike") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_filelike") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_output_binding( + arg_name="file", + type="blob", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-filelike.txt") def put_blob_filelike(req: func.HttpRequest, file: func.Out[io.StringIO]) -> str: file.set(io.StringIO('filelike')) @@ -290,30 +348,40 @@ def put_blob_filelike(req: func.HttpRequest, @app.function_name(name="put_blob_return") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-return.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_return", binding_arg_name="resp") -def put_blob_return(req: func.HttpRequest, - resp: func.Out[func.HttpResponse]) -> str: +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_return") +@app.generic_output_binding( + arg_name="$return", + type="blob", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-return.txt") +def put_blob_return(req: func.HttpRequest) -> str: return 'FROM RETURN' @app.function_name(name="put_blob_str") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-str.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_str") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_str") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_output_binding( + arg_name="file", + type="blob", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-str.txt") def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: file.set(req.get_body()) return 'OK' @app.function_name(name="put_blob_trigger") -@app.blob_output(arg_name="file", - path="python-worker-tests/test-blob-trigger.txt", - connection="AzureWebJobsStorage") -@app.route(route="put_blob_trigger") +@app.generic_output_binding(arg_name="$return", type="http") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="put_blob_trigger") +@app.generic_output_binding( + arg_name="file", + type="blob", + connection="AzureWebJobsStorage", + path="python-worker-tests/test-blob-trigger.txt") def put_blob_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: file.set(req.get_body()) return 'OK' @@ -326,23 +394,33 @@ def _generate_content_and_digest(content_size): @app.function_name(name="put_get_multiple_blobs_as_bytes_return_http_response") -@app.blob_input(arg_name="inputfile1", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-1.txt", - connection="AzureWebJobsStorage") -@app.blob_input(arg_name="inputfile2", - data_type="BINARY", - path="python-worker-tests/shmem-test-bytes-2.txt", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile1", - path="python-worker-tests/shmem-test-bytes-out-1.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.blob_output(arg_name="outputfile2", - path="python-worker-tests/shmem-test-bytes-out-2.txt", - data_type="BINARY", - connection="AzureWebJobsStorage") -@app.route(route="put_get_multiple_blobs_as_bytes_return_http_response") +@app.generic_trigger( + arg_name="req", type="httpTrigger", + route="put_get_multiple_blobs_as_bytes_return_http_response") +@app.generic_input_binding( + arg_name="inputfile1", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-1.txt") +@app.generic_input_binding( + arg_name="inputfile2", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-2.txt") +@app.generic_output_binding( + arg_name="outputfile1", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-out-1.txt") +@app.generic_output_binding( + arg_name="outputfile2", + connection="AzureWebJobsStorage", + type="blob", + data_type="BINARY", + path="python-worker-tests/shmem-test-bytes-out-2.txt") def put_get_multiple_blobs_as_bytes_return_http_response( req: func.HttpRequest, inputfile1: bytes, From 5f6e2a9e5e506417cd8b18be378b5abfbb9ddcdb Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Dec 2024 10:28:07 -0600 Subject: [PATCH 19/25] move servicebus tests to emulators --- eng/ci/official-build.yml | 4 +- eng/ci/public-build.yml | 4 +- eng/templates/jobs/ci-emulator-tests.yml | 25 ++++-- .../official/jobs/ci-emulator-tests.yml | 85 ------------------- python/test/worker.config.json | 2 +- .../get_servicebus_triggered/__init__.py | 8 ++ .../get_servicebus_triggered/function.json | 23 +++++ .../put_message/__init__.py | 9 ++ .../put_message/function.json | 24 ++++++ .../function_app.py | 73 ++++++++++++++++ .../generic/function_app.py | 81 ++++++++++++++++++ .../servicebus_trigger/__init__.py | 46 ++++++++++ .../servicebus_trigger/function.json | 20 +++++ .../test_servicebus_functions.py | 66 ++++++++++++++ .../utils/{ => eventhub}/config.json | 0 .../utils/{ => eventhub}/docker-compose.yml | 0 .../utils/servicebus/config.json | 28 ++++++ .../utils/servicebus/docker-compose.yml | 40 +++++++++ .../test_worker_process_count_functions.py | 19 ++++- .../http_v2_tests/test_http_v2.py | 5 ++ 20 files changed, 465 insertions(+), 97 deletions(-) delete mode 100644 eng/templates/official/jobs/ci-emulator-tests.yml create mode 100644 tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py create mode 100644 tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json create mode 100644 tests/emulator_tests/servicebus_functions/put_message/__init__.py create mode 100644 tests/emulator_tests/servicebus_functions/put_message/function.json create mode 100644 tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py create mode 100644 tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py create mode 100644 tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py create mode 100644 tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json create mode 100644 tests/emulator_tests/test_servicebus_functions.py rename tests/emulator_tests/utils/{ => eventhub}/config.json (100%) rename tests/emulator_tests/utils/{ => eventhub}/docker-compose.yml (100%) create mode 100644 tests/emulator_tests/utils/servicebus/config.json create mode 100644 tests/emulator_tests/utils/servicebus/docker-compose.yml diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 361e3bf7d..568fdf16b 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -57,7 +57,9 @@ extends: - stage: RunEmulatorTests dependsOn: Build jobs: - - template: /eng/templates/official/jobs/ci-emulator-tests.yml@self + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc - stage: RunUnitTests dependsOn: Build jobs: diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index fd6b5800b..26f1b6625 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -57,4 +57,6 @@ extends: - stage: RunEmulatorTests dependsOn: Build jobs: - - template: /eng/templates/jobs/ci-emulator-tests.yml@self \ No newline at end of file + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc-public \ No newline at end of file diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index f4ebd63fc..498eb1ea0 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -3,7 +3,7 @@ jobs: displayName: "Run Python Emulator Tests" pool: - name: 1es-pool-azfunc-public + name: ${{ parameters.PoolName }} image: 1es-ubuntu-22.04 os: linux @@ -74,12 +74,25 @@ jobs: displayName: 'Install test python extension, dependencies and the worker' condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - bash: | - docker compose -f tests/emulator_tests/utils/docker-compose.yml pull - docker compose -f tests/emulator_tests/utils/docker-compose.yml up -d - displayName: 'Install Azurite and Start Emulators' + docker compose -f tests/emulator_tests/utils/eventhub/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/eventhub/docker-compose.yml up -d + displayName: 'Install Azurite and Start EventHub Emulator' - bash: | - python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests + python -m pytest -q -n auto --dist loadfile --reruns 4 --ignore=tests/emulator_tests/test_servicebus_functions.py tests/emulator_tests env: AzureWebJobsStorage: "UseDevelopmentStorage=true" AzureWebJobsEventHubConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" - displayName: "Running $(PYTHON_VERSION) Python Emulator Tests" + displayName: "Running $(PYTHON_VERSION) Python Linux Emulator Tests" + - bash: | + # Stop and remove EventHub Emulator container to free up the port + docker stop microsoft-azure-eventhub-emulator + docker container rm --force microsoft-azure-eventhub-emulator + docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml pull + docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml up -d + displayName: 'Install Azurite and Start ServiceBus Emulator' + - bash: | + python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests/test_servicebus_functions.py + env: + AzureWebJobsStorage: "UseDevelopmentStorage=true" + AzureWebJobsServiceBusConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" + displayName: "Running $(PYTHON_VERSION) Python ServiceBus Linux Emulator Tests" diff --git a/eng/templates/official/jobs/ci-emulator-tests.yml b/eng/templates/official/jobs/ci-emulator-tests.yml deleted file mode 100644 index f9683af69..000000000 --- a/eng/templates/official/jobs/ci-emulator-tests.yml +++ /dev/null @@ -1,85 +0,0 @@ -jobs: - - job: "TestPython" - displayName: "Run Python Emulator Tests" - - pool: - name: 1es-pool-azfunc - image: 1es-ubuntu-22.04 - os: linux - - strategy: - matrix: - Python37: - PYTHON_VERSION: '3.7' - Python38: - PYTHON_VERSION: '3.8' - Python39: - PYTHON_VERSION: '3.9' - Python310: - PYTHON_VERSION: '3.10' - Python311: - PYTHON_VERSION: '3.11' - Python312: - PYTHON_VERSION: '3.12' - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - - task: UseDotNet@2 - displayName: 'Install .NET 8' - inputs: - version: 8.0.x - - bash: | - chmod +x eng/scripts/install-dependencies.sh - chmod +x eng/scripts/test-setup.sh - - eng/scripts/install-dependencies.sh $(PYTHON_VERSION) - eng/scripts/test-setup.sh - displayName: 'Install dependencies and the worker' - condition: and(eq(variables.isSdkRelease, false), eq(variables.isExtensionsRelease, false), eq(variables['USETESTPYTHONSDK'], false), eq(variables['USETESTPYTHONEXTENSIONS'], false)) - - task: DownloadPipelineArtifact@2 - displayName: 'Download Python SDK Artifact' - inputs: - buildType: specific - artifactName: 'azure-functions' - project: 'internal' - definition: 679 - buildVersionToDownload: latest - targetPath: '$(Pipeline.Workspace)/PythonSdkArtifact' - condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - - bash: | - chmod +x eng/scripts/test-sdk.sh - chmod +x eng/scripts/test-setup.sh - - eng/scripts/test-sdk.sh $(Pipeline.Workspace) $(PYTHON_VERSION) - eng/scripts/test-setup.sh - displayName: 'Install test python sdk, dependencies and the worker' - condition: or(eq(variables.isSdkRelease, true), eq(variables['USETESTPYTHONSDK'], true)) - - task: DownloadPipelineArtifact@2 - displayName: 'Download Python Extension Artifact' - inputs: - buildType: specific - artifactName: $(PYTHONEXTENSIONNAME) - project: 'internal' - definition: 798 - buildVersionToDownload: latest - targetPath: '$(Pipeline.Workspace)/PythonExtensionArtifact' - condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - - bash: | - chmod +x eng/scripts/test-setup.sh - chmod +x eng/scripts/test-extensions.sh - - eng/scripts/test-extensions.sh $(Pipeline.Workspace) $(PYTHON_VERSION) - eng/scripts/test-setup.sh - displayName: 'Install test python extension, dependencies and the worker' - condition: or(eq(variables.isExtensionsRelease, true), eq(variables['USETESTPYTHONEXTENSIONS'], true)) - - bash: | - docker compose -f tests/emulator_tests/utils/docker-compose.yml pull - docker compose -f tests/emulator_tests/utils/docker-compose.yml up -d - displayName: 'Install Azurite and Start Emulators' - - bash: | - python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests - env: - AzureWebJobsStorage: "UseDevelopmentStorage=true" - AzureWebJobsEventHubConnectionString: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" - displayName: "Running $(PYTHON_VERSION) Python Emulator Tests" diff --git a/python/test/worker.config.json b/python/test/worker.config.json index dbb1480b7..c8f9bf626 100644 --- a/python/test/worker.config.json +++ b/python/test/worker.config.json @@ -2,7 +2,7 @@ "description":{ "language":"python", "extensions":[".py"], - "defaultExecutablePath":"python", + "defaultExecutablePath":"C:\\Users\\victoriahall\\Documents\\repos\\azure-functions-python-worker\\.venv\\Scripts\\python.exe", "defaultWorkerPath":"worker.py", "workerIndexing": "true" } diff --git a/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py new file mode 100644 index 000000000..1e5ac3d90 --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import azure.functions as func + + +def main(req: func.HttpRequest, file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') diff --git a/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json new file mode 100644 index 000000000..944dd45a9 --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/get_servicebus_triggered/function.json @@ -0,0 +1,23 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "name": "req", + "authLevel": "anonymous" + }, + { + "type": "blob", + "direction": "in", + "name": "file", + "connection": "AzureWebJobsStorage", + "path": "python-worker-tests/test-servicebus-triggered.txt" + }, + { + "type": "http", + "direction": "out", + "name": "$return", + } + ] +} diff --git a/tests/emulator_tests/servicebus_functions/put_message/__init__.py b/tests/emulator_tests/servicebus_functions/put_message/__init__.py new file mode 100644 index 000000000..85ad99bf7 --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/put_message/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import azure.functions as azf + + +def main(req: azf.HttpRequest, msg: azf.Out[str]): + msg.set(req.get_body().decode('utf-8')) + + return 'OK' diff --git a/tests/emulator_tests/servicebus_functions/put_message/function.json b/tests/emulator_tests/servicebus_functions/put_message/function.json new file mode 100644 index 000000000..722f19541 --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/put_message/function.json @@ -0,0 +1,24 @@ +{ + "scriptFile": "__init__.py", + + "bindings": [ + { + "type": "httpTrigger", + "direction": "in", + "name": "req", + "authLevel": "anonymous" + }, + { + "direction": "out", + "name": "msg", + "queueName": "testqueue", + "connection": "AzureWebJobsServiceBusConnectionString", + "type": "serviceBus" + }, + { + "direction": "out", + "name": "$return", + "type": "http" + } + ] +} diff --git a/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py new file mode 100644 index 000000000..9e9d12246 --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/function_app.py @@ -0,0 +1,73 @@ +import json + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.route(route="put_message") +@app.service_bus_queue_output( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +def put_message(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body().decode('utf-8')) + return 'OK' + + +@app.route(route="get_servicebus_triggered") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-servicebus-triggered.txt", + connection="AzureWebJobsStorage") +def get_servicebus_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') + + +@app.service_bus_queue_trigger( + arg_name="msg", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-servicebus-triggered.txt", + connection="AzureWebJobsStorage") +def servicebus_trigger(msg: func.ServiceBusMessage) -> str: + result = json.dumps({ + 'message_id': msg.message_id, + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + + 'application_properties': msg.application_properties, + 'correlation_id': msg.correlation_id, + 'dead_letter_error_description': msg.dead_letter_error_description, + 'dead_letter_reason': msg.dead_letter_reason, + 'dead_letter_source': msg.dead_letter_source, + 'enqueued_sequence_number': msg.enqueued_sequence_number, + 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if + msg.enqueued_time_utc else None), + 'expires_at_utc': (msg.expires_at_utc.isoformat() if + msg.expires_at_utc else None), + 'locked_until': (msg.locked_until.isoformat() if + msg.locked_until else None), + 'lock_token': msg.lock_token, + 'sequence_number': msg.sequence_number, + 'state': msg.state, + 'subject': msg.subject, + 'transaction_partition_key': msg.transaction_partition_key + }) + + return result diff --git a/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py new file mode 100644 index 000000000..4fd48785a --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/servicebus_functions_stein/generic/function_app.py @@ -0,0 +1,81 @@ +import json + +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="put_message") +@app.generic_trigger(arg_name="req", type="httpTrigger", route="put_message") +@app.generic_output_binding(arg_name="msg", + type="serviceBus", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.generic_output_binding(arg_name="$return", type="http") +def put_message(req: func.HttpRequest, msg: func.Out[str]): + msg.set(req.get_body().decode('utf-8')) + return 'OK' + + +@app.function_name(name="get_servicebus_triggered") +@app.generic_trigger(arg_name="req", type="httpTrigger", + route="get_servicebus_triggered") +@app.generic_input_binding(arg_name="file", + type="blob", + path="python-worker-tests/test-servicebus-triggered.txt", # NoQA + connection="AzureWebJobsStorage") +@app.generic_output_binding(arg_name="$return", type="http") +def get_servicebus_triggered(req: func.HttpRequest, + file: func.InputStream) -> str: + return func.HttpResponse( + file.read().decode('utf-8'), mimetype='application/json') + + +@app.generic_trigger( + arg_name="msg", + type="serviceBusTrigger", + connection="AzureWebJobsServiceBusConnectionString", + queue_name="testqueue") +@app.generic_output_binding(arg_name="$return", + path="python-worker-tests/test-servicebus-triggered.txt", # NoQA + type="blob", + connection="AzureWebJobsStorage") +def servicebus_trigger(msg: func.ServiceBusMessage) -> str: + result = json.dumps({ + 'message_id': msg.message_id, + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + + 'application_properties': msg.application_properties, + 'correlation_id': msg.correlation_id, + 'dead_letter_error_description': msg.dead_letter_error_description, + 'dead_letter_reason': msg.dead_letter_reason, + 'dead_letter_source': msg.dead_letter_source, + 'enqueued_sequence_number': msg.enqueued_sequence_number, + 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if + msg.enqueued_time_utc else None), + 'expires_at_utc': (msg.expires_at_utc.isoformat() if + msg.expires_at_utc else None), + 'locked_until': (msg.locked_until.isoformat() if + msg.locked_until else None), + 'lock_token': msg.lock_token, + 'sequence_number': msg.sequence_number, + 'state': msg.state, + 'subject': msg.subject, + 'transaction_partition_key': msg.transaction_partition_key + }) + + return result diff --git a/tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py b/tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py new file mode 100644 index 000000000..341779a4d --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/servicebus_trigger/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json + +import azure.functions as azf + + +def main(msg: azf.ServiceBusMessage) -> str: + result = json.dumps({ + 'message_id': msg.message_id, + 'body': msg.get_body().decode('utf-8'), + 'content_type': msg.content_type, + 'delivery_count': msg.delivery_count, + 'expiration_time': (msg.expiration_time.isoformat() if + msg.expiration_time else None), + 'label': msg.label, + 'partition_key': msg.partition_key, + 'reply_to': msg.reply_to, + 'reply_to_session_id': msg.reply_to_session_id, + 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if + msg.scheduled_enqueue_time else None), + 'session_id': msg.session_id, + 'time_to_live': msg.time_to_live, + 'to': msg.to, + 'user_properties': msg.user_properties, + + 'application_properties': msg.application_properties, + 'correlation_id': msg.correlation_id, + 'dead_letter_error_description': msg.dead_letter_error_description, + 'dead_letter_reason': msg.dead_letter_reason, + 'dead_letter_source': msg.dead_letter_source, + 'enqueued_sequence_number': msg.enqueued_sequence_number, + 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if + msg.enqueued_time_utc else None), + 'expires_at_utc': (msg.expires_at_utc.isoformat() if + msg.expires_at_utc else None), + 'locked_until': (msg.locked_until.isoformat() if + msg.locked_until else None), + 'lock_token': msg.lock_token, + 'sequence_number': msg.sequence_number, + 'state': msg.state, + 'subject': msg.subject, + 'transaction_partition_key': msg.transaction_partition_key + }) + + return result diff --git a/tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json b/tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json new file mode 100644 index 000000000..b6fe4355e --- /dev/null +++ b/tests/emulator_tests/servicebus_functions/servicebus_trigger/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + + "bindings": [ + { + "type": "serviceBusTrigger", + "direction": "in", + "name": "msg", + "queueName": "testqueue", + "connection": "AzureWebJobsServiceBusConnectionString", + }, + { + "type": "blob", + "direction": "out", + "name": "$return", + "connection": "AzureWebJobsStorage", + "path": "python-worker-tests/test-servicebus-triggered.txt" + } + ] +} diff --git a/tests/emulator_tests/test_servicebus_functions.py b/tests/emulator_tests/test_servicebus_functions.py new file mode 100644 index 000000000..ce15c0951 --- /dev/null +++ b/tests/emulator_tests/test_servicebus_functions.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import time + +from tests.utils import testutils + + +class TestServiceBusFunctions(testutils.WebHostTestCase): + + @classmethod + def get_script_dir(cls): + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' + + @testutils.retryable_test(3, 5) + def test_servicebus_basic(self): + data = str(round(time.time())) + r = self.webhost.request('POST', 'put_message', + data=data) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + max_retries = 10 + + for try_no in range(max_retries): + # wait for trigger to process the queue item + time.sleep(1) + + try: + r = self.webhost.request('GET', 'get_servicebus_triggered') + self.assertEqual(r.status_code, 200) + msg = r.json() + self.assertEqual(msg['body'], data) + for attr in {'message_id', 'body', 'content_type', 'delivery_count', + 'expiration_time', 'label', 'partition_key', 'reply_to', + 'reply_to_session_id', 'scheduled_enqueue_time', + 'session_id', 'time_to_live', 'to', 'user_properties', + 'application_properties', 'correlation_id', + 'dead_letter_error_description', 'dead_letter_reason', + 'dead_letter_source', 'enqueued_sequence_number', + 'enqueued_time_utc', 'expires_at_utc', 'locked_until', + 'lock_token', 'sequence_number', 'state', 'subject', + 'transaction_partition_key'}: + self.assertIn(attr, msg) + except (AssertionError, json.JSONDecodeError): + if try_no == max_retries - 1: + raise + else: + break + + +class TestServiceBusFunctionsStein(TestServiceBusFunctions): + + @classmethod + def get_script_dir(cls): + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ + 'servicebus_functions_stein' + + +class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): + + @classmethod + def get_script_dir(cls): + return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ + 'servicebus_functions_stein' / \ + 'generic' diff --git a/tests/emulator_tests/utils/config.json b/tests/emulator_tests/utils/eventhub/config.json similarity index 100% rename from tests/emulator_tests/utils/config.json rename to tests/emulator_tests/utils/eventhub/config.json diff --git a/tests/emulator_tests/utils/docker-compose.yml b/tests/emulator_tests/utils/eventhub/docker-compose.yml similarity index 100% rename from tests/emulator_tests/utils/docker-compose.yml rename to tests/emulator_tests/utils/eventhub/docker-compose.yml diff --git a/tests/emulator_tests/utils/servicebus/config.json b/tests/emulator_tests/utils/servicebus/config.json new file mode 100644 index 000000000..20cf83447 --- /dev/null +++ b/tests/emulator_tests/utils/servicebus/config.json @@ -0,0 +1,28 @@ +{ + "UserConfig": { + "Namespaces": [ + { + "Name": "sbemulatorns", + "Queues": [ + { + "Name": "testqueue", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + } + ] + } + ], + "Logging": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/tests/emulator_tests/utils/servicebus/docker-compose.yml b/tests/emulator_tests/utils/servicebus/docker-compose.yml new file mode 100644 index 000000000..9c6d92216 --- /dev/null +++ b/tests/emulator_tests/utils/servicebus/docker-compose.yml @@ -0,0 +1,40 @@ +name: microsoft-azure-servicebus +services: + # Service for the Service Bus Emulator + sbemulator: + container_name: "servicebus-emulator" + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest + volumes: + - "./config.json:/ServiceBus_Emulator/ConfigFiles/Config.json" + ports: + - "5672:5672" + environment: + SQL_SERVER: sqledge + MSSQL_SA_PASSWORD: Password123! + ACCEPT_EULA: Y + depends_on: + - sqledge + networks: + sb-emulator: + aliases: + - "sb-emulator" + sqledge: + container_name: "sqledge" + image: "mcr.microsoft.com/azure-sql-edge:latest" + networks: + sb-emulator: + aliases: + - "sqledge" + environment: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: Password123! + # Service for the Azurite Storage Emulator + azurite: + container_name: "azurite-sb" + image: "mcr.microsoft.com/azure-storage/azurite:latest" + ports: + - "10003:10003" + - "10004:10004" + - "10005:10005" +networks: + sb-emulator: \ No newline at end of file diff --git a/tests/endtoend/test_worker_process_count_functions.py b/tests/endtoend/test_worker_process_count_functions.py index 8ee6577bc..be61c43ea 100644 --- a/tests/endtoend/test_worker_process_count_functions.py +++ b/tests/endtoend/test_worker_process_count_functions.py @@ -1,12 +1,19 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os +import unittest + from datetime import datetime from threading import Thread from tests.utils import testutils +from azure_functions_worker.utils.common import is_envvar_true +from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCount(testutils.WebHostTestCase): """Test the Http Trigger with setting up the python worker process count to 2. this test will check if both requests should be processed at the @@ -63,24 +70,30 @@ def http_req(res_num): self.assertTrue(time_diff_in_seconds < 1) +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'http_functions' /\ 'http_functions_stein' +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' /\ 'functions_in_blueprint_only' +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintDiffDirStein(TestWorkerProcessCount): - @classmethod def get_script_dir(cls): return testutils.E2E_TESTS_FOLDER / 'blueprint_functions' /\ diff --git a/tests/extension_tests/http_v2_tests/test_http_v2.py b/tests/extension_tests/http_v2_tests/test_http_v2.py index 8409b7e58..670b5b54a 100644 --- a/tests/extension_tests/http_v2_tests/test_http_v2.py +++ b/tests/extension_tests/http_v2_tests/test_http_v2.py @@ -8,12 +8,17 @@ import requests from tests.utils import testutils +from azure_functions_worker.utils.common import is_envvar_true +from tests.utils.constants import CONSUMPTION_DOCKER_TEST,DEDICATED_DOCKER_TEST from azure_functions_worker.constants import PYTHON_ENABLE_INIT_INDEXING REQUEST_TIMEOUT_SEC = 5 +@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") @unittest.skipIf(sys.version_info.minor < 8, "HTTPv2" "is only supported for 3.8+.") class TestHttpFunctionsWithInitIndexing(testutils.WebHostTestCase): From 7e3b79ca3d1476661a51adebff92520aba48cb35 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Dec 2024 10:29:49 -0600 Subject: [PATCH 20/25] parameter miss --- eng/ci/emulator-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml index 21744c317..129aa146c 100644 --- a/eng/ci/emulator-tests.yml +++ b/eng/ci/emulator-tests.yml @@ -41,4 +41,6 @@ extends: stages: - stage: RunEmulatorTests jobs: - - template: /eng/templates/jobs/ci-emulator-tests.yml@self \ No newline at end of file + - template: /eng/templates/jobs/ci-emulator-tests.yml@self + parameters: + PoolName: 1es-pool-azfunc \ No newline at end of file From b262797dc23df8465787bea549e502ecb9254472 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Dec 2024 10:30:35 -0600 Subject: [PATCH 21/25] formatting --- eng/ci/emulator-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml index 129aa146c..adb1016f3 100644 --- a/eng/ci/emulator-tests.yml +++ b/eng/ci/emulator-tests.yml @@ -43,4 +43,4 @@ extends: jobs: - template: /eng/templates/jobs/ci-emulator-tests.yml@self parameters: - PoolName: 1es-pool-azfunc \ No newline at end of file + PoolName: 1es-pool-azfunc \ No newline at end of file From 23ea45c69de76e4d4df81f9c7eae9638997ef0d6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Dec 2024 10:33:08 -0600 Subject: [PATCH 22/25] merge miss --- python/test/worker.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/test/worker.config.json b/python/test/worker.config.json index c8f9bf626..dbb1480b7 100644 --- a/python/test/worker.config.json +++ b/python/test/worker.config.json @@ -2,7 +2,7 @@ "description":{ "language":"python", "extensions":[".py"], - "defaultExecutablePath":"C:\\Users\\victoriahall\\Documents\\repos\\azure-functions-python-worker\\.venv\\Scripts\\python.exe", + "defaultExecutablePath":"python", "defaultWorkerPath":"worker.py", "workerIndexing": "true" } From 584e8e4dfbc863c871c40e6e9ad331cd65a478f1 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Dec 2024 10:41:20 -0600 Subject: [PATCH 23/25] lint --- .../emulator_tests/test_servicebus_functions.py | 5 ++--- .../test_worker_process_count_functions.py | 16 ++++++++-------- .../http_v2_tests/test_http_v2.py | 6 +++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/emulator_tests/test_servicebus_functions.py b/tests/emulator_tests/test_servicebus_functions.py index ce15c0951..2e6bd7310 100644 --- a/tests/emulator_tests/test_servicebus_functions.py +++ b/tests/emulator_tests/test_servicebus_functions.py @@ -54,7 +54,7 @@ class TestServiceBusFunctionsStein(TestServiceBusFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' + 'servicebus_functions_stein' class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): @@ -62,5 +62,4 @@ class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): @classmethod def get_script_dir(cls): return testutils.EMULATOR_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' / \ - 'generic' + 'servicebus_functions_stein' / 'generic' diff --git a/tests/endtoend/test_worker_process_count_functions.py b/tests/endtoend/test_worker_process_count_functions.py index be61c43ea..b1277edec 100644 --- a/tests/endtoend/test_worker_process_count_functions.py +++ b/tests/endtoend/test_worker_process_count_functions.py @@ -12,8 +12,8 @@ @unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCount(testutils.WebHostTestCase): """Test the Http Trigger with setting up the python worker process count to 2. this test will check if both requests should be processed at the @@ -71,8 +71,8 @@ def http_req(res_num): @unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): @@ -81,8 +81,8 @@ def get_script_dir(cls): @unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): @@ -91,8 +91,8 @@ def get_script_dir(cls): @unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintDiffDirStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): diff --git a/tests/extension_tests/http_v2_tests/test_http_v2.py b/tests/extension_tests/http_v2_tests/test_http_v2.py index 670b5b54a..8c1d5b48e 100644 --- a/tests/extension_tests/http_v2_tests/test_http_v2.py +++ b/tests/extension_tests/http_v2_tests/test_http_v2.py @@ -9,7 +9,7 @@ import requests from tests.utils import testutils from azure_functions_worker.utils.common import is_envvar_true -from tests.utils.constants import CONSUMPTION_DOCKER_TEST,DEDICATED_DOCKER_TEST +from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST from azure_functions_worker.constants import PYTHON_ENABLE_INIT_INDEXING @@ -17,8 +17,8 @@ @unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") @unittest.skipIf(sys.version_info.minor < 8, "HTTPv2" "is only supported for 3.8+.") class TestHttpFunctionsWithInitIndexing(testutils.WebHostTestCase): From a150fe7ab21d2af9c402b68e4f12b04285b486c5 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 19 Dec 2024 14:27:21 -0600 Subject: [PATCH 24/25] feedback --- eng/templates/jobs/ci-emulator-tests.yml | 2 + .../utils/servicebus/docker-compose.yml | 4 +- .../get_servicebus_triggered/__init__.py | 8 -- .../get_servicebus_triggered/function.json | 23 ------ .../put_message/__init__.py | 9 --- .../put_message/function.json | 24 ------ .../function_app.py | 73 ----------------- .../generic/function_app.py | 81 ------------------- .../servicebus_trigger/__init__.py | 46 ----------- .../servicebus_trigger/function.json | 20 ----- tests/endtoend/test_servicebus_functions.py | 66 --------------- .../test_worker_process_count_functions.py | 26 +++--- 12 files changed, 17 insertions(+), 365 deletions(-) delete mode 100644 tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py delete mode 100644 tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json delete mode 100644 tests/endtoend/servicebus_functions/put_message/__init__.py delete mode 100644 tests/endtoend/servicebus_functions/put_message/function.json delete mode 100644 tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py delete mode 100644 tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py delete mode 100644 tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py delete mode 100644 tests/endtoend/servicebus_functions/servicebus_trigger/function.json delete mode 100644 tests/endtoend/test_servicebus_functions.py diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index 498eb1ea0..667ee6549 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -89,6 +89,8 @@ jobs: docker container rm --force microsoft-azure-eventhub-emulator docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml pull docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml up -d + env: + AzureWebJobsSQLPassword: $(AzureWebJobsSQLPassword) displayName: 'Install Azurite and Start ServiceBus Emulator' - bash: | python -m pytest -q -n auto --dist loadfile --reruns 4 tests/emulator_tests/test_servicebus_functions.py diff --git a/tests/emulator_tests/utils/servicebus/docker-compose.yml b/tests/emulator_tests/utils/servicebus/docker-compose.yml index 9c6d92216..c1781a858 100644 --- a/tests/emulator_tests/utils/servicebus/docker-compose.yml +++ b/tests/emulator_tests/utils/servicebus/docker-compose.yml @@ -10,7 +10,7 @@ services: - "5672:5672" environment: SQL_SERVER: sqledge - MSSQL_SA_PASSWORD: Password123! + MSSQL_SA_PASSWORD: ${AzureWebJobsSQLPassword} ACCEPT_EULA: Y depends_on: - sqledge @@ -27,7 +27,7 @@ services: - "sqledge" environment: ACCEPT_EULA: Y - MSSQL_SA_PASSWORD: Password123! + MSSQL_SA_PASSWORD: ${AzureWebJobsSQLPassword} # Service for the Azurite Storage Emulator azurite: container_name: "azurite-sb" diff --git a/tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py b/tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py deleted file mode 100644 index 1e5ac3d90..000000000 --- a/tests/endtoend/servicebus_functions/get_servicebus_triggered/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import azure.functions as func - - -def main(req: func.HttpRequest, file: func.InputStream) -> str: - return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') diff --git a/tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json b/tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json deleted file mode 100644 index 944dd45a9..000000000 --- a/tests/endtoend/servicebus_functions/get_servicebus_triggered/function.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "type": "httpTrigger", - "direction": "in", - "name": "req", - "authLevel": "anonymous" - }, - { - "type": "blob", - "direction": "in", - "name": "file", - "connection": "AzureWebJobsStorage", - "path": "python-worker-tests/test-servicebus-triggered.txt" - }, - { - "type": "http", - "direction": "out", - "name": "$return", - } - ] -} diff --git a/tests/endtoend/servicebus_functions/put_message/__init__.py b/tests/endtoend/servicebus_functions/put_message/__init__.py deleted file mode 100644 index 85ad99bf7..000000000 --- a/tests/endtoend/servicebus_functions/put_message/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import azure.functions as azf - - -def main(req: azf.HttpRequest, msg: azf.Out[str]): - msg.set(req.get_body().decode('utf-8')) - - return 'OK' diff --git a/tests/endtoend/servicebus_functions/put_message/function.json b/tests/endtoend/servicebus_functions/put_message/function.json deleted file mode 100644 index 722f19541..000000000 --- a/tests/endtoend/servicebus_functions/put_message/function.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "scriptFile": "__init__.py", - - "bindings": [ - { - "type": "httpTrigger", - "direction": "in", - "name": "req", - "authLevel": "anonymous" - }, - { - "direction": "out", - "name": "msg", - "queueName": "testqueue", - "connection": "AzureWebJobsServiceBusConnectionString", - "type": "serviceBus" - }, - { - "direction": "out", - "name": "$return", - "type": "http" - } - ] -} diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py b/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py deleted file mode 100644 index 4064085be..000000000 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/function_app.py +++ /dev/null @@ -1,73 +0,0 @@ -import json - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.route(route="put_message") -@app.service_bus_queue_output( - arg_name="msg", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -def put_message(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body().decode('utf-8')) - return 'OK' - - -@app.route(route="get_servicebus_triggered") -@app.blob_input(arg_name="file", - path="python-worker-tests/test-servicebus-triggered.txt", - connection="AzureWebJobsStorage") -def get_servicebus_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') - - -@app.service_bus_queue_trigger( - arg_name="msg", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.blob_output(arg_name="$return", - path="python-worker-tests/test-servicebus-triggered.txt", - connection="AzureWebJobsStorage") -def servicebus_trigger(msg: func.ServiceBusMessage) -> str: - result = json.dumps({ - 'message_id': msg.message_id, - 'body': msg.get_body().decode('utf-8'), - 'content_type': msg.content_type, - 'delivery_count': msg.delivery_count, - 'expiration_time': (msg.expiration_time.isoformat() if - msg.expiration_time else None), - 'label': msg.label, - 'partition_key': msg.partition_key, - 'reply_to': msg.reply_to, - 'reply_to_session_id': msg.reply_to_session_id, - 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if - msg.scheduled_enqueue_time else None), - 'session_id': msg.session_id, - 'time_to_live': msg.time_to_live, - 'to': msg.to, - 'user_properties': msg.user_properties, - - 'application_properties': msg.application_properties, - 'correlation_id': msg.correlation_id, - 'dead_letter_error_description': msg.dead_letter_error_description, - 'dead_letter_reason': msg.dead_letter_reason, - 'dead_letter_source': msg.dead_letter_source, - 'enqueued_sequence_number': msg.enqueued_sequence_number, - 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if - msg.enqueued_time_utc else None), - 'expires_at_utc': (msg.expires_at_utc.isoformat() if - msg.expires_at_utc else None), - 'locked_until': (msg.locked_until.isoformat() if - msg.locked_until else None), - 'lock_token': msg.lock_token, - 'sequence_number': msg.sequence_number, - 'state': msg.state, - 'subject': msg.subject, - 'transaction_partition_key': msg.transaction_partition_key - }) - - return result diff --git a/tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py b/tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py deleted file mode 100644 index 3b340153a..000000000 --- a/tests/endtoend/servicebus_functions/servicebus_functions_stein/generic/function_app.py +++ /dev/null @@ -1,81 +0,0 @@ -import json - -import azure.functions as func - -app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) - - -@app.function_name(name="put_message") -@app.generic_trigger(arg_name="req", type="httpTrigger", route="put_message") -@app.generic_output_binding(arg_name="msg", - type="serviceBus", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.generic_output_binding(arg_name="$return", type="http") -def put_message(req: func.HttpRequest, msg: func.Out[str]): - msg.set(req.get_body().decode('utf-8')) - return 'OK' - - -@app.function_name(name="get_servicebus_triggered") -@app.generic_trigger(arg_name="req", type="httpTrigger", - route="get_servicebus_triggered") -@app.generic_input_binding(arg_name="file", - type="blob", - path="python-worker-tests/test-servicebus-triggered.txt", # NoQA - connection="AzureWebJobsStorage") -@app.generic_output_binding(arg_name="$return", type="http") -def get_servicebus_triggered(req: func.HttpRequest, - file: func.InputStream) -> str: - return func.HttpResponse( - file.read().decode('utf-8'), mimetype='application/json') - - -@app.generic_trigger( - arg_name="msg", - type="serviceBusTrigger", - connection="AzureWebJobsServiceBusConnectionString", - queue_name="testqueue") -@app.generic_output_binding(arg_name="$return", - path="python-worker-tests/test-servicebus-triggered.txt", # NoQA - type="blob", - connection="AzureWebJobsStorage") -def servicebus_trigger(msg: func.ServiceBusMessage) -> str: - result = json.dumps({ - 'message_id': msg.message_id, - 'body': msg.get_body().decode('utf-8'), - 'content_type': msg.content_type, - 'delivery_count': msg.delivery_count, - 'expiration_time': (msg.expiration_time.isoformat() if - msg.expiration_time else None), - 'label': msg.label, - 'partition_key': msg.partition_key, - 'reply_to': msg.reply_to, - 'reply_to_session_id': msg.reply_to_session_id, - 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if - msg.scheduled_enqueue_time else None), - 'session_id': msg.session_id, - 'time_to_live': msg.time_to_live, - 'to': msg.to, - 'user_properties': msg.user_properties, - - 'application_properties': msg.application_properties, - 'correlation_id': msg.correlation_id, - 'dead_letter_error_description': msg.dead_letter_error_description, - 'dead_letter_reason': msg.dead_letter_reason, - 'dead_letter_source': msg.dead_letter_source, - 'enqueued_sequence_number': msg.enqueued_sequence_number, - 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if - msg.enqueued_time_utc else None), - 'expires_at_utc': (msg.expires_at_utc.isoformat() if - msg.expires_at_utc else None), - 'locked_until': (msg.locked_until.isoformat() if - msg.locked_until else None), - 'lock_token': msg.lock_token, - 'sequence_number': msg.sequence_number, - 'state': msg.state, - 'subject': msg.subject, - 'transaction_partition_key': msg.transaction_partition_key - }) - - return result diff --git a/tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py b/tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py deleted file mode 100644 index 341779a4d..000000000 --- a/tests/endtoend/servicebus_functions/servicebus_trigger/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import json - -import azure.functions as azf - - -def main(msg: azf.ServiceBusMessage) -> str: - result = json.dumps({ - 'message_id': msg.message_id, - 'body': msg.get_body().decode('utf-8'), - 'content_type': msg.content_type, - 'delivery_count': msg.delivery_count, - 'expiration_time': (msg.expiration_time.isoformat() if - msg.expiration_time else None), - 'label': msg.label, - 'partition_key': msg.partition_key, - 'reply_to': msg.reply_to, - 'reply_to_session_id': msg.reply_to_session_id, - 'scheduled_enqueue_time': (msg.scheduled_enqueue_time.isoformat() if - msg.scheduled_enqueue_time else None), - 'session_id': msg.session_id, - 'time_to_live': msg.time_to_live, - 'to': msg.to, - 'user_properties': msg.user_properties, - - 'application_properties': msg.application_properties, - 'correlation_id': msg.correlation_id, - 'dead_letter_error_description': msg.dead_letter_error_description, - 'dead_letter_reason': msg.dead_letter_reason, - 'dead_letter_source': msg.dead_letter_source, - 'enqueued_sequence_number': msg.enqueued_sequence_number, - 'enqueued_time_utc': (msg.enqueued_time_utc.isoformat() if - msg.enqueued_time_utc else None), - 'expires_at_utc': (msg.expires_at_utc.isoformat() if - msg.expires_at_utc else None), - 'locked_until': (msg.locked_until.isoformat() if - msg.locked_until else None), - 'lock_token': msg.lock_token, - 'sequence_number': msg.sequence_number, - 'state': msg.state, - 'subject': msg.subject, - 'transaction_partition_key': msg.transaction_partition_key - }) - - return result diff --git a/tests/endtoend/servicebus_functions/servicebus_trigger/function.json b/tests/endtoend/servicebus_functions/servicebus_trigger/function.json deleted file mode 100644 index b6fe4355e..000000000 --- a/tests/endtoend/servicebus_functions/servicebus_trigger/function.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "scriptFile": "__init__.py", - - "bindings": [ - { - "type": "serviceBusTrigger", - "direction": "in", - "name": "msg", - "queueName": "testqueue", - "connection": "AzureWebJobsServiceBusConnectionString", - }, - { - "type": "blob", - "direction": "out", - "name": "$return", - "connection": "AzureWebJobsStorage", - "path": "python-worker-tests/test-servicebus-triggered.txt" - } - ] -} diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py deleted file mode 100644 index aaacd76d6..000000000 --- a/tests/endtoend/test_servicebus_functions.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -import json -import time - -from tests.utils import testutils - - -class TestServiceBusFunctions(testutils.WebHostTestCase): - - @classmethod - def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' - - @testutils.retryable_test(3, 5) - def test_servicebus_basic(self): - data = str(round(time.time())) - r = self.webhost.request('POST', 'put_message', - data=data) - self.assertEqual(r.status_code, 200) - self.assertEqual(r.text, 'OK') - - max_retries = 10 - - for try_no in range(max_retries): - # wait for trigger to process the queue item - time.sleep(1) - - try: - r = self.webhost.request('GET', 'get_servicebus_triggered') - self.assertEqual(r.status_code, 200) - msg = r.json() - self.assertEqual(msg['body'], data) - for attr in {'message_id', 'body', 'content_type', 'delivery_count', - 'expiration_time', 'label', 'partition_key', 'reply_to', - 'reply_to_session_id', 'scheduled_enqueue_time', - 'session_id', 'time_to_live', 'to', 'user_properties', - 'application_properties', 'correlation_id', - 'dead_letter_error_description', 'dead_letter_reason', - 'dead_letter_source', 'enqueued_sequence_number', - 'enqueued_time_utc', 'expires_at_utc', 'locked_until', - 'lock_token', 'sequence_number', 'state', 'subject', - 'transaction_partition_key'}: - self.assertIn(attr, msg) - except (AssertionError, json.JSONDecodeError): - if try_no == max_retries - 1: - raise - else: - break - - -class TestServiceBusFunctionsStein(TestServiceBusFunctions): - - @classmethod - def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' - - -class TestServiceBusFunctionsSteinGeneric(TestServiceBusFunctions): - - @classmethod - def get_script_dir(cls): - return testutils.E2E_TESTS_FOLDER / 'servicebus_functions' / \ - 'servicebus_functions_stein' / \ - 'generic' diff --git a/tests/endtoend/test_worker_process_count_functions.py b/tests/endtoend/test_worker_process_count_functions.py index b1277edec..44abcd2e2 100644 --- a/tests/endtoend/test_worker_process_count_functions.py +++ b/tests/endtoend/test_worker_process_count_functions.py @@ -1,19 +1,19 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os -import unittest from datetime import datetime from threading import Thread +from unittest import skipIf from tests.utils import testutils from azure_functions_worker.utils.common import is_envvar_true from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCount(testutils.WebHostTestCase): """Test the Http Trigger with setting up the python worker process count to 2. this test will check if both requests should be processed at the @@ -70,9 +70,9 @@ def http_req(res_num): self.assertTrue(time_diff_in_seconds < 1) -@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): @@ -80,9 +80,9 @@ def get_script_dir(cls): 'http_functions_stein' -@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): @@ -90,9 +90,9 @@ def get_script_dir(cls): 'functions_in_blueprint_only' -@unittest.skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), - "Tests are flaky when running on Docker") +@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) + or is_envvar_true(CONSUMPTION_DOCKER_TEST), + "Tests are flaky when running on Docker") class TestWorkerProcessCountWithBlueprintDiffDirStein(TestWorkerProcessCount): @classmethod def get_script_dir(cls): From 463d4ebf6421da3f7e53a1838370391479c4fae7 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 19 Dec 2024 14:51:00 -0600 Subject: [PATCH 25/25] container name --- eng/templates/jobs/ci-emulator-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/templates/jobs/ci-emulator-tests.yml b/eng/templates/jobs/ci-emulator-tests.yml index 667ee6549..d2ab3ce87 100644 --- a/eng/templates/jobs/ci-emulator-tests.yml +++ b/eng/templates/jobs/ci-emulator-tests.yml @@ -85,8 +85,8 @@ jobs: displayName: "Running $(PYTHON_VERSION) Python Linux Emulator Tests" - bash: | # Stop and remove EventHub Emulator container to free up the port - docker stop microsoft-azure-eventhub-emulator - docker container rm --force microsoft-azure-eventhub-emulator + docker stop eventhubs-emulator + docker container rm --force eventhubs-emulator docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml pull docker compose -f tests/emulator_tests/utils/servicebus/docker-compose.yml up -d env: